From 218b5ec0fa6a3d8411adf43e692cc6e0ab7238a1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 9 Sep 2026 15:30:37 +0100 Subject: [PATCH 1/7] FEAT: Read the Terms a 51Did was created under The 51Did payload gains one byte after the match key, the Terms, which says which terms document the identifier was created under so that the terms travel with the identifier instead of alongside it. The byte is an index into a table in the specification and not a version number, so a later document can live at any address, and an index is never reused or repointed once published because an identifier issued under it has to stay readable years later. Three members are added to FodId, being terms as a named value, the raw terms_index, and terms_url which is the address or None. A new Terms enumeration in terms.py carries the vocabulary and the addresses, the same shape as the existing Usage and IdType. Terms.NOT_STATED and Terms.UNKNOWN are different answers. NOT_STATED is index 0 and says the identifier does not carry the answer, so it has to come from the surrounding protocol, whilst UNKNOWN says the identifier does state terms this package cannot name because the index was added after the package was released. Reading UNKNOWN as NOT_STATED would read an identifier created under terms as one created under none, so the two are separate members and terms_index is exposed to let a caller say which index it could not read. Neither has an address, and the package answers with an address and never fetches it. Existing identifiers are unaffected. A payload ending at the match key has no byte to read and answers index 0, which is the same answer as a zero byte, so absence and zero mean the same thing and no presence flag exists. No field before the Terms moves and no existing test changed. A Reserved type has no defined match key length and keeps its documented best-effort reading, taking every byte after the header as its match key, so it leaves no byte to read as the Terms and answers index 0. Depends on 51Degrees/specifications#27 and must not merge before it. --- .../examples/fodid_example.py | 21 +- fiftyone_pipeline_did/readme.md | 75 +++++- .../src/fiftyone_pipeline_did/__init__.py | 5 +- .../src/fiftyone_pipeline_did/_layout.py | 10 + .../src/fiftyone_pipeline_did/fod_id.py | 117 ++++++-- .../src/fiftyone_pipeline_did/terms.py | 104 ++++++++ fiftyone_pipeline_did/tests/envelope.py | 11 +- fiftyone_pipeline_did/tests/test_fodid.py | 251 ++++++++++++++++++ 8 files changed, 548 insertions(+), 46 deletions(-) create mode 100644 fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py diff --git a/fiftyone_pipeline_did/examples/fodid_example.py b/fiftyone_pipeline_did/examples/fodid_example.py index 7cbd716..2e92235 100644 --- a/fiftyone_pipeline_did/examples/fodid_example.py +++ b/fiftyone_pipeline_did/examples/fodid_example.py @@ -24,8 +24,8 @@ The 51Degrees Cloud service issues real 51Dids. To keep this example self-contained and offline, it builds a sample 51Did in process - generate an -ECDSA P-256 key pair, sign a canonical 37-byte payload - then parses it back -and prints the three payload fields. It also shows the headline use case: a +ECDSA P-256 key pair, sign a canonical 38-byte payload - then parses it back +and prints the payload fields. It also shows the headline use case: a 51Did is re-issued fresh on every call (the envelope, hence the base64, changes), but the match key is stable. Compare match keys, never envelopes. @@ -43,19 +43,25 @@ # specified at # https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md SAMPLE_FLAGS = 0b0000_0011 # standard usage, Probabilistic type -SAMPLE_PAYLOAD_LENGTH = 37 # 1 flags byte, 4 licence id, 32 key +SAMPLE_PAYLOAD_LENGTH = 38 # 1 flags byte, 4 licence id, 32 key, + # 1 terms SAMPLE_MATCH_KEY_OFFSET = 5 SAMPLE_MATCH_KEY_LENGTH = 32 +SAMPLE_TERMS_OFFSET = 37 # the Terms byte follows the match key +SAMPLE_TERMS = 1 # index 1, the Model Terms for Marketing + # version 2 def sample_payload(): - """A canonical 37-byte Probabilistic payload: the flags byte, License - Id 0x12345678 (little-endian) and a 32-byte match key 0x20..0x3F.""" + """A canonical 38-byte Probabilistic payload: the flags byte, License + Id 0x12345678 (little-endian), a 32-byte match key 0x20..0x3F and the + Terms byte.""" payload = bytearray(SAMPLE_PAYLOAD_LENGTH) payload[0] = SAMPLE_FLAGS payload[1:5] = bytes([0x78, 0x56, 0x34, 0x12]) for i in range(SAMPLE_MATCH_KEY_LENGTH): payload[SAMPLE_MATCH_KEY_OFFSET + i] = 0x20 + i + payload[SAMPLE_TERMS_OFFSET] = SAMPLE_TERMS return bytes(payload) @@ -83,6 +89,11 @@ def run(): print(" Consent :", fod_id.usage_from_consent) print(" LicenseId :", fod_id.license_id) print(" Match key :", fod_id.match_key.hex()) + print(" Terms :", fod_id.terms.name) + print(" Terms idx :", fod_id.terms_index) + # The address is answered and never fetched. What to do with the + # document is the receiver's decision. + print(" Terms url :", fod_id.terms_url) print(" Verifies :", fod_id.verify(crypto.public_key_pem())) reissued = FodId.from_base64(issue(creator, payload)) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index d9dbc1d..94c1e86 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -14,6 +14,9 @@ Identifier) returned by the 51Degrees Cloud service. Mirrors the .NET the Flags and License Id, being a 32-byte SHA-256 for Probabilistic and HashedEmail identifiers, or 16 GUID bytes for Random. Two 51Dids for the same inputs share the same match key even though their envelopes differ. +- The **Terms** is the byte after the match key that says which terms + document the identifier was created under, so the terms travel with the + identifier instead of alongside it. **Comparing two 51Dids means comparing their match keys, never their envelopes.** @@ -31,8 +34,11 @@ where the two ever disagree. In short, the payload opens with a header carrying the flags byte and the License Id, and the identifier type in that byte then fixes the length of the match key that follows, being 32 bytes for `PROBABILISTIC` and -`HASHED_EMAIL`, 16 for `RANDOM`, and whatever remains for `RESERVED`. -Identifiers issued before the type tag existed decode as `PROBABILISTIC`. +`HASHED_EMAIL`, 16 for `RANDOM`, and whatever remains for `RESERVED`. The +Terms byte follows the match key, and the bytes after it are a creator +context section. Identifiers issued before the type tag existed decode as +`PROBABILISTIC`, and identifiers issued before the Terms existed end at +the match key and read as terms that are not stated. This package does not publish the offsets or the raw flags byte, and it does not need to, because every field has a typed accessor that reads it @@ -83,7 +89,7 @@ way to hold an unsigned or partly built envelope. ## Usage ```python -from fiftyone_pipeline_did import FodId, IdType, Usage +from fiftyone_pipeline_did import FodId, IdType, Terms, Usage fod_id = FodId.from_base64(base64_from_cloud_service) # either alphabet @@ -93,6 +99,11 @@ from_consent = fod_id.usage_from_consent # True when read from a consent # string the caller sent license_id = fod_id.license_id match_key = fod_id.match_key # SHA-256 or GUID bytes, see type +terms = fod_id.terms # Terms.NOT_STATED / Terms.UNKNOWN, or a + # named document, being + # MODEL_TERMS_FOR_MARKETING_VERSION_2 +terms_index = fod_id.terms_index # the raw byte, 0 to 255 +terms_url = fod_id.terms_url # the address, or None where there is none # Delegated OWID-level fields and operations. domain = fod_id.domain @@ -130,6 +141,44 @@ from an IAB consent string the caller sent rather than stated by the caller directly. Both are legitimate ways to arrive at a usage and it says nothing about which usage was reached. +### The terms an identifier was created under + +`fod_id.terms` says which terms document the identifier was created under, +as a `Terms`, so the terms travel with the identifier instead of alongside +it and a receiver can tell which document was in force when the identifier +was made. `fod_id.terms_url` gives the address of that document, and +`fod_id.terms_index` gives the raw byte behind both, being an index into a +table of terms documents in the +[layout specification](https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md). +The byte is an index and not a version number, so that a later document +can live at any address rather than only at one the specification could +compose from a number, and an index is never reused or repointed once +published, because an identifier issued under it has to stay readable +years later. + +`Terms.NOT_STATED` and `Terms.UNKNOWN` are different answers and must +never be read as the same one. `NOT_STATED` is index 0 and says this +identifier does not carry the answer, so the answer has to come from +somewhere else, being the Terms Document Locator in an OpenRTB request or +whatever the surrounding protocol provides, and it does not mean the +identifier is unrestricted. `UNKNOWN` says the identifier does state its +terms and that this package cannot name them, because the index was added +after the package was released. A caller meeting `UNKNOWN` should treat +the identifier as covered by terms it cannot yet read, and either update +the package or refuse the identifier, and `terms_index` is there so it can +say which index it could not read. + +`terms_url` is `None` for both of those, using absence rather than an +empty string, and this package answers with the address and never fetches +it, because what to do with the document is the receiver's decision. + +An identifier issued before the Terms existed has a payload ending at the +match key, and a payload with no byte after the match key reads as index +0, so absence and zero mean the same thing and no presence flag exists to +tell them apart. The Usage and the Terms answer different questions and a +receiver needs both, because the Usage says where an identifier may go and +the Terms says under which document it was created. + The raw flags byte, the byte layout constants and the old `hash` names are not part of this package. `fod_id.flags`, `fod_id.hash`, `fod_id.date_minutes`, `FodId.MATCH_KEY_OFFSET` and every other offset and @@ -208,14 +257,17 @@ same whichever language parsed the bytes. The payload must hold the 5 byte header before the type can be read, and the type then says how many match key bytes must follow, being 16 for `RANDOM` and 32 for `PROBABILISTIC` and `HASHED_EMAIL`, as the payload -layout specification says. `RESERVED` keeps the best-effort reading, being -the header fields and whatever bytes follow. Anything beyond the match key +layout specification says. The Terms byte follows the match key, and a +payload ending at the match key reads as terms that are not stated. +`RESERVED` keeps the best-effort reading, being the header fields and +whatever bytes follow, and because no match key length is defined for it +there is no byte left over to read as its Terms. Anything beyond the Terms is a creator context section whose lengths belong to the cloud, so a longer payload, a longer creator domain (a self-hosted container may sign with one) or a longer envelope is accepted and this package places no upper bound of its own on any of them. An older reader meeting a context -section of a version it does not know still reads the header and the -match key. +section of a version it does not know still reads the header, the match +key and the Terms. `DidClient` refuses text longer than 4096 characters before it parses it, fetches a key or calls the cloud. That figure is client policy, @@ -533,9 +585,12 @@ is refreshed by common-ci's `update-example-assets` step. - **No signature verification on parsing.** A parsed 51Did is not known to be genuine. Call `verify(public_key_pem)`, `signature_status(public_key_pem)` or a `DidClient` check when needed. +- **No fetching of a terms document.** `fod_id.terms_url` answers with the + address and nothing more. What to do with the document is the receiver's + decision. - **No upper bound on the size of an identifier.** The lengths beyond the - header and match key belong to the cloud. The 4096 character figure in - `DidClient` is client policy against obviously malformed text, not a - format limit. + header, match key and Terms belong to the cloud. The 4096 character + figure in `DidClient` is client policy against obviously malformed text, + not a format limit. - **No creation of new 51Dids.** This is a parser; new 51Dids are issued by the 51Degrees cloud / on-premise hashing engines. diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py index ccd920e..4671a6d 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py @@ -27,7 +27,8 @@ OWID form in either alphabet, exposes a typed accessor for every field it carries (the identifier :class:`~fiftyone_pipeline_did.id_type.IdType`, the :class:`~fiftyone_pipeline_did.usage.Usage` it was created for and whether -that usage came from a consent string, the License Id and the match key), +that usage came from a consent string, the License Id, the match key and +the :class:`~fiftyone_pipeline_did.terms.Terms` it was created under), and delegates OWID-level concerns to the wrapped envelope. The raw bytes and offsets behind those accessors are not part of this surface, which is specified at @@ -72,6 +73,7 @@ from ._owid import Owid, OwidError, SignatureStatus from .fod_id import DATE_EPOCH, FodId, FodIdParseResult, FodIdParseStatus from .id_type import IdType +from .terms import Terms from .usage import Usage __all__ = [ @@ -79,6 +81,7 @@ "FodIdParseResult", "FodIdParseStatus", "IdType", + "Terms", "Usage", "DATE_EPOCH", "DidClient", diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py index c55ad94..a4f3518 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py @@ -61,3 +61,13 @@ RANDOM_PAYLOAD_LENGTH = HEADER_LENGTH + GUID_LENGTH #: Minimum byte length of a Probabilistic or HashedEmail 51Did payload. PAYLOAD_LENGTH = MATCH_KEY_OFFSET + MATCH_KEY_LENGTH +#: Byte length of the Terms field, which follows the match key. There is no +#: offset constant for it, because the identifier type fixes the match key +#: length and so the type says where the field starts. +TERMS_LENGTH = 1 +#: The Terms index a payload with no byte after the match key reads as, +#: being the index that says the terms are not stated in the identifier. An +#: identifier issued before the Terms existed ends at the match key, so +#: absence and zero mean the same thing and neither has to be told apart +#: from the other. +ABSENT_TERMS_INDEX = 0 diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index 118608e..bea04f9 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -27,6 +27,7 @@ from typing import NamedTuple, Optional, Tuple from ._layout import ( + ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, HEADER_LENGTH, @@ -34,6 +35,7 @@ LICENSE_ID_OFFSET, MATCH_KEY_LENGTH, MATCH_KEY_OFFSET, + TERMS_LENGTH, ) from ._owid import ( Owid, @@ -45,6 +47,7 @@ ) from .id_type import IdType +from .terms import Terms from .usage import Usage #: The moment the envelope's date field counts minutes from, being the OWID @@ -166,7 +169,8 @@ class FodId: Payload layout. Every field has a typed accessor here, being :attr:`type`, :attr:`usage`, :attr:`usage_from_consent`, - :attr:`license_id` and :attr:`match_key`, and those accessors are the + :attr:`license_id`, :attr:`match_key`, :attr:`terms`, + :attr:`terms_index` and :attr:`terms_url`, and those accessors are the supported way to read an identifier. The bytes and offsets behind them are specified at https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md @@ -176,10 +180,12 @@ class FodId: which are the authority for both. In short, the header is shared by every identifier type and the type then fixes the length of the match key that follows, being a 32-byte SHA-256 for Probabilistic and - HashedEmail or 16 GUID bytes for Random. A payload longer than the - header and match key is accepted, because the bytes after the match key - are a creator context section whose lengths belong to the cloud, so - this package places no upper bound on a payload or an envelope. + HashedEmail or 16 GUID bytes for Random, and the Terms byte follows the + match key. A payload longer than that is accepted, because the bytes + after the Terms are a creator context section whose lengths belong to + the cloud, so this package places no upper bound on a payload or an + envelope. A payload that ends at the match key was issued before the + Terms existed and reads as terms that are not stated. Reading and verifying are separate steps. :meth:`try_from_base64` and :meth:`try_from_byte_array` read external data without raising and @@ -217,19 +223,20 @@ def __init__(self, owid: Owid) -> None: self._assign(read.owid, *_unpack_or_raise(read.owid.payload)) def _assign(self, owid: Owid, flags: int, license_id: int, - match_key: bytes) -> None: + match_key: bytes, terms_index: int) -> None: self._owid = owid self._flags = flags self._license_id = license_id self._match_key = match_key + self._terms_index = terms_index @classmethod def _build(cls, owid: Owid, flags: int, license_id: int, - match_key: bytes) -> "FodId": + match_key: bytes, terms_index: int) -> "FodId": """An identifier over fields :func:`_read_payload` has already checked, so the constructor's read is not repeated.""" fod_id = cls.__new__(cls) - fod_id._assign(owid, flags, license_id, match_key) + fod_id._assign(owid, flags, license_id, match_key, terms_index) return fod_id @classmethod @@ -239,12 +246,13 @@ def _from_read(cls, read: ParseResult) -> FodIdParseResult: builds the identifier only when both have passed.""" if not read.ok: return _failed(FodIdParseStatus.of(read.status)) - status, flags, license_id, match_key = _read_payload( + status, flags, license_id, match_key, terms_index = _read_payload( read.owid.payload) if status is not FodIdParseStatus.PARSED: return _failed(status) return FodIdParseResult( - True, cls._build(read.owid, flags, license_id, match_key), + True, + cls._build(read.owid, flags, license_id, match_key, terms_index), FodIdParseStatus.PARSED) @classmethod @@ -411,6 +419,45 @@ def match_key(self) -> bytes: """ return self._match_key + @property + def terms(self) -> Terms: + """The terms document the identifier was created under, so the + terms travel with the identifier instead of alongside it. See + :class:`~fiftyone_pipeline_did.Terms`, which sets out why terms + that are not stated and terms this package cannot name are + different answers and must never be read as the same one.""" + return Terms.from_index(self._terms_index) + + @property + def terms_index(self) -> int: + """The raw value of the Terms byte (0 to 255), being the index into + the table of terms documents in the specification. + + A payload that ends at the match key was issued before the Terms + existed and reads as 0, which says the terms are not stated in the + identifier, so absence and zero mean the same thing. The index is + exposed because a caller will meet one added after this package was + released, and it can then name which index it could not read, or + look the document up by hand, neither of which :attr:`terms` alone + allows. + """ + return self._terms_index + + @property + def terms_url(self) -> Optional[str]: + """The address of the terms document, and ``None`` where the terms + are not stated in the identifier or where the index is one this + package does not know. + + The address is answered and never fetched, because what to do with + the document is the receiver's decision. ``None`` for an index this + package does not know says only that the document cannot be named + here, and never that there is no document, which is what + :attr:`terms` and :attr:`terms_index` are read together to tell + apart. + """ + return self.terms.url + @property def version(self) -> Version: """The OWID version.""" @@ -496,25 +543,27 @@ def _date_minutes(fod_id: "FodId") -> int: return int((fod_id.date - DATE_EPOCH).total_seconds() // 60) -def _read_payload(payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes]: - """Applies the two 51Did payload rules and unpacks the three fields. +def _read_payload( + payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes, int]: + """Applies the two 51Did payload rules and unpacks the four fields. The header must be present before the type can be read, and the type - then says how many match key bytes must follow. Anything beyond the - match key is a creator context section whose lengths belong to the - cloud, so a longer payload passes. A Reserved type has no known match - key length and keeps the documented best-effort reading, being the - header fields and whatever bytes follow. - - Returns the status and, on success, the flags, the licence id and the - match key bytes. On failure the three fields are zero and empty. + then says how many match key bytes must follow. The Terms byte follows + the match key, and anything beyond it is a creator context section + whose lengths belong to the cloud, so a longer payload passes. A + Reserved type has no known match key length and keeps the documented + best-effort reading, being the header fields and whatever bytes follow. + + Returns the status and, on success, the flags, the licence id, the + match key bytes and the Terms index. On failure the four fields are + zero and empty. """ if payload is None or len(payload) < HEADER_LENGTH: - return FodIdParseStatus.PAYLOAD_TOO_SHORT, 0, 0, b"" + return FodIdParseStatus.PAYLOAD_TOO_SHORT, 0, 0, b"", 0 flags = payload[FLAGS_OFFSET] match_key_length = _match_key_length(IdType.from_flags(flags), payload) if len(payload) < HEADER_LENGTH + match_key_length: - return FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, 0, 0, b"" + return FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, 0, 0, b"", 0 # Little-endian uint32, unsigned (Python ints are unbounded and # non-negative here, so the high bit never becomes negative). license_id = int.from_bytes( @@ -526,16 +575,32 @@ def _read_payload(payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes]: # to change the underlying payload and no defensive copy is required. match_key = bytes( payload[MATCH_KEY_OFFSET:MATCH_KEY_OFFSET + match_key_length]) - return FodIdParseStatus.PARSED, flags, license_id, match_key + return (FodIdParseStatus.PARSED, flags, license_id, match_key, + _read_terms_index(payload, MATCH_KEY_OFFSET + match_key_length)) + + +def _read_terms_index(payload: bytes, offset: int) -> int: + """The Terms byte at the offset the match key ends at, and the index + that says the terms are not stated where the payload ends there. + + An identifier issued before the Terms existed has a payload ending at + the match key, so absence and zero mean the same thing and neither has + to be told apart from the other. A Reserved type takes every byte after + the header as its match key, since no length is defined for it, and so + leaves nothing here to read. + """ + if len(payload) < offset + TERMS_LENGTH: + return ABSENT_TERMS_INDEX + return payload[offset] -def _unpack_or_raise(payload: bytes) -> Tuple[int, int, bytes]: +def _unpack_or_raise(payload: bytes) -> Tuple[int, int, bytes, int]: """The payload rules for the raising readers, with the messages they have always given.""" - status, flags, license_id, match_key = _read_payload(payload) + status, flags, license_id, match_key, terms_index = _read_payload(payload) if status is not FodIdParseStatus.PARSED: raise ValueError(_payload_message(status, payload)) - return flags, license_id, match_key + return flags, license_id, match_key, terms_index def _match_key_length(id_type: IdType, payload: bytes) -> int: diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py new file mode 100644 index 0000000..1cd7a71 --- /dev/null +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py @@ -0,0 +1,104 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + + +from enum import Enum +from typing import Optional + + +class Terms(Enum): + """The terms document a 51Did was created under, read from the + identifier through :attr:`~fiftyone_pipeline_did.FodId.terms`, so that + the terms travel with the identifier instead of alongside it and a + receiver can tell which document was in force when the identifier was + made. + + The payload carries an index into a table of terms documents and not a + version number, so that a later document can live at any address rather + than only at one a number could be put into. That table is the whole of + the definition, so a new terms document is a new index there and every + package has to be released to know it, and an index is never reused or + repointed once published. It is specified at + https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md + + :attr:`NOT_STATED` and :attr:`UNKNOWN` are different answers and must + never be read as the same one. :attr:`NOT_STATED` says this identifier + does not carry the answer, so the answer has to come from somewhere + else, being the Terms Document Locator in an OpenRTB request or + whatever the surrounding protocol provides, and it does not mean the + identifier is unrestricted. :attr:`UNKNOWN` says the identifier does + state its terms and that this package cannot name them, because the + index was added after the package was released. A caller meeting + :attr:`UNKNOWN` should treat the identifier as covered by terms it + cannot yet read, and either update the package or refuse the + identifier. + + The members carry a name and not the index, because :attr:`UNKNOWN` + stands for any index this package does not know and so has no single + index to carry. Where a caller needs the index itself, for instance to + report which one it could not read, it reads + :attr:`~fiftyone_pipeline_did.FodId.terms_index`. + + The Usage and the Terms answer different questions and a receiver needs + both, because the Usage says where an identifier may go and the Terms + says under which document it was created.""" + + #: The terms are not stated in the identifier, which is index 0 and + #: also what an identifier issued before the Terms existed reads as, + #: its payload ending at the match key. + NOT_STATED = "NotStated" + #: Index 1, the Model Terms for Marketing version 2, at + #: https://m4ow.uk/mtm/2.txt + MODEL_TERMS_FOR_MARKETING_VERSION_2 = "ModelTermsForMarketingVersion2" + #: An index added after this package was released, so the identifier + #: states terms this package cannot name. Never treat it as + #: :attr:`NOT_STATED`, which would read an identifier created under + #: terms as one created under none. + UNKNOWN = "Unknown" + + @classmethod + def from_index(cls, index: int) -> "Terms": + """The member for a Terms index, and :attr:`UNKNOWN` for an index + this package does not know.""" + return _BY_INDEX.get(index, cls.UNKNOWN) + + @property + def url(self) -> Optional[str]: + """The address of the terms document, and ``None`` for + :attr:`NOT_STATED` and for :attr:`UNKNOWN`, where there is no + document this package can name. + + The address is answered and never fetched, because what to do with + the document is the receiver's decision.""" + return _URL[self] + + +_BY_INDEX = { + 0: Terms.NOT_STATED, + 1: Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, +} + +_URL = { + Terms.NOT_STATED: None, + Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2: "https://m4ow.uk/mtm/2.txt", + Terms.UNKNOWN: None, +} diff --git a/fiftyone_pipeline_did/tests/envelope.py b/fiftyone_pipeline_did/tests/envelope.py index 6049028..1d08335 100644 --- a/fiftyone_pipeline_did/tests/envelope.py +++ b/fiftyone_pipeline_did/tests/envelope.py @@ -38,6 +38,7 @@ # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md # says the package's own tests may. from fiftyone_pipeline_did._layout import ( + ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, LICENSE_ID_OFFSET, @@ -75,10 +76,12 @@ def random_payload(): def context_payload(): - """A Probabilistic payload followed by a creator context - section. How long a section is belongs to the cloud and changes with - the section version, so an arbitrary length is used here.""" - return probabilistic_payload() + bytes([0]) + bytes(range(1, 24)) + """A Probabilistic payload, a Terms byte saying the terms are not + stated in the identifier, and then a creator context section. How long + a section is belongs to the cloud and changes with the section version, + so an arbitrary length is used here.""" + return (probabilistic_payload() + bytes([ABSENT_TERMS_INDEX]) + + bytes(range(1, 24))) def envelope_bytes(crypto, payload, date=None, version=Version.VERSION3, diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 6536ba8..788e73d 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -33,6 +33,7 @@ FodIdParseStatus, IdType, OwidError, + Terms, Usage, SignatureStatus, ) @@ -42,6 +43,7 @@ # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md # says the package's own tests may. from fiftyone_pipeline_did._layout import ( + ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, HEADER_LENGTH, @@ -51,6 +53,7 @@ MATCH_KEY_OFFSET, PAYLOAD_LENGTH, RANDOM_PAYLOAD_LENGTH, + TERMS_LENGTH, ) from .envelope import envelope_bytes, signed_envelope @@ -83,6 +86,25 @@ def canonical_payload(): return bytearray(payload) +#: The address the specification gives for Terms index 1, the Model Terms +#: for Marketing version 2. +MODEL_TERMS_URL = "https://m4ow.uk/mtm/2.txt" +#: The Terms index of the Model Terms for Marketing version 2. +MODEL_TERMS_INDEX = 1 +#: An index the specification has not assigned, standing for one added +#: after this package was released. +UNKNOWN_TERMS_INDEX = 200 +#: A creator context section. How long a section is belongs to the cloud +#: and changes with the section version, so an arbitrary length is used. +CONTEXT_SECTION = bytes(range(1, 24)) + + +def with_terms(payload, index): + """The payload with a Terms byte after the match key, which is where + the specification puts it.""" + return bytearray(bytes(payload) + bytes([index])) + + def canonical_random_payload(): payload = bytearray(RANDOM_PAYLOAD_LENGTH) payload[FLAGS_OFFSET] = (1 << 6) | 0b001 # Random tag + usage bits @@ -774,6 +796,235 @@ def test_raising_and_non_raising_readers_agree_on_success(self): result.value.as_byte_array()) self.assertEqual(raising.match_key, result.value.match_key) +class FodIdTermsTests(unittest.TestCase): + """The Terms byte, which says which terms document the identifier was + created under so that the terms travel with the identifier. The byte is + an index into a table in the specification and not a version number, + and it follows the match key, so the identifier type fixes where it + sits. + """ + + def setUp(self): + self.factory = FodIdTestFactory() + + def _read(self, payload): + return FodId.from_base64(self.factory.signed_owid_base64(payload)) + + # ----- A payload that ends at the match key ----- + + def test_payload_ending_at_the_match_key_states_no_terms(self): + # An identifier issued before the Terms existed ends at the match + # key. A missing byte is index 0, which says the terms are not + # stated in the identifier, so absence and zero mean the same + # thing and no presence flag is needed to tell them apart. + for name, payload, length in ( + ("probabilistic", canonical_payload(), MATCH_KEY_LENGTH), + ("random", canonical_random_payload(), GUID_LENGTH)): + with self.subTest(name): + fod = self._read(payload) + self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.NOT_STATED, fod.terms) + self.assertIsNone(fod.terms_url) + self.assertEqual(length, len(fod.match_key)) + + def test_an_absent_byte_and_a_zero_byte_read_the_same(self): + absent = self._read(canonical_payload()) + stated = self._read(with_terms(canonical_payload(), + ABSENT_TERMS_INDEX)) + self.assertEqual(absent.terms_index, stated.terms_index) + self.assertIs(absent.terms, stated.terms) + self.assertIsNone(absent.terms_url) + self.assertIsNone(stated.terms_url) + + # ----- An index this package knows ----- + + def test_index_one_is_the_model_terms_and_names_its_address(self): + for name, payload, length in ( + ("probabilistic", canonical_payload(), MATCH_KEY_LENGTH), + ("random", canonical_random_payload(), GUID_LENGTH)): + with self.subTest(name): + fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) + self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + fod.terms) + self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + self.assertEqual(length, len(fod.match_key)) + + def test_the_address_is_the_versioned_document(self): + # The address names the exact document in force when the + # identifier was made, because a receiver has to be able to check + # years later what it agreed to, and an address whose contents can + # be edited cannot answer that. + fod = self._read(with_terms(canonical_payload(), MODEL_TERMS_INDEX)) + self.assertEqual("https://m4ow.uk/mtm/2.txt", fod.terms_url) + + # ----- An index this package does not know ----- + + def test_an_unknown_index_is_reported_and_has_no_address(self): + fod = self._read(with_terms(canonical_payload(), + UNKNOWN_TERMS_INDEX)) + self.assertEqual(UNKNOWN_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.UNKNOWN, fod.terms) + self.assertIsNone(fod.terms_url) + + def test_an_unknown_index_is_not_read_as_no_terms(self): + # Zero says no terms are stated whilst an unknown index says terms + # are stated that this package cannot name, so a receiver that + # confused the two would read an identifier created under terms as + # one created under none. + unknown = self._read(with_terms(canonical_payload(), + UNKNOWN_TERMS_INDEX)) + none = self._read(with_terms(canonical_payload(), + ABSENT_TERMS_INDEX)) + self.assertIsNot(none.terms, unknown.terms) + self.assertNotEqual(none.terms, unknown.terms) + self.assertNotEqual(none.terms_index, unknown.terms_index) + # Both answer with no address, so the address alone cannot tell + # them apart and the named value and the index are what do. + self.assertIsNone(none.terms_url) + self.assertIsNone(unknown.terms_url) + + def test_every_index_the_package_does_not_know_reads_as_unknown(self): + for index in (2, 3, 127, 128, 255): + with self.subTest(index=index): + fod = self._read(with_terms(canonical_payload(), index)) + self.assertEqual(index, fod.terms_index) + self.assertIs(Terms.UNKNOWN, fod.terms) + self.assertIsNone(fod.terms_url) + + # ----- The byte is read at the right offset ----- + + def test_terms_is_read_before_a_creator_context_section(self): + # The bytes after the Terms are a creator context section whose + # lengths belong to the cloud, so the byte has to be read at the + # offset the match key ends at and not at the end of the payload. + for name, payload, key in ( + ("probabilistic", canonical_payload(), CANONICAL_MATCH_KEY), + ("random", canonical_random_payload(), + bytes((0x40 + i) for i in range(GUID_LENGTH)))): + with self.subTest(name): + built = (with_terms(payload, MODEL_TERMS_INDEX) + + CONTEXT_SECTION) + fod = self._read(built) + self.assertEqual(key, fod.match_key) + self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + fod.terms) + self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + + def test_a_context_section_alone_does_not_state_terms(self): + # The first byte after the match key is the Terms and not the + # start of the context section, so a section opening with a zero + # reads as terms that are not stated. + built = (with_terms(canonical_payload(), ABSENT_TERMS_INDEX) + + CONTEXT_SECTION) + fod = self._read(built) + self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.NOT_STATED, fod.terms) + self.assertEqual(CANONICAL_MATCH_KEY, fod.match_key) + + def test_reserved_type_takes_every_byte_as_its_match_key(self): + # A Reserved type has no defined match key length, so its + # documented best-effort reading takes every byte after the header + # and leaves none to read as the Terms. It therefore states no + # terms, which is what it stated before the Terms existed. + payload = canonical_payload() + payload[FLAGS_OFFSET] = 0b1100_0101 + fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) + self.assertIs(IdType.RESERVED, fod.type) + self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.NOT_STATED, fod.terms) + + # ----- The same answer from every reader ----- + + def test_every_reader_reads_the_same_terms(self): + payload = with_terms(canonical_payload(), MODEL_TERMS_INDEX) + base64 = self.factory.signed_owid_base64(payload) + owid = self.factory.signed_owid(payload) + readers = ( + FodId.from_base64(base64), + FodId.from_byte_array(self.factory.signed_bytes(payload)), + FodId.from_owid(owid), + FodId(owid), + FodId.try_from_base64(base64).value, + FodId.try_from_byte_array( + self.factory.signed_bytes(payload)).value, + ) + for fod in readers: + self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + fod.terms) + self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + + def test_base64_roundtrip_preserves_the_terms(self): + first = self._read(with_terms(canonical_payload(), + UNKNOWN_TERMS_INDEX)) + for base64 in (first.as_base64(), first.as_base64_url()): + fod = FodId.from_base64(base64) + self.assertEqual(first.terms_index, fod.terms_index) + self.assertIs(first.terms, fod.terms) + self.assertEqual(first.terms_url, fod.terms_url) + + def test_terms_does_not_change_how_the_other_fields_read(self): + # Adding the byte must not move any field before it, so an + # identifier reads exactly as it did with the byte and without. + without = self._read(canonical_payload()) + stated = self._read(with_terms(canonical_payload(), + MODEL_TERMS_INDEX)) + self.assertEqual(without.type, stated.type) + self.assertEqual(without.usage, stated.usage) + self.assertEqual(without.usage_from_consent, + stated.usage_from_consent) + self.assertEqual(without.license_id, stated.license_id) + self.assertEqual(without.match_key, stated.match_key) + + def test_the_payload_rules_are_unchanged_by_the_terms(self): + # A payload one byte short of its match key still fails, and a + # Terms byte written onto it only makes up the match key, because + # the type says how many bytes the match key takes and only then + # does the Terms begin. + short = canonical_random_payload()[:RANDOM_PAYLOAD_LENGTH - 1] + result = FodId.try_from_base64( + self.factory.signed_owid_base64(short)) + self.assertFalse(result.ok) + self.assertIs(FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, + result.status) + fod = self._read(with_terms(short, MODEL_TERMS_INDEX)) + self.assertEqual(GUID_LENGTH, len(fod.match_key)) + self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) + + +class TermsTests(unittest.TestCase): + """The Terms vocabulary on its own, without an identifier around it.""" + + def test_the_layout_gives_the_terms_one_byte(self): + self.assertEqual(1, TERMS_LENGTH) + self.assertEqual(0, ABSENT_TERMS_INDEX) + + def test_from_index_names_the_indexes_the_package_knows(self): + self.assertIs(Terms.NOT_STATED, Terms.from_index(0)) + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + Terms.from_index(1)) + + def test_from_index_names_every_other_index_unknown(self): + for index in range(2, 256): + self.assertIs(Terms.UNKNOWN, Terms.from_index(index)) + + def test_only_a_named_document_has_an_address(self): + self.assertIsNone(Terms.NOT_STATED.url) + self.assertIsNone(Terms.UNKNOWN.url) + self.assertEqual("https://m4ow.uk/mtm/2.txt", + Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2.url) + + def test_every_member_answers_the_address_question(self): + # A member added without an address entry would raise here rather + # than at the caller. + for member in Terms: + self.assertIn(member.url, (None, MODEL_TERMS_URL)) + + def test_no_terms_and_an_unknown_index_are_different_members(self): + self.assertIsNot(Terms.NOT_STATED, Terms.UNKNOWN) + self.assertNotEqual(Terms.NOT_STATED, Terms.UNKNOWN) if __name__ == "__main__": unittest.main() From bd6ddf0eee5071460c786e631ace29a5a0949ef2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 9 Sep 2026 15:32:23 +0100 Subject: [PATCH 2/7] REFACTOR: Name the Terms members the same in every language The specification gives the table of terms documents but no names for the enumeration members, and six language packages were about to invent six different sets. The names are now fixed across all of them, being NotStated for index 0, ModelTermsForMarketing2 for index 1 and Unknown for an index the package does not know, cased the way each language cases its own enumeration members. Python therefore uses NOT_STATED, MODEL_TERMS_FOR_MARKETING_2 and UNKNOWN. Only the middle one changes here, from MODEL_TERMS_FOR_MARKETING_VERSION_2, along with the string behind it. The comment on reading the Terms of a Reserved identifier now says plainly that answering with terms not stated is the right answer and not a defect, because no match key length is defined for that type, so every byte after the header is its match key and none is left to read. The next reader should not spend time looking for a bug there. --- fiftyone_pipeline_did/readme.md | 5 ++--- .../src/fiftyone_pipeline_did/fod_id.py | 11 ++++++++--- .../src/fiftyone_pipeline_did/terms.py | 6 +++--- fiftyone_pipeline_did/tests/test_fodid.py | 10 +++++----- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index 94c1e86..d06adb5 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -99,9 +99,8 @@ from_consent = fod_id.usage_from_consent # True when read from a consent # string the caller sent license_id = fod_id.license_id match_key = fod_id.match_key # SHA-256 or GUID bytes, see type -terms = fod_id.terms # Terms.NOT_STATED / Terms.UNKNOWN, or a - # named document, being - # MODEL_TERMS_FOR_MARKETING_VERSION_2 +terms = fod_id.terms # Terms.NOT_STATED / UNKNOWN, or a named + # document, MODEL_TERMS_FOR_MARKETING_2 terms_index = fod_id.terms_index # the raw byte, 0 to 255 terms_url = fod_id.terms_url # the address, or None where there is none diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index bea04f9..5f93907 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -585,9 +585,14 @@ def _read_terms_index(payload: bytes, offset: int) -> int: An identifier issued before the Terms existed has a payload ending at the match key, so absence and zero mean the same thing and neither has - to be told apart from the other. A Reserved type takes every byte after - the header as its match key, since no length is defined for it, and so - leaves nothing here to read. + to be told apart from the other. + + A Reserved type cannot carry a Terms byte this package can find, + because no match key length is defined for that type and so every byte + after the header is its match key. The offset then lands at the end of + the payload, nothing is left to read, and the identifier answers with + the index that says the terms are not stated. That is the right answer + and not a defect, so no special case is written for it. """ if len(payload) < offset + TERMS_LENGTH: return ABSENT_TERMS_INDEX diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py index 1cd7a71..4d9b4d4 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py @@ -68,7 +68,7 @@ class Terms(Enum): NOT_STATED = "NotStated" #: Index 1, the Model Terms for Marketing version 2, at #: https://m4ow.uk/mtm/2.txt - MODEL_TERMS_FOR_MARKETING_VERSION_2 = "ModelTermsForMarketingVersion2" + MODEL_TERMS_FOR_MARKETING_2 = "ModelTermsForMarketing2" #: An index added after this package was released, so the identifier #: states terms this package cannot name. Never treat it as #: :attr:`NOT_STATED`, which would read an identifier created under @@ -94,11 +94,11 @@ def url(self) -> Optional[str]: _BY_INDEX = { 0: Terms.NOT_STATED, - 1: Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + 1: Terms.MODEL_TERMS_FOR_MARKETING_2, } _URL = { Terms.NOT_STATED: None, - Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2: "https://m4ow.uk/mtm/2.txt", + Terms.MODEL_TERMS_FOR_MARKETING_2: "https://m4ow.uk/mtm/2.txt", Terms.UNKNOWN: None, } diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 788e73d..28573a1 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -845,7 +845,7 @@ def test_index_one_is_the_model_terms_and_names_its_address(self): with self.subTest(name): fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, fod.terms) self.assertEqual(MODEL_TERMS_URL, fod.terms_url) self.assertEqual(length, len(fod.match_key)) @@ -908,7 +908,7 @@ def test_terms_is_read_before_a_creator_context_section(self): fod = self._read(built) self.assertEqual(key, fod.match_key) self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, fod.terms) self.assertEqual(MODEL_TERMS_URL, fod.terms_url) @@ -952,7 +952,7 @@ def test_every_reader_reads_the_same_terms(self): ) for fod in readers: self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, fod.terms) self.assertEqual(MODEL_TERMS_URL, fod.terms_url) @@ -1003,7 +1003,7 @@ def test_the_layout_gives_the_terms_one_byte(self): def test_from_index_names_the_indexes_the_package_knows(self): self.assertIs(Terms.NOT_STATED, Terms.from_index(0)) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2, + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, Terms.from_index(1)) def test_from_index_names_every_other_index_unknown(self): @@ -1014,7 +1014,7 @@ def test_only_a_named_document_has_an_address(self): self.assertIsNone(Terms.NOT_STATED.url) self.assertIsNone(Terms.UNKNOWN.url) self.assertEqual("https://m4ow.uk/mtm/2.txt", - Terms.MODEL_TERMS_FOR_MARKETING_VERSION_2.url) + Terms.MODEL_TERMS_FOR_MARKETING_2.url) def test_every_member_answers_the_address_question(self): # A member added without an address entry would raise here rather From fb1f3d90a95613ba6245e2f708271bd90d635846 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 9 Sep 2026 18:31:06 +0100 Subject: [PATCH 3/7] DOC: Describe the Terms byte as it is, rather than as a change No reader has ever seen a 51Did without the Terms byte, so describing the field as a change from a previous state gives a reader history they cannot use. Every rule the history was wrapped around is kept and reworded to describe the payload instead. A payload that ends at the match key reads as index 0, which says the terms are not stated in the identifier, so absence and zero mean the same thing and no presence flag is needed. A Reserved identifier still reads as index 0, because no match key length is defined for that type so every byte after the header is its match key. --- fiftyone_pipeline_did/readme.md | 11 ++++----- .../src/fiftyone_pipeline_did/_layout.py | 5 ++-- .../src/fiftyone_pipeline_did/fod_id.py | 23 +++++++++---------- .../src/fiftyone_pipeline_did/terms.py | 4 ++-- fiftyone_pipeline_did/tests/test_fodid.py | 10 ++++---- 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index d06adb5..03de81a 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -37,8 +37,8 @@ the match key that follows, being 32 bytes for `PROBABILISTIC` and `HASHED_EMAIL`, 16 for `RANDOM`, and whatever remains for `RESERVED`. The Terms byte follows the match key, and the bytes after it are a creator context section. Identifiers issued before the type tag existed decode as -`PROBABILISTIC`, and identifiers issued before the Terms existed end at -the match key and read as terms that are not stated. +`PROBABILISTIC`, and a payload that ends at the match key reads as terms +that are not stated. This package does not publish the offsets or the raw flags byte, and it does not need to, because every field has a typed accessor that reads it @@ -171,10 +171,9 @@ say which index it could not read. empty string, and this package answers with the address and never fetches it, because what to do with the document is the receiver's decision. -An identifier issued before the Terms existed has a payload ending at the -match key, and a payload with no byte after the match key reads as index -0, so absence and zero mean the same thing and no presence flag exists to -tell them apart. The Usage and the Terms answer different questions and a +A payload with no byte after the match key reads as index 0, so absence +and zero mean the same thing and no presence flag exists to tell them +apart. The Usage and the Terms answer different questions and a receiver needs both, because the Usage says where an identifier may go and the Terms says under which document it was created. diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py index a4f3518..64fb6ce 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py @@ -66,8 +66,7 @@ #: length and so the type says where the field starts. TERMS_LENGTH = 1 #: The Terms index a payload with no byte after the match key reads as, -#: being the index that says the terms are not stated in the identifier. An -#: identifier issued before the Terms existed ends at the match key, so -#: absence and zero mean the same thing and neither has to be told apart +#: being the index that says the terms are not stated in the identifier. +#: Absence and zero mean the same thing and neither has to be told apart #: from the other. ABSENT_TERMS_INDEX = 0 diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index 5f93907..aae79fe 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -184,8 +184,8 @@ class FodId: match key. A payload longer than that is accepted, because the bytes after the Terms are a creator context section whose lengths belong to the cloud, so this package places no upper bound on a payload or an - envelope. A payload that ends at the match key was issued before the - Terms existed and reads as terms that are not stated. + envelope. A payload that ends at the match key reads as terms that are + not stated. Reading and verifying are separate steps. :meth:`try_from_base64` and :meth:`try_from_byte_array` read external data without raising and @@ -433,13 +433,12 @@ def terms_index(self) -> int: """The raw value of the Terms byte (0 to 255), being the index into the table of terms documents in the specification. - A payload that ends at the match key was issued before the Terms - existed and reads as 0, which says the terms are not stated in the - identifier, so absence and zero mean the same thing. The index is - exposed because a caller will meet one added after this package was - released, and it can then name which index it could not read, or - look the document up by hand, neither of which :attr:`terms` alone - allows. + A payload that ends at the match key reads as 0, which says the + terms are not stated in the identifier, so absence and zero mean + the same thing. The index is exposed because a caller will meet one + added after this package was released, and it can then name which + index it could not read, or look the document up by hand, neither + of which :attr:`terms` alone allows. """ return self._terms_index @@ -583,9 +582,9 @@ def _read_terms_index(payload: bytes, offset: int) -> int: """The Terms byte at the offset the match key ends at, and the index that says the terms are not stated where the payload ends there. - An identifier issued before the Terms existed has a payload ending at - the match key, so absence and zero mean the same thing and neither has - to be told apart from the other. + A payload ending at the match key has no byte to read, so absence and + zero mean the same thing and neither has to be told apart from the + other. A Reserved type cannot carry a Terms byte this package can find, because no match key length is defined for that type and so every byte diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py index 4d9b4d4..4c50e22 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py @@ -63,8 +63,8 @@ class Terms(Enum): says under which document it was created.""" #: The terms are not stated in the identifier, which is index 0 and - #: also what an identifier issued before the Terms existed reads as, - #: its payload ending at the match key. + #: also what an identifier whose payload ends at the match key reads + #: as. NOT_STATED = "NotStated" #: Index 1, the Model Terms for Marketing version 2, at #: https://m4ow.uk/mtm/2.txt diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 28573a1..d3a68ff 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -813,10 +813,10 @@ def _read(self, payload): # ----- A payload that ends at the match key ----- def test_payload_ending_at_the_match_key_states_no_terms(self): - # An identifier issued before the Terms existed ends at the match - # key. A missing byte is index 0, which says the terms are not - # stated in the identifier, so absence and zero mean the same - # thing and no presence flag is needed to tell them apart. + # There is no byte after the match key to read. A missing byte is + # index 0, which says the terms are not stated in the identifier, + # so absence and zero mean the same thing and no presence flag is + # needed to tell them apart. for name, payload, length in ( ("probabilistic", canonical_payload(), MATCH_KEY_LENGTH), ("random", canonical_random_payload(), GUID_LENGTH)): @@ -927,7 +927,7 @@ def test_reserved_type_takes_every_byte_as_its_match_key(self): # A Reserved type has no defined match key length, so its # documented best-effort reading takes every byte after the header # and leaves none to read as the Terms. It therefore states no - # terms, which is what it stated before the Terms existed. + # terms. payload = canonical_payload() payload[FLAGS_OFFSET] = 0b1100_0101 fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) From c8958f538a9f37ad1f3abbf2b750eaa62c6f65b4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 9 Sep 2026 19:20:42 +0100 Subject: [PATCH 4/7] FEAT: Refuse a payload version this package cannot read, and answer the Terms with its address Bits 4 and 5 of the flags byte are the payload version. This package reads version 0 and refuses any other with FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, and the raising readers name the version they found. No field is read under the layout this package knows once the version says otherwise, because a later version exists precisely because a field moved, so reading such a payload here would answer with values that are wrong rather than absent. The version is not exposed, because either the package read the layout or there is no identifier to read fields from. The Terms is one member rather than three. fod_id.terms answers with the address of the document the identifier was created under, and the package turns the index into the address so a caller never handles the byte. The raw index and the separate address member are gone, and terms.py is now _terms.py, private to the package as the layout module is, and is no longer exported. An index of zero and an index this package cannot name both answer with None, which a caller cannot tell apart, and that is deliberate because both say the identifier does not give the terms and the answer has to come from somewhere else. No address is ever built from an index, since that would name a document nobody wrote. The test payload builders are the creating side, so they write both new fields. The canonical flags byte carries version 0, the canonical payload carries the Terms of a personalized marketing identifier and the canonical Random payload carries the zero a non-marketing identifier carries. A payload that ends at the match key is now a fixture of its own, since a reader takes it as a Terms of zero and no issuer would write one. Tests: 171 passed, 2 skipped, 29 subtests passed. The offline example prints the address. --- .../examples/fodid_example.py | 7 +- fiftyone_pipeline_did/readme.md | 84 +++-- .../src/fiftyone_pipeline_did/__init__.py | 4 +- .../src/fiftyone_pipeline_did/_layout.py | 4 + .../{terms.py => _terms.py} | 11 +- .../src/fiftyone_pipeline_did/fod_id.py | 104 ++++--- fiftyone_pipeline_did/tests/test_fodid.py | 289 ++++++++++++------ 7 files changed, 332 insertions(+), 171 deletions(-) rename fiftyone_pipeline_did/src/fiftyone_pipeline_did/{terms.py => _terms.py} (92%) diff --git a/fiftyone_pipeline_did/examples/fodid_example.py b/fiftyone_pipeline_did/examples/fodid_example.py index 2e92235..2012448 100644 --- a/fiftyone_pipeline_did/examples/fodid_example.py +++ b/fiftyone_pipeline_did/examples/fodid_example.py @@ -42,7 +42,8 @@ # from the package, which does not publish it. The layout is # specified at # https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md -SAMPLE_FLAGS = 0b0000_0011 # standard usage, Probabilistic type +SAMPLE_FLAGS = 0b0000_0011 # standard usage, payload version 0, + # Probabilistic type SAMPLE_PAYLOAD_LENGTH = 38 # 1 flags byte, 4 licence id, 32 key, # 1 terms SAMPLE_MATCH_KEY_OFFSET = 5 @@ -89,11 +90,9 @@ def run(): print(" Consent :", fod_id.usage_from_consent) print(" LicenseId :", fod_id.license_id) print(" Match key :", fod_id.match_key.hex()) - print(" Terms :", fod_id.terms.name) - print(" Terms idx :", fod_id.terms_index) # The address is answered and never fetched. What to do with the # document is the receiver's decision. - print(" Terms url :", fod_id.terms_url) + print(" Terms :", fod_id.terms) print(" Verifies :", fod_id.verify(crypto.public_key_pem())) reissued = FodId.from_base64(issue(creator, payload)) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index 03de81a..a05403a 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -16,7 +16,8 @@ Identifier) returned by the 51Degrees Cloud service. Mirrors the .NET same inputs share the same match key even though their envelopes differ. - The **Terms** is the byte after the match key that says which terms document the identifier was created under, so the terms travel with the - identifier instead of alongside it. + identifier instead of alongside it. The package turns that byte into the + address of the document. **Comparing two 51Dids means comparing their match keys, never their envelopes.** @@ -89,7 +90,7 @@ way to hold an unsigned or partly built envelope. ## Usage ```python -from fiftyone_pipeline_did import FodId, IdType, Terms, Usage +from fiftyone_pipeline_did import FodId, IdType, Usage fod_id = FodId.from_base64(base64_from_cloud_service) # either alphabet @@ -99,10 +100,9 @@ from_consent = fod_id.usage_from_consent # True when read from a consent # string the caller sent license_id = fod_id.license_id match_key = fod_id.match_key # SHA-256 or GUID bytes, see type -terms = fod_id.terms # Terms.NOT_STATED / UNKNOWN, or a named - # document, MODEL_TERMS_FOR_MARKETING_2 -terms_index = fod_id.terms_index # the raw byte, 0 to 255 -terms_url = fod_id.terms_url # the address, or None where there is none +terms = fod_id.terms # address of the terms document it was + # created under, None where it names none + # this package knows # Delegated OWID-level fields and operations. domain = fod_id.domain @@ -142,34 +142,39 @@ nothing about which usage was reached. ### The terms an identifier was created under -`fod_id.terms` says which terms document the identifier was created under, -as a `Terms`, so the terms travel with the identifier instead of alongside -it and a receiver can tell which document was in force when the identifier -was made. `fod_id.terms_url` gives the address of that document, and -`fod_id.terms_index` gives the raw byte behind both, being an index into a +`fod_id.terms` answers with the address of the terms document the +identifier was created under, so the terms travel with the identifier +instead of alongside it and a receiver can tell which document was in +force when the identifier was made. The byte behind it is an index into a table of terms documents in the -[layout specification](https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md). -The byte is an index and not a version number, so that a later document -can live at any address rather than only at one the specification could -compose from a number, and an index is never reused or repointed once -published, because an identifier issued under it has to stay readable +[layout specification](https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md), +and the package turns the index into the address so a caller never handles +the byte. The byte is an index and not a version number, so that a later +document can live at any address rather than only at one the specification +could compose from a number, and an index is never reused or repointed +once published, because an identifier issued under it has to stay readable years later. -`Terms.NOT_STATED` and `Terms.UNKNOWN` are different answers and must -never be read as the same one. `NOT_STATED` is index 0 and says this -identifier does not carry the answer, so the answer has to come from +| Index | Document | `fod_id.terms` | +|------:|----------|----------------| +| `0` | Not stated in the identifier | `None` | +| `1` | Model Terms for Marketing, version 2 | `https://m4ow.uk/mtm/2.txt` | +| any other | One this package cannot name | `None` | + +The answer is `None` where the identifier does not state its terms and +where it states an index added after this package was released, using +absence rather than an empty string. No address is ever built from an +index this package cannot name, because that would name a document nobody +wrote and a receiver would record having accepted terms that do not exist. +A caller therefore cannot tell those two apart, which is deliberate, since +both lead to the same place. This package answers with the address and +never fetches it, because what to do with the document is the receiver's +decision. + +No address does not mean the identifier is unrestricted. It says only that +this identifier does not carry the answer, so the answer has to come from somewhere else, being the Terms Document Locator in an OpenRTB request or -whatever the surrounding protocol provides, and it does not mean the -identifier is unrestricted. `UNKNOWN` says the identifier does state its -terms and that this package cannot name them, because the index was added -after the package was released. A caller meeting `UNKNOWN` should treat -the identifier as covered by terms it cannot yet read, and either update -the package or refuse the identifier, and `terms_index` is there so it can -say which index it could not read. - -`terms_url` is `None` for both of those, using absence rather than an -empty string, and this package answers with the address and never fetches -it, because what to do with the document is the receiver's decision. +whatever the surrounding protocol provides. A payload with no byte after the match key reads as index 0, so absence and zero mean the same thing and no presence flag exists to tell them @@ -177,6 +182,23 @@ apart. The Usage and the Terms answer different questions and a receiver needs both, because the Usage says where an identifier may go and the Terms says under which document it was created. +### The payload version + +Bits 4 and 5 of the flags byte say which payload layout the identifier +follows, and this package reads version 0. A payload naming version 1, 2 +or 3 is refused with `FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION`, and +the raising readers name the version they found in the message. + +No field is read under the layout this package knows once the version says +otherwise. A later version exists precisely because a field moved, so +reading such a payload here would answer with values that are wrong rather +than absent, which is worse than refusing. A version that nothing checks +protects nothing. + +The version is not exposed. Either this package read the layout, in which +case the accessors are the answer, or it did not, in which case there is +no identifier to read fields from. + The raw flags byte, the byte layout constants and the old `hash` names are not part of this package. `fod_id.flags`, `fod_id.hash`, `fod_id.date_minutes`, `FodId.MATCH_KEY_OFFSET` and every other offset and @@ -583,7 +605,7 @@ is refreshed by common-ci's `update-example-assets` step. - **No signature verification on parsing.** A parsed 51Did is not known to be genuine. Call `verify(public_key_pem)`, `signature_status(public_key_pem)` or a `DidClient` check when needed. -- **No fetching of a terms document.** `fod_id.terms_url` answers with the +- **No fetching of a terms document.** `fod_id.terms` answers with the address and nothing more. What to do with the document is the receiver's decision. - **No upper bound on the size of an identifier.** The lengths beyond the diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py index 4671a6d..4d7373e 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py @@ -28,7 +28,7 @@ carries (the identifier :class:`~fiftyone_pipeline_did.id_type.IdType`, the :class:`~fiftyone_pipeline_did.usage.Usage` it was created for and whether that usage came from a consent string, the License Id, the match key and -the :class:`~fiftyone_pipeline_did.terms.Terms` it was created under), +the address of the terms document it was created under), and delegates OWID-level concerns to the wrapped envelope. The raw bytes and offsets behind those accessors are not part of this surface, which is specified at @@ -73,7 +73,6 @@ from ._owid import Owid, OwidError, SignatureStatus from .fod_id import DATE_EPOCH, FodId, FodIdParseResult, FodIdParseStatus from .id_type import IdType -from .terms import Terms from .usage import Usage __all__ = [ @@ -81,7 +80,6 @@ "FodIdParseResult", "FodIdParseStatus", "IdType", - "Terms", "Usage", "DATE_EPOCH", "DidClient", diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py index 64fb6ce..6aaba08 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py @@ -65,6 +65,10 @@ #: offset constant for it, because the identifier type fixes the match key #: length and so the type says where the field starts. TERMS_LENGTH = 1 +#: The payload layout version this package reads, carried in bits 4 and 5 +#: of the flags byte. Any other version is refused rather than read under +#: this layout. +SUPPORTED_PAYLOAD_VERSION = 0 #: The Terms index a payload with no byte after the match key reads as, #: being the index that says the terms are not stated in the identifier. #: Absence and zero mean the same thing and neither has to be told apart diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py similarity index 92% rename from fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py rename to fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py index 4c50e22..1b6dc85 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/terms.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py @@ -54,9 +54,14 @@ class Terms(Enum): The members carry a name and not the index, because :attr:`UNKNOWN` stands for any index this package does not know and so has no single - index to carry. Where a caller needs the index itself, for instance to - report which one it could not read, it reads - :attr:`~fiftyone_pipeline_did.FodId.terms_index`. + index to carry. + + This module is private to the package, as its underscore name says. The + package turns the index into the address that + :attr:`~fiftyone_pipeline_did.FodId.terms` answers with, so a caller + never handles the byte, and the names here are the ones the + specification gives so that every package describes one document the + same way. The Usage and the Terms answer different questions and a receiver needs both, because the Usage says where an identifier may go and the Terms diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index aae79fe..29e4806 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -35,6 +35,7 @@ LICENSE_ID_OFFSET, MATCH_KEY_LENGTH, MATCH_KEY_OFFSET, + SUPPORTED_PAYLOAD_VERSION, TERMS_LENGTH, ) from ._owid import ( @@ -47,7 +48,7 @@ ) from .id_type import IdType -from .terms import Terms +from ._terms import Terms from .usage import Usage #: The moment the envelope's date field counts minutes from, being the OWID @@ -116,6 +117,12 @@ class FodIdParseStatus(Enum): #: the header for Random and a 32 byte SHA-256 match key for #: Probabilistic and HashedEmail. INVALID_TYPE_PAYLOAD_LENGTH = "InvalidTypePayloadLength" + #: Bits 4 and 5 of the flags byte name a payload layout version this + #: package does not know, so no field is read. A later version exists + #: precisely because a field moved, so reading the payload under the + #: layout this package knows would answer with values that are wrong + #: rather than absent. + UNSUPPORTED_PAYLOAD_VERSION = "UnsupportedPayloadVersion" @classmethod def of(cls, status: ParseStatus) -> "FodIdParseStatus": @@ -169,9 +176,8 @@ class FodId: Payload layout. Every field has a typed accessor here, being :attr:`type`, :attr:`usage`, :attr:`usage_from_consent`, - :attr:`license_id`, :attr:`match_key`, :attr:`terms`, - :attr:`terms_index` and :attr:`terms_url`, and those accessors are the - supported way to read an identifier. The bytes and offsets behind them + :attr:`license_id`, :attr:`match_key` and :attr:`terms`, and those + accessors are the supported way to read an identifier. The bytes and offsets behind them are specified at https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md and the surface this class offers, which is the same in every 51Did @@ -187,6 +193,15 @@ class FodId: envelope. A payload that ends at the match key reads as terms that are not stated. + Bits 4 and 5 of the flags byte say which payload layout the identifier + follows, and this package reads version 0. A payload naming any other + version is refused with + :attr:`FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION` rather than read + under the layout this package knows, because a later version exists + precisely because a field moved, so reading one here would answer with + values that are wrong rather than absent. The version is not exposed, + because a caller has nothing to decide with it. + Reading and verifying are separate steps. :meth:`try_from_base64` and :meth:`try_from_byte_array` read external data without raising and answer with a :class:`FodIdParseResult` naming the reason either way, @@ -420,42 +435,32 @@ def match_key(self) -> bytes: return self._match_key @property - def terms(self) -> Terms: - """The terms document the identifier was created under, so the - terms travel with the identifier instead of alongside it. See - :class:`~fiftyone_pipeline_did.Terms`, which sets out why terms - that are not stated and terms this package cannot name are - different answers and must never be read as the same one.""" - return Terms.from_index(self._terms_index) - - @property - def terms_index(self) -> int: - """The raw value of the Terms byte (0 to 255), being the index into - the table of terms documents in the specification. - - A payload that ends at the match key reads as 0, which says the - terms are not stated in the identifier, so absence and zero mean - the same thing. The index is exposed because a caller will meet one - added after this package was released, and it can then name which - index it could not read, or look the document up by hand, neither - of which :attr:`terms` alone allows. + def terms(self) -> Optional[str]: + """The address of the terms document the identifier was created + under, so the terms travel with the identifier instead of + alongside it. + + The byte after the match key is an index into a table in the + specification and this package turns the index into the address, + so a caller never handles the byte. The address is answered and + never fetched, because what to do with the document is the + receiver's decision. + + ``None`` covers both an index of zero, which says the terms are + not stated in the identifier, and an index added to the table + after this package was released, which it cannot name. A caller + cannot tell those two apart, which is deliberate, because both + lead to the same place, being that the identifier does not say + which terms it was created under and the answer has to come from + somewhere else. No address is ever built from an index this + package does not know, since that would name a document nobody + wrote. + + No address does not mean the identifier is unrestricted. Where an + identifier may go is a separate question :attr:`usage` answers, + which still bars a non-marketing identifier from a demand source. """ - return self._terms_index - - @property - def terms_url(self) -> Optional[str]: - """The address of the terms document, and ``None`` where the terms - are not stated in the identifier or where the index is one this - package does not know. - - The address is answered and never fetched, because what to do with - the document is the receiver's decision. ``None`` for an index this - package does not know says only that the document cannot be named - here, and never that there is no document, which is what - :attr:`terms` and :attr:`terms_index` are read together to tell - apart. - """ - return self.terms.url + return Terms.from_index(self._terms_index).url @property def version(self) -> Version: @@ -560,6 +565,14 @@ def _read_payload( if payload is None or len(payload) < HEADER_LENGTH: return FodIdParseStatus.PAYLOAD_TOO_SHORT, 0, 0, b"", 0 flags = payload[FLAGS_OFFSET] + # The version is read before any field, because a later version exists + # precisely because a field moved. Reading a payload of a version this + # package does not know under the layout it does know would answer with + # values that are wrong rather than absent, which is worse than + # refusing, and a version that nothing checks protects nothing. + if _payload_version(flags) != SUPPORTED_PAYLOAD_VERSION: + return ( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, 0, 0, b"", 0) match_key_length = _match_key_length(IdType.from_flags(flags), payload) if len(payload) < HEADER_LENGTH + match_key_length: return FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, 0, 0, b"", 0 @@ -607,6 +620,15 @@ def _unpack_or_raise(payload: bytes) -> Tuple[int, int, bytes, int]: return flags, license_id, match_key, terms_index +def _payload_version(flags: int) -> int: + """Bits 4 and 5 of the flags byte, being the version of the payload + layout the identifier follows. The envelope carries a version of its + own at its first byte, which versions the envelope, whilst this one + versions the payload. + """ + return (flags >> 4) & 0b11 + + def _match_key_length(id_type: IdType, payload: bytes) -> int: """How many match key bytes the type needs after the header.""" if id_type is IdType.RANDOM: @@ -622,6 +644,10 @@ def _payload_message(status: FodIdParseStatus, payload: bytes) -> str: if status is FodIdParseStatus.PAYLOAD_TOO_SHORT: return "51Did payload must be at least {0} bytes; got {1}.".format( HEADER_LENGTH, length) + if status is FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION: + return ( + "51Did payload version {0} is not one this package can " + "read.".format(_payload_version(payload[FLAGS_OFFSET]))) id_type = IdType.from_flags(payload[FLAGS_OFFSET]) return ("51Did payload for the {0} type must be at least {1} bytes; " "got {2}.".format( diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index d3a68ff..25a9baa 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -33,11 +33,15 @@ FodIdParseStatus, IdType, OwidError, - Terms, Usage, SignatureStatus, ) +# The named terms value is private to the package and is not exported, so +# the vocabulary tests below reach the private module directly, as the +# package surface page says the package's own tests may. +from fiftyone_pipeline_did._terms import Terms + # The byte layout is not part of the package's public surface. These tests # build payloads byte by byte, so they read it from the private module, as # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md @@ -53,13 +57,15 @@ MATCH_KEY_OFFSET, PAYLOAD_LENGTH, RANDOM_PAYLOAD_LENGTH, + SUPPORTED_PAYLOAD_VERSION, TERMS_LENGTH, ) from .envelope import envelope_bytes, signed_envelope TEST_DOMAIN = "51degrees.com" -# 0xA5: usage bits plus the HashedEmail type tag in bits 6-7. -CANONICAL_FLAGS = 0xA5 +# 0x85: the personalized marketing usage in bits 0-2, the payload version 0 +# in bits 4-5 and the HashedEmail type tag in bits 6-7. +CANONICAL_FLAGS = 0x85 CANONICAL_LICENSE_ID = 0x12345678 CANONICAL_MATCH_KEY = bytes((0x20 + i) for i in range(MATCH_KEY_LENGTH)) @@ -76,7 +82,11 @@ def _write_license_id(payload): payload[LICENSE_ID_OFFSET + 3] = 0x12 -def canonical_payload(): +def payload_ending_at_match_key(): + """The canonical payload cut off at the end of the match key, so it + carries no Terms byte. A reader takes that as a Terms of zero, and + this is the fixture for that rule rather than anything an issuer + would write.""" payload = bytearray(PAYLOAD_LENGTH) payload[FLAGS_OFFSET] = CANONICAL_FLAGS _write_license_id(payload) @@ -86,6 +96,14 @@ def canonical_payload(): return bytearray(payload) +def canonical_payload(): + """The canonical payload as an issuer writes one, carrying the + payload version 0 in its flags byte and the Terms byte of the + document a personalized marketing identifier is created under. This + is the creating side, so it writes every field an issuer writes.""" + return with_terms(payload_ending_at_match_key(), MODEL_TERMS_INDEX) + + #: The address the specification gives for Terms index 1, the Model Terms #: for Marketing version 2. MODEL_TERMS_URL = "https://m4ow.uk/mtm/2.txt" @@ -105,7 +123,8 @@ def with_terms(payload, index): return bytearray(bytes(payload) + bytes([index])) -def canonical_random_payload(): +def random_payload_ending_at_match_key(): + """The canonical Random payload cut off at the end of its GUID.""" payload = bytearray(RANDOM_PAYLOAD_LENGTH) payload[FLAGS_OFFSET] = (1 << 6) | 0b001 # Random tag + usage bits _write_license_id(payload) @@ -114,6 +133,22 @@ def canonical_random_payload(): return bytearray(payload) +def canonical_random_payload(): + """The canonical Random payload as an issuer writes one, carrying the + zero Terms byte a non-marketing identifier carries.""" + return with_terms( + random_payload_ending_at_match_key(), ABSENT_TERMS_INDEX) + + +def with_payload_version(payload, version): + """The payload with its version bits set to the given version, leaving + every other bit of the flags byte alone.""" + changed = bytearray(payload) + changed[FLAGS_OFFSET] = ( + (payload[FLAGS_OFFSET] & 0b1100_1111) | (version << 4)) + return changed + + class FodIdTestFactory: """Generates a fresh ECDSA P-256 key pair and signs real OWID envelopes. A Creator is the only way the OWID library brings a new @@ -236,11 +271,15 @@ def test_flags_byte_of_zero_is_read(self): fod = FodId.from_base64(self.factory.signed_owid_base64(payload)) self.assertEqual(0, fod._flags) - def test_flags_byte_of_all_bits_set_is_read(self): + def test_every_flags_bit_outside_the_version_is_read(self): + # Bits 4 and 5 are the payload version and only version 0 is read, + # so every other bit is set and those two are left clear. A payload + # with them set is refused rather than read, which + # FodIdVersionTests covers. payload = canonical_payload() - payload[FLAGS_OFFSET] = 0xFF + payload[FLAGS_OFFSET] = 0xCF fod = FodId.from_base64(self.factory.signed_owid_base64(payload)) - self.assertEqual(255, fod._flags) + self.assertEqual(0xCF, fod._flags) def test_match_key_is_immutable(self): fod = FodId.from_base64( @@ -509,9 +548,9 @@ def assert_canonical(self, fod): # ----- Vocabulary ----- - def test_status_vocabulary_is_the_owid_one_plus_two(self): + def test_status_vocabulary_is_the_owid_one_plus_three(self): # Every OWID status has a member of the same name and value, so an - # OWID failure is carried through unchanged, and the two 51Did + # OWID failure is carried through unchanged, and the three 51Did # payload statuses are the only additions. for status in ParseStatus: member = FodIdParseStatus.of(status) @@ -520,7 +559,12 @@ def test_status_vocabulary_is_the_owid_one_plus_two(self): owid_names = {status.name for status in ParseStatus} extra = {member.name for member in FodIdParseStatus} - owid_names self.assertEqual( - {"PAYLOAD_TOO_SHORT", "INVALID_TYPE_PAYLOAD_LENGTH"}, extra) + { + "PAYLOAD_TOO_SHORT", + "INVALID_TYPE_PAYLOAD_LENGTH", + "UNSUPPORTED_PAYLOAD_VERSION", + }, + extra) def test_result_is_immutable_and_carries_exactly_three_facts(self): result = FodId.try_from_base64( @@ -801,7 +845,8 @@ class FodIdTermsTests(unittest.TestCase): created under so that the terms travel with the identifier. The byte is an index into a table in the specification and not a version number, and it follows the match key, so the identifier type fixes where it - sits. + sits. The package turns the index into the address, so ``terms`` + answers with the address and a caller never handles the byte. """ def setUp(self): @@ -812,42 +857,40 @@ def _read(self, payload): # ----- A payload that ends at the match key ----- - def test_payload_ending_at_the_match_key_states_no_terms(self): + def test_payload_ending_at_the_match_key_has_no_address(self): # There is no byte after the match key to read. A missing byte is # index 0, which says the terms are not stated in the identifier, # so absence and zero mean the same thing and no presence flag is # needed to tell them apart. for name, payload, length in ( - ("probabilistic", canonical_payload(), MATCH_KEY_LENGTH), - ("random", canonical_random_payload(), GUID_LENGTH)): + ("probabilistic", payload_ending_at_match_key(), + MATCH_KEY_LENGTH), + ("random", random_payload_ending_at_match_key(), + GUID_LENGTH)): with self.subTest(name): fod = self._read(payload) - self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.NOT_STATED, fod.terms) - self.assertIsNone(fod.terms_url) + self.assertIsNone(fod.terms) self.assertEqual(length, len(fod.match_key)) def test_an_absent_byte_and_a_zero_byte_read_the_same(self): - absent = self._read(canonical_payload()) - stated = self._read(with_terms(canonical_payload(), + absent = self._read(payload_ending_at_match_key()) + stated = self._read(with_terms(payload_ending_at_match_key(), ABSENT_TERMS_INDEX)) - self.assertEqual(absent.terms_index, stated.terms_index) - self.assertIs(absent.terms, stated.terms) - self.assertIsNone(absent.terms_url) - self.assertIsNone(stated.terms_url) + self.assertEqual(absent.terms, stated.terms) + self.assertIsNone(absent.terms) + self.assertIsNone(stated.terms) # ----- An index this package knows ----- - def test_index_one_is_the_model_terms_and_names_its_address(self): + def test_index_one_answers_with_the_model_terms_address(self): for name, payload, length in ( - ("probabilistic", canonical_payload(), MATCH_KEY_LENGTH), - ("random", canonical_random_payload(), GUID_LENGTH)): + ("probabilistic", payload_ending_at_match_key(), + MATCH_KEY_LENGTH), + ("random", random_payload_ending_at_match_key(), + GUID_LENGTH)): with self.subTest(name): fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) - self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, - fod.terms) - self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + self.assertEqual(MODEL_TERMS_URL, fod.terms) self.assertEqual(length, len(fod.match_key)) def test_the_address_is_the_versioned_document(self): @@ -855,42 +898,38 @@ def test_the_address_is_the_versioned_document(self): # identifier was made, because a receiver has to be able to check # years later what it agreed to, and an address whose contents can # be edited cannot answer that. - fod = self._read(with_terms(canonical_payload(), MODEL_TERMS_INDEX)) - self.assertEqual("https://m4ow.uk/mtm/2.txt", fod.terms_url) + fod = self._read(with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX)) + self.assertEqual("https://m4ow.uk/mtm/2.txt", fod.terms) # ----- An index this package does not know ----- - def test_an_unknown_index_is_reported_and_has_no_address(self): - fod = self._read(with_terms(canonical_payload(), + def test_an_unknown_index_has_no_address(self): + # No address is ever built from an index this package cannot name, + # because that would name a document nobody wrote and a receiver + # would record having accepted terms that do not exist. + fod = self._read(with_terms(payload_ending_at_match_key(), UNKNOWN_TERMS_INDEX)) - self.assertEqual(UNKNOWN_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.UNKNOWN, fod.terms) - self.assertIsNone(fod.terms_url) - - def test_an_unknown_index_is_not_read_as_no_terms(self): - # Zero says no terms are stated whilst an unknown index says terms - # are stated that this package cannot name, so a receiver that - # confused the two would read an identifier created under terms as - # one created under none. - unknown = self._read(with_terms(canonical_payload(), + self.assertIsNone(fod.terms) + + def test_an_unknown_index_answers_as_no_terms_stated_does(self): + # A caller cannot tell the two apart, which is deliberate, since + # both say the identifier does not give the terms and the answer + # has to come from somewhere else. + unknown = self._read(with_terms(payload_ending_at_match_key(), UNKNOWN_TERMS_INDEX)) - none = self._read(with_terms(canonical_payload(), + none = self._read(with_terms(payload_ending_at_match_key(), ABSENT_TERMS_INDEX)) - self.assertIsNot(none.terms, unknown.terms) - self.assertNotEqual(none.terms, unknown.terms) - self.assertNotEqual(none.terms_index, unknown.terms_index) - # Both answer with no address, so the address alone cannot tell - # them apart and the named value and the index are what do. - self.assertIsNone(none.terms_url) - self.assertIsNone(unknown.terms_url) - - def test_every_index_the_package_does_not_know_reads_as_unknown(self): + self.assertIsNone(none.terms) + self.assertIsNone(unknown.terms) + + def test_every_index_the_package_does_not_know_has_no_address(self): for index in (2, 3, 127, 128, 255): with self.subTest(index=index): - fod = self._read(with_terms(canonical_payload(), index)) - self.assertEqual(index, fod.terms_index) - self.assertIs(Terms.UNKNOWN, fod.terms) - self.assertIsNone(fod.terms_url) + fod = self._read( + with_terms(payload_ending_at_match_key(), index)) + self.assertIsNone(fod.terms) + self.assertIs(Terms.UNKNOWN, Terms.from_index(index)) # ----- The byte is read at the right offset ----- @@ -899,28 +938,26 @@ def test_terms_is_read_before_a_creator_context_section(self): # lengths belong to the cloud, so the byte has to be read at the # offset the match key ends at and not at the end of the payload. for name, payload, key in ( - ("probabilistic", canonical_payload(), CANONICAL_MATCH_KEY), - ("random", canonical_random_payload(), + ("probabilistic", payload_ending_at_match_key(), + CANONICAL_MATCH_KEY), + ("random", random_payload_ending_at_match_key(), bytes((0x40 + i) for i in range(GUID_LENGTH)))): with self.subTest(name): built = (with_terms(payload, MODEL_TERMS_INDEX) + CONTEXT_SECTION) fod = self._read(built) self.assertEqual(key, fod.match_key) - self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, - fod.terms) - self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + self.assertEqual(MODEL_TERMS_URL, fod.terms) def test_a_context_section_alone_does_not_state_terms(self): # The first byte after the match key is the Terms and not the # start of the context section, so a section opening with a zero # reads as terms that are not stated. - built = (with_terms(canonical_payload(), ABSENT_TERMS_INDEX) + built = (with_terms(payload_ending_at_match_key(), + ABSENT_TERMS_INDEX) + CONTEXT_SECTION) fod = self._read(built) - self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.NOT_STATED, fod.terms) + self.assertIsNone(fod.terms) self.assertEqual(CANONICAL_MATCH_KEY, fod.match_key) def test_reserved_type_takes_every_byte_as_its_match_key(self): @@ -928,17 +965,17 @@ def test_reserved_type_takes_every_byte_as_its_match_key(self): # documented best-effort reading takes every byte after the header # and leaves none to read as the Terms. It therefore states no # terms. - payload = canonical_payload() + payload = payload_ending_at_match_key() payload[FLAGS_OFFSET] = 0b1100_0101 fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) self.assertIs(IdType.RESERVED, fod.type) - self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.NOT_STATED, fod.terms) + self.assertIsNone(fod.terms) # ----- The same answer from every reader ----- def test_every_reader_reads_the_same_terms(self): - payload = with_terms(canonical_payload(), MODEL_TERMS_INDEX) + payload = with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX) base64 = self.factory.signed_owid_base64(payload) owid = self.factory.signed_owid(payload) readers = ( @@ -951,25 +988,21 @@ def test_every_reader_reads_the_same_terms(self): self.factory.signed_bytes(payload)).value, ) for fod in readers: - self.assertEqual(MODEL_TERMS_INDEX, fod.terms_index) - self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, - fod.terms) - self.assertEqual(MODEL_TERMS_URL, fod.terms_url) + self.assertEqual(MODEL_TERMS_URL, fod.terms) def test_base64_roundtrip_preserves_the_terms(self): - first = self._read(with_terms(canonical_payload(), - UNKNOWN_TERMS_INDEX)) + first = self._read(with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX)) for base64 in (first.as_base64(), first.as_base64_url()): fod = FodId.from_base64(base64) - self.assertEqual(first.terms_index, fod.terms_index) - self.assertIs(first.terms, fod.terms) - self.assertEqual(first.terms_url, fod.terms_url) + self.assertEqual(first.terms, fod.terms) + self.assertEqual(MODEL_TERMS_URL, fod.terms) def test_terms_does_not_change_how_the_other_fields_read(self): - # Adding the byte must not move any field before it, so an - # identifier reads exactly as it did with the byte and without. - without = self._read(canonical_payload()) - stated = self._read(with_terms(canonical_payload(), + # The byte must not move any field before it, so an identifier + # reads the same with the byte and without. + without = self._read(payload_ending_at_match_key()) + stated = self._read(with_terms(payload_ending_at_match_key(), MODEL_TERMS_INDEX)) self.assertEqual(without.type, stated.type) self.assertEqual(without.usage, stated.usage) @@ -983,7 +1016,8 @@ def test_the_payload_rules_are_unchanged_by_the_terms(self): # Terms byte written onto it only makes up the match key, because # the type says how many bytes the match key takes and only then # does the Terms begin. - short = canonical_random_payload()[:RANDOM_PAYLOAD_LENGTH - 1] + short = random_payload_ending_at_match_key()[ + :RANDOM_PAYLOAD_LENGTH - 1] result = FodId.try_from_base64( self.factory.signed_owid_base64(short)) self.assertFalse(result.ok) @@ -991,11 +1025,82 @@ def test_the_payload_rules_are_unchanged_by_the_terms(self): result.status) fod = self._read(with_terms(short, MODEL_TERMS_INDEX)) self.assertEqual(GUID_LENGTH, len(fod.match_key)) - self.assertEqual(ABSENT_TERMS_INDEX, fod.terms_index) + self.assertIsNone(fod.terms) + + +class FodIdVersionTests(unittest.TestCase): + """Bits 4 and 5 of the flags byte, being the payload layout version. + This package reads version 0 and refuses every other version rather + than reading fields that may have moved. + """ + + def setUp(self): + self.factory = FodIdTestFactory() + + def _read(self, payload): + return FodId.try_from_base64( + self.factory.signed_owid_base64(payload)) + + def test_version_zero_reads_every_field(self): + result = self._read(canonical_payload()) + self.assertTrue(result.ok) + fod = result.value + self.assertIs(IdType.HASHED_EMAIL, fod.type) + self.assertIs(Usage.PERSONALIZED, fod.usage) + self.assertEqual(CANONICAL_LICENSE_ID, fod.license_id) + self.assertEqual(CANONICAL_MATCH_KEY, fod.match_key) + self.assertEqual(MODEL_TERMS_URL, fod.terms) + + def test_the_layout_names_the_version_this_package_reads(self): + self.assertEqual(0, SUPPORTED_PAYLOAD_VERSION) + + def test_an_unassigned_version_is_refused(self): + for version in (1, 2, 3): + with self.subTest(version=version): + result = self._read( + with_payload_version(canonical_payload(), version)) + self.assertFalse(result.ok) + self.assertIs( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + result.status) + # Nothing is handed back, rather than a value with some + # fields filled in, because there is no identifier to + # expose fields for when the layout was not understood. + self.assertIsNone(result.value) + + def test_the_raising_readers_name_the_version(self): + for version in (1, 2, 3): + with self.subTest(version=version): + base64 = self.factory.signed_owid_base64( + with_payload_version(canonical_payload(), version)) + with self.assertRaises(ValueError) as caught: + FodId.from_base64(base64) + self.assertIn("version {0}".format(version), + str(caught.exception)) + + def test_the_version_is_read_apart_from_the_usage_and_type_bits(self): + # A reader masking the wrong bits would refuse a version 0 + # identifier or let a later version through, so every combination + # is tried. + for usage in (0b000, 0b001, 0b011, 0b111): + for id_type in (0b00, 0b10, 0b11): + flags = (id_type << 6) | usage + payload = payload_ending_at_match_key() + payload[FLAGS_OFFSET] = flags + with self.subTest(flags=flags): + self.assertTrue(self._read(payload).ok) + for version in (1, 2, 3): + refused = self._read( + with_payload_version(payload, version)) + self.assertIs( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + refused.status) + self.assertIsNone(refused.value) class TermsTests(unittest.TestCase): - """The Terms vocabulary on its own, without an identifier around it.""" + """The Terms vocabulary on its own, without an identifier around it. + It is private to the package, so these tests reach it directly.""" def test_the_layout_gives_the_terms_one_byte(self): self.assertEqual(1, TERMS_LENGTH) @@ -1022,9 +1127,11 @@ def test_every_member_answers_the_address_question(self): for member in Terms: self.assertIn(member.url, (None, MODEL_TERMS_URL)) - def test_no_terms_and_an_unknown_index_are_different_members(self): - self.assertIsNot(Terms.NOT_STATED, Terms.UNKNOWN) - self.assertNotEqual(Terms.NOT_STATED, Terms.UNKNOWN) + def test_the_named_value_is_not_part_of_the_package_surface(self): + # The address on FodId is the whole of what a caller reads, so the + # named value is not exported from the package. + import fiftyone_pipeline_did as package + self.assertFalse(hasattr(package, "Terms")) if __name__ == "__main__": unittest.main() From 4466c1525e7e2c892f4ab0f9f091634cb8d3e3cc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 9 Sep 2026 21:36:56 +0100 Subject: [PATCH 5/7] REFACTOR: One terms table, so a new document is one member Adding a terms document meant editing the member, an index to member dict and an address dict, three places that could disagree. The address lookup also subscripted the dict directly, so a member missing from it would raise KeyError at the caller rather than answer with no address. Each member now carries its index and its address, and the index to member map is built from the members rather than written out again, so a new document is one new member and nothing else. The address comes off the member, so there is nothing left to raise. A test that claimed a member without an address entry would raise there described the old design and could not fail under the new one. It is replaced by two that check what now holds, being that every member reads back from its own index and that a member naming a document has an https address, and that no two members share an index. pytest: 89 passed, 29 subtests, 0 failures. --- .../src/fiftyone_pipeline_did/_terms.py | 37 ++++++++++++------- fiftyone_pipeline_did/tests/test_fodid.py | 34 +++++++++++++++-- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py index 1b6dc85..73c726a 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py @@ -69,16 +69,23 @@ class Terms(Enum): #: The terms are not stated in the identifier, which is index 0 and #: also what an identifier whose payload ends at the match key reads - #: as. - NOT_STATED = "NotStated" + #: as. It names no document, so it has no address. + NOT_STATED = (0, None) #: Index 1, the Model Terms for Marketing version 2, at #: https://m4ow.uk/mtm/2.txt - MODEL_TERMS_FOR_MARKETING_2 = "ModelTermsForMarketing2" + MODEL_TERMS_FOR_MARKETING_2 = (1, "https://m4ow.uk/mtm/2.txt") #: An index added after this package was released, so the identifier #: states terms this package cannot name. Never treat it as #: :attr:`NOT_STATED`, which would read an identifier created under #: terms as one created under none. - UNKNOWN = "Unknown" + #: Its index is -1 rather than a real one. A Terms index read from a + #: payload is one byte, so it is 0 to 255 and can never be negative, + #: which is what makes -1 safe as the index that is not in the table. + UNKNOWN = (-1, None) + + def __init__(self, index: int, url: Optional[str]) -> None: + self._index = index + self._url = url @classmethod def from_index(cls, index: int) -> "Terms": @@ -86,6 +93,12 @@ def from_index(cls, index: int) -> "Terms": this package does not know.""" return _BY_INDEX.get(index, cls.UNKNOWN) + @property + def index(self) -> int: + """The Terms index this member is carried as in a payload, and -1 + for :attr:`UNKNOWN`, which has none of its own.""" + return self._index + @property def url(self) -> Optional[str]: """The address of the terms document, and ``None`` for @@ -94,16 +107,14 @@ def url(self) -> Optional[str]: The address is answered and never fetched, because what to do with the document is the receiver's decision.""" - return _URL[self] + return self._url +#: The table above read by index. It is built from the members rather than +#: written out a second time, so a member and its index can never disagree, +#: and a new terms document is one new member above and nothing here. The +#: negative index of :attr:`Terms.UNKNOWN` is left out, because it stands +#: for every index the table does not carry. _BY_INDEX = { - 0: Terms.NOT_STATED, - 1: Terms.MODEL_TERMS_FOR_MARKETING_2, -} - -_URL = { - Terms.NOT_STATED: None, - Terms.MODEL_TERMS_FOR_MARKETING_2: "https://m4ow.uk/mtm/2.txt", - Terms.UNKNOWN: None, + terms.index: terms for terms in Terms if terms.index >= 0 } diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 25a9baa..3ba7c0c 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -1121,11 +1121,37 @@ def test_only_a_named_document_has_an_address(self): self.assertEqual("https://m4ow.uk/mtm/2.txt", Terms.MODEL_TERMS_FOR_MARKETING_2.url) - def test_every_member_answers_the_address_question(self): - # A member added without an address entry would raise here rather - # than at the caller. + def test_every_member_agrees_with_the_table(self): + # Each member carries its own index and address, and the index to + # member map is built from the members, so this fails if a member + # does not read back from its own index or if one that names a + # document was added without an address. for member in Terms: - self.assertIn(member.url, (None, MODEL_TERMS_URL)) + if member is Terms.UNKNOWN: + # Stands for every index the table does not carry, so it + # has no index of its own and no address. + self.assertEqual(-1, member.index) + self.assertIsNone(member.url) + continue + self.assertIs( + member, Terms.from_index(member.index), + "{0} does not read back from its own index".format(member)) + if member is Terms.NOT_STATED: + # Names no document, so it has no address. + self.assertEqual(0, member.index) + self.assertIsNone(member.url) + else: + self.assertIsNotNone( + member.url, + "{0} names a document with no address".format(member)) + self.assertTrue(member.url.startswith("https://")) + + def test_no_two_members_share_an_index(self): + # One index stands for one document, so which document an + # identifier was created under never depends on the order the + # members happen to be written in. + indexes = [m.index for m in Terms if m.index >= 0] + self.assertEqual(len(indexes), len(set(indexes))) def test_the_named_value_is_not_part_of_the_package_surface(self): # The address on FodId is the whole of what a caller reads, so the From 0c7e2236728f039310c4c56ad207cd2d4312a1fa Mon Sep 17 00:00:00 2001 From: Oleksandr Lazarenko Date: Fri, 11 Sep 2026 07:48:15 -0400 Subject: [PATCH 6/7] DOC: Correct what the package says about the version refusal The readme told a caller to catch OwidError for a payload version this package cannot read. _unpack_or_raise raises ValueError for every status _read_payload returns, so the one new failure this branch introduces was the one a caller following the readme would not catch. The status joins the ValueError list, and the table of statuses, which described the version refusal in prose without ever listing it. Five counts still said two where the version check made them three, being the members added to the OWID status vocabulary, the payload rules _read_payload applies and the ones _from_read applies in turn. The test pinning the vocabulary was renamed to _plus_three; the prose around it was not. The Terms docstring told a caller to treat UNKNOWN as terms it cannot read and to refuse the identifier. No caller can meet UNKNOWN. It is a private member and terms answers None for it exactly as it does for NOT_STATED, which the package surface specification requires, so the paragraph described a surface removed earlier on this branch. It now says why the two reach a caller as one answer and why UNKNOWN is still a member of its own, being that nothing may compose an address from an index the table does not carry. The claim that members carry a name and not the index went with it, every member having carried its index since the terms table was made one place. _payload_length_valid kept the old wording about the bytes beyond the match key, which was reworded in fod_id.py, the readme and the envelope builder when the Terms byte took that position. The readme now also says that no identifier already issued is refused by the version check, the two bits having been written as zero by every issuer before the field was defined, which is the question a reader of that section asks. Also tidied whilst here: two PEP 8 blank-line slips in test_fodid.py, a docstring line in fod_id.py left at 92 characters by a reflow, and the terms constants in test_fodid.py moved to the top block with with_terms above its first caller. --- fiftyone_pipeline_did/readme.md | 22 ++++++--- .../src/fiftyone_pipeline_did/_terms.py | 46 +++++++++++-------- .../src/fiftyone_pipeline_did/did_client.py | 8 ++-- .../src/fiftyone_pipeline_did/fod_id.py | 23 +++++----- fiftyone_pipeline_did/tests/test_fodid.py | 41 +++++++++-------- 5 files changed, 81 insertions(+), 59 deletions(-) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index a05403a..0868176 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -199,6 +199,11 @@ The version is not exposed. Either this package read the layout, in which case the accessors are the answer, or it did not, in which case there is no identifier to read fields from. +No identifier already issued is refused by this. The two bits held no +field before the version was defined and every issuer wrote them as zero, +which is version 0, so an identifier from before the field existed reads +exactly as it did. + The raw flags byte, the byte layout constants and the old `hash` names are not part of this package. `fod_id.flags`, `fod_id.hash`, `fod_id.date_minutes`, `FodId.MATCH_KEY_OFFSET` and every other offset and @@ -251,7 +256,7 @@ reason. ### Status meanings The `FodIdParseStatus` vocabulary is the OWID one, member for member and -value for value, plus two members for the payload rules this package +value for value, plus three members for the payload rules this package applies once the envelope has been read. A failure inside the envelope is carried through with the OWID status unchanged, so the reason reads the same whichever language parsed the bytes. @@ -271,6 +276,7 @@ same whichever language parsed the bytes. | `MALFORMED_ENVELOPE` | Malformed in a way none of the above describes | | `PAYLOAD_TOO_SHORT` | The envelope was read but the payload is shorter than the 5 byte header, so the type cannot be read | | `INVALID_TYPE_PAYLOAD_LENGTH` | The header names a type whose match key needs more bytes than the payload holds | +| `UNSUPPORTED_PAYLOAD_VERSION` | Bits 4 and 5 of the flags byte name a payload layout version this package does not know, so no field is read | ### Lower bounds and no upper bound @@ -302,12 +308,14 @@ from the `try_` readers and never an exception. The raising readers, `from_base64`, `from_byte_array`, `from_owid` and the constructor, read through the same logic and keep their documented exceptions for callers who prefer them, being `TypeError` for `None` or a wrong input type, -`ValueError` for `PAYLOAD_TOO_SHORT` and `INVALID_TYPE_PAYLOAD_LENGTH`, -and `OwidError` for every other status, with the message naming the -status. Signature verification against a key that cannot be decoded, a -key list that cannot be fetched, and a cloud answer other than the one -asked for remain exceptions, because they are faults in the surroundings -and not properties of the identifier. +`ValueError` for `PAYLOAD_TOO_SHORT`, `INVALID_TYPE_PAYLOAD_LENGTH` and +`UNSUPPORTED_PAYLOAD_VERSION`, and `OwidError` for every other status, +with the message naming the status. The three raising `ValueError` are +the three the payload rules produce, whilst every other status comes +from the envelope. Signature verification against a key that cannot be +decoded, a key list that cannot be fetched, and a cloud answer other than +the one asked for remain exceptions, because they are faults in the +surroundings and not properties of the identifier. ### Migrating from the removed OWID API diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py index 73c726a..2acbf72 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py @@ -40,21 +40,29 @@ class Terms(Enum): repointed once published. It is specified at https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md - :attr:`NOT_STATED` and :attr:`UNKNOWN` are different answers and must - never be read as the same one. :attr:`NOT_STATED` says this identifier - does not carry the answer, so the answer has to come from somewhere - else, being the Terms Document Locator in an OpenRTB request or - whatever the surrounding protocol provides, and it does not mean the - identifier is unrestricted. :attr:`UNKNOWN` says the identifier does - state its terms and that this package cannot name them, because the - index was added after the package was released. A caller meeting - :attr:`UNKNOWN` should treat the identifier as covered by terms it - cannot yet read, and either update the package or refuse the - identifier. - - The members carry a name and not the index, because :attr:`UNKNOWN` - stands for any index this package does not know and so has no single - index to carry. + :attr:`NOT_STATED` and :attr:`UNKNOWN` are separate members here + because the index they came from is different, and they reach a caller + as one answer because neither names a document. :attr:`NOT_STATED` is + index 0 and says the identifier does not carry the answer, so it has + to come from somewhere else, being the Terms Document Locator in an + OpenRTB request or whatever the surrounding protocol provides, and it + does not mean the identifier is unrestricted. :attr:`UNKNOWN` stands + for an index added after this package was released, so the identifier + does state its terms and this package cannot name them. + + Both answer :attr:`url` with ``None``, which is what + :attr:`~fiftyone_pipeline_did.FodId.terms` hands a caller, so the two + cannot be told apart from outside the package. That is deliberate and + the specification requires it, because both leave a caller in the same + place, being that the identifier does not give them the terms and they + have to look elsewhere. What must never happen is an address composed + from an index the table does not carry, since that would name a + document nobody wrote, which is why :attr:`UNKNOWN` exists as a member + rather than the lookup falling back to :attr:`NOT_STATED`. + + Every member carries the index it is written as in a payload, and + :attr:`UNKNOWN` carries -1 because it stands for every index the table + does not carry and so has none of its own. This module is private to the package, as its underscore name says. The package turns the index into the address that @@ -75,9 +83,11 @@ class Terms(Enum): #: https://m4ow.uk/mtm/2.txt MODEL_TERMS_FOR_MARKETING_2 = (1, "https://m4ow.uk/mtm/2.txt") #: An index added after this package was released, so the identifier - #: states terms this package cannot name. Never treat it as - #: :attr:`NOT_STATED`, which would read an identifier created under - #: terms as one created under none. + #: states terms this package cannot name. It is a member of its own + #: rather than :attr:`NOT_STATED` so that the table keeps saying what + #: the payload said, being that terms were stated, and so that no + #: later reader of this module mistakes the index for one the table + #: carries and composes an address from it. #: Its index is -1 rather than a real one. A Terms index read from a #: payload is one byte, so it is 0 to 255 and can never be negative, #: which is what makes -1 safe as the index that is not in the table. diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 1219756..de7ef53 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -795,9 +795,11 @@ def _date_of(fod_id: FodId) -> datetime: def _payload_length_valid(fod_id: FodId) -> bool: """Whether the payload is at least the base length for its type, being five header bytes plus a 32 byte match key, or 16 for a Random - identifier. Anything beyond the base is a creator context section, - whose exact lengths belong to the cloud, so any longer payload is - accepted here.""" + identifier. Beyond the base come the Terms byte and then a creator + context section, whose exact lengths belong to the cloud, so any + longer payload is accepted here. The base does not include the Terms, + because a payload ending at the match key carries no byte for it and + reads as terms that are not stated.""" match_key_length = GUID_LENGTH if fod_id.type is IdType.RANDOM \ else MATCH_KEY_LENGTH return len(fod_id.payload) >= HEADER_LENGTH + match_key_length diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index 29e4806..272ccbd 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -62,11 +62,11 @@ class FodIdParseStatus(Enum): """Why reading a 51Did succeeded or failed. The vocabulary is the OWID one, member for member and value for value, - with two members added for the checks this package makes on the payload - once the envelope has been read. A failure in the envelope keeps the - OWID status unchanged, so a caller sees the same reason whichever + with three members added for the checks this package makes on the + payload once the envelope has been read. A failure in the envelope keeps + the OWID status unchanged, so a caller sees the same reason whichever language read the bytes, and a failure in the payload names which of the - two 51Did rules was broken. + three 51Did rules was broken. Every member other than :attr:`PARSED` is an expected outcome for data that arrived from outside, not a fault in the program. A parse that @@ -177,8 +177,8 @@ class FodId: Payload layout. Every field has a typed accessor here, being :attr:`type`, :attr:`usage`, :attr:`usage_from_consent`, :attr:`license_id`, :attr:`match_key` and :attr:`terms`, and those - accessors are the supported way to read an identifier. The bytes and offsets behind them - are specified at + accessors are the supported way to read an identifier. The bytes and + offsets behind them are specified at https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md and the surface this class offers, which is the same in every 51Did package, at @@ -257,8 +257,8 @@ def _build(cls, owid: Owid, flags: int, license_id: int, @classmethod def _from_read(cls, read: ParseResult) -> FodIdParseResult: """The non-raising reader over an OWID read. Carries an OWID failure - through unchanged, then applies the two 51Did payload rules, and - builds the identifier only when both have passed.""" + through unchanged, then applies the three 51Did payload rules, and + builds the identifier only when all of them have passed.""" if not read.ok: return _failed(FodIdParseStatus.of(read.status)) status, flags, license_id, match_key, terms_index = _read_payload( @@ -549,10 +549,11 @@ def _date_minutes(fod_id: "FodId") -> int: def _read_payload( payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes, int]: - """Applies the two 51Did payload rules and unpacks the four fields. + """Applies the three 51Did payload rules and unpacks the four fields. - The header must be present before the type can be read, and the type - then says how many match key bytes must follow. The Terms byte follows + The header must be present before the type can be read, the version it + carries must be one this package reads, and the type then says how many + match key bytes must follow. The Terms byte follows the match key, and anything beyond it is a creator context section whose lengths belong to the cloud, so a longer payload passes. A Reserved type has no known match key length and keeps the documented diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 3ba7c0c..42200f8 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -73,6 +73,18 @@ #: self-hosted container may be configured to use. LONG_DOMAIN = "identifiers." + ("a" * 120) + ".example" +#: The address the specification gives for Terms index 1, the Model Terms +#: for Marketing version 2. +MODEL_TERMS_URL = "https://m4ow.uk/mtm/2.txt" +#: The Terms index of the Model Terms for Marketing version 2. +MODEL_TERMS_INDEX = 1 +#: An index the specification has not assigned, standing for one added +#: after this package was released. +UNKNOWN_TERMS_INDEX = 200 +#: A creator context section. How long a section is belongs to the cloud +#: and changes with the section version, so an arbitrary length is used. +CONTEXT_SECTION = bytes(range(1, 24)) + def _write_license_id(payload): # Little-endian: low byte first (0x12345678 -> 78 56 34 12). @@ -82,6 +94,12 @@ def _write_license_id(payload): payload[LICENSE_ID_OFFSET + 3] = 0x12 +def with_terms(payload, index): + """The payload with a Terms byte after the match key, which is where + the specification puts it.""" + return bytearray(bytes(payload) + bytes([index])) + + def payload_ending_at_match_key(): """The canonical payload cut off at the end of the match key, so it carries no Terms byte. A reader takes that as a Terms of zero, and @@ -104,25 +122,6 @@ def canonical_payload(): return with_terms(payload_ending_at_match_key(), MODEL_TERMS_INDEX) -#: The address the specification gives for Terms index 1, the Model Terms -#: for Marketing version 2. -MODEL_TERMS_URL = "https://m4ow.uk/mtm/2.txt" -#: The Terms index of the Model Terms for Marketing version 2. -MODEL_TERMS_INDEX = 1 -#: An index the specification has not assigned, standing for one added -#: after this package was released. -UNKNOWN_TERMS_INDEX = 200 -#: A creator context section. How long a section is belongs to the cloud -#: and changes with the section version, so an arbitrary length is used. -CONTEXT_SECTION = bytes(range(1, 24)) - - -def with_terms(payload, index): - """The payload with a Terms byte after the match key, which is where - the specification puts it.""" - return bytearray(bytes(payload) + bytes([index])) - - def random_payload_ending_at_match_key(): """The canonical Random payload cut off at the end of its GUID.""" payload = bytearray(RANDOM_PAYLOAD_LENGTH) @@ -638,7 +637,7 @@ def test_success_does_not_verify_the_signature(self): fod = self.assert_parsed(FodId.try_from_byte_array(raw)) self.assertFalse(fod.verify(self.factory.public_pem)) - # ----- The two 51Did payload rules ----- + # ----- The three 51Did payload rules ----- def test_short_random_payload_reports_invalid_type_payload_length(self): payload = canonical_random_payload()[:RANDOM_PAYLOAD_LENGTH - 1] @@ -840,6 +839,7 @@ def test_raising_and_non_raising_readers_agree_on_success(self): result.value.as_byte_array()) self.assertEqual(raising.match_key, result.value.match_key) + class FodIdTermsTests(unittest.TestCase): """The Terms byte, which says which terms document the identifier was created under so that the terms travel with the identifier. The byte is @@ -1159,5 +1159,6 @@ def test_the_named_value_is_not_part_of_the_package_surface(self): import fiftyone_pipeline_did as package self.assertFalse(hasattr(package, "Terms")) + if __name__ == "__main__": unittest.main() From bc044ae21a18036af25b749e4cf9838e29fdfebf Mon Sep 17 00:00:00 2001 From: Oleksandr Lazarenko Date: Fri, 11 Sep 2026 07:48:22 -0400 Subject: [PATCH 7/7] REORG: One place records the index that states no terms ABSENT_TERMS_INDEX in _layout.py and Terms.NOT_STATED in _terms.py both recorded that index 0 says the terms are not stated in the identifier, so the fact the terms table was made the single place for was still written down twice and could disagree. _layout.py holds offsets and lengths, and an index value is neither. _read_terms_index now reads Terms.NOT_STATED.index, and the tests and the envelope builder that wrote the constant name the same member. TERMS_LENGTH stays in _layout.py, being a length. The assertion that the layout gives the constant the value zero is dropped, because test_every_member_agrees_with_the_table already pins NOT_STATED to index 0 and there is no second place left to disagree with it. --- .../src/fiftyone_pipeline_did/_layout.py | 5 ----- .../src/fiftyone_pipeline_did/fod_id.py | 5 +++-- fiftyone_pipeline_did/tests/envelope.py | 7 +++++-- fiftyone_pipeline_did/tests/test_fodid.py | 10 ++++------ 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py index 6aaba08..a52e522 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py @@ -69,8 +69,3 @@ #: of the flags byte. Any other version is refused rather than read under #: this layout. SUPPORTED_PAYLOAD_VERSION = 0 -#: The Terms index a payload with no byte after the match key reads as, -#: being the index that says the terms are not stated in the identifier. -#: Absence and zero mean the same thing and neither has to be told apart -#: from the other. -ABSENT_TERMS_INDEX = 0 diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index 272ccbd..7257019 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -27,7 +27,6 @@ from typing import NamedTuple, Optional, Tuple from ._layout import ( - ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, HEADER_LENGTH, @@ -608,7 +607,9 @@ def _read_terms_index(payload: bytes, offset: int) -> int: and not a defect, so no special case is written for it. """ if len(payload) < offset + TERMS_LENGTH: - return ABSENT_TERMS_INDEX + # Read off the table rather than written here a second time, so the + # index that says the terms are not stated is recorded once. + return Terms.NOT_STATED.index return payload[offset] diff --git a/fiftyone_pipeline_did/tests/envelope.py b/fiftyone_pipeline_did/tests/envelope.py index 1d08335..0ab7a09 100644 --- a/fiftyone_pipeline_did/tests/envelope.py +++ b/fiftyone_pipeline_did/tests/envelope.py @@ -33,12 +33,15 @@ from owid import io as owid_io from fiftyone_pipeline_did import FodId +# The named terms value is private to the package and is not exported, so +# the payload builders here reach the private module directly, as the +# package surface page says the package's own tests may. +from fiftyone_pipeline_did._terms import Terms # The byte layout is not part of the package's public surface. These tests # build payloads byte by byte, so they read it from the private module, as # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md # says the package's own tests may. from fiftyone_pipeline_did._layout import ( - ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, LICENSE_ID_OFFSET, @@ -80,7 +83,7 @@ def context_payload(): stated in the identifier, and then a creator context section. How long a section is belongs to the cloud and changes with the section version, so an arbitrary length is used here.""" - return (probabilistic_payload() + bytes([ABSENT_TERMS_INDEX]) + return (probabilistic_payload() + bytes([Terms.NOT_STATED.index]) + bytes(range(1, 24))) diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 42200f8..55592d0 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -47,7 +47,6 @@ # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md # says the package's own tests may. from fiftyone_pipeline_did._layout import ( - ABSENT_TERMS_INDEX, FLAGS_OFFSET, GUID_LENGTH, HEADER_LENGTH, @@ -136,7 +135,7 @@ def canonical_random_payload(): """The canonical Random payload as an issuer writes one, carrying the zero Terms byte a non-marketing identifier carries.""" return with_terms( - random_payload_ending_at_match_key(), ABSENT_TERMS_INDEX) + random_payload_ending_at_match_key(), Terms.NOT_STATED.index) def with_payload_version(payload, version): @@ -875,7 +874,7 @@ def test_payload_ending_at_the_match_key_has_no_address(self): def test_an_absent_byte_and_a_zero_byte_read_the_same(self): absent = self._read(payload_ending_at_match_key()) stated = self._read(with_terms(payload_ending_at_match_key(), - ABSENT_TERMS_INDEX)) + Terms.NOT_STATED.index)) self.assertEqual(absent.terms, stated.terms) self.assertIsNone(absent.terms) self.assertIsNone(stated.terms) @@ -919,7 +918,7 @@ def test_an_unknown_index_answers_as_no_terms_stated_does(self): unknown = self._read(with_terms(payload_ending_at_match_key(), UNKNOWN_TERMS_INDEX)) none = self._read(with_terms(payload_ending_at_match_key(), - ABSENT_TERMS_INDEX)) + Terms.NOT_STATED.index)) self.assertIsNone(none.terms) self.assertIsNone(unknown.terms) @@ -954,7 +953,7 @@ def test_a_context_section_alone_does_not_state_terms(self): # start of the context section, so a section opening with a zero # reads as terms that are not stated. built = (with_terms(payload_ending_at_match_key(), - ABSENT_TERMS_INDEX) + Terms.NOT_STATED.index) + CONTEXT_SECTION) fod = self._read(built) self.assertIsNone(fod.terms) @@ -1104,7 +1103,6 @@ class TermsTests(unittest.TestCase): def test_the_layout_gives_the_terms_one_byte(self): self.assertEqual(1, TERMS_LENGTH) - self.assertEqual(0, ABSENT_TERMS_INDEX) def test_from_index_names_the_indexes_the_package_knows(self): self.assertIs(Terms.NOT_STATED, Terms.from_index(0))