diff --git a/fiftyone_pipeline_did/examples/fodid_example.py b/fiftyone_pipeline_did/examples/fodid_example.py index 7cbd716..2012448 100644 --- a/fiftyone_pipeline_did/examples/fodid_example.py +++ b/fiftyone_pipeline_did/examples/fodid_example.py @@ -24,8 +24,8 @@ The 51Degrees Cloud service issues real 51Dids. To keep this example self-contained and offline, it builds a sample 51Did in process - generate an -ECDSA P-256 key pair, sign a canonical 37-byte payload - then parses it back -and prints the three payload fields. It also shows the headline use case: a +ECDSA P-256 key pair, sign a canonical 38-byte payload - then parses it back +and prints the payload fields. It also shows the headline use case: a 51Did is re-issued fresh on every call (the envelope, hence the base64, changes), but the match key is stable. Compare match keys, never envelopes. @@ -42,20 +42,27 @@ # from the package, which does not publish it. The layout is # specified at # https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md -SAMPLE_FLAGS = 0b0000_0011 # standard usage, Probabilistic type -SAMPLE_PAYLOAD_LENGTH = 37 # 1 flags byte, 4 licence id, 32 key +SAMPLE_FLAGS = 0b0000_0011 # standard usage, payload version 0, + # Probabilistic type +SAMPLE_PAYLOAD_LENGTH = 38 # 1 flags byte, 4 licence id, 32 key, + # 1 terms SAMPLE_MATCH_KEY_OFFSET = 5 SAMPLE_MATCH_KEY_LENGTH = 32 +SAMPLE_TERMS_OFFSET = 37 # the Terms byte follows the match key +SAMPLE_TERMS = 1 # index 1, the Model Terms for Marketing + # version 2 def sample_payload(): - """A canonical 37-byte Probabilistic payload: the flags byte, License - Id 0x12345678 (little-endian) and a 32-byte match key 0x20..0x3F.""" + """A canonical 38-byte Probabilistic payload: the flags byte, License + Id 0x12345678 (little-endian), a 32-byte match key 0x20..0x3F and the + Terms byte.""" payload = bytearray(SAMPLE_PAYLOAD_LENGTH) payload[0] = SAMPLE_FLAGS payload[1:5] = bytes([0x78, 0x56, 0x34, 0x12]) for i in range(SAMPLE_MATCH_KEY_LENGTH): payload[SAMPLE_MATCH_KEY_OFFSET + i] = 0x20 + i + payload[SAMPLE_TERMS_OFFSET] = SAMPLE_TERMS return bytes(payload) @@ -83,6 +90,9 @@ def run(): print(" Consent :", fod_id.usage_from_consent) print(" LicenseId :", fod_id.license_id) print(" Match key :", fod_id.match_key.hex()) + # The address is answered and never fetched. What to do with the + # document is the receiver's decision. + print(" Terms :", fod_id.terms) print(" Verifies :", fod_id.verify(crypto.public_key_pem())) reissued = FodId.from_base64(issue(creator, payload)) diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index d9dbc1d..0868176 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -14,6 +14,10 @@ Identifier) returned by the 51Degrees Cloud service. Mirrors the .NET the Flags and License Id, being a 32-byte SHA-256 for Probabilistic and HashedEmail identifiers, or 16 GUID bytes for Random. Two 51Dids for the same inputs share the same match key even though their envelopes differ. +- The **Terms** is the byte after the match key that says which terms + document the identifier was created under, so the terms travel with the + identifier instead of alongside it. The package turns that byte into the + address of the document. **Comparing two 51Dids means comparing their match keys, never their envelopes.** @@ -31,8 +35,11 @@ where the two ever disagree. In short, the payload opens with a header carrying the flags byte and the License Id, and the identifier type in that byte then fixes the length of the match key that follows, being 32 bytes for `PROBABILISTIC` and -`HASHED_EMAIL`, 16 for `RANDOM`, and whatever remains for `RESERVED`. -Identifiers issued before the type tag existed decode as `PROBABILISTIC`. +`HASHED_EMAIL`, 16 for `RANDOM`, and whatever remains for `RESERVED`. The +Terms byte follows the match key, and the bytes after it are a creator +context section. Identifiers issued before the type tag existed decode as +`PROBABILISTIC`, and a payload that ends at the match key reads as terms +that are not stated. This package does not publish the offsets or the raw flags byte, and it does not need to, because every field has a typed accessor that reads it @@ -93,6 +100,9 @@ from_consent = fod_id.usage_from_consent # True when read from a consent # string the caller sent license_id = fod_id.license_id match_key = fod_id.match_key # SHA-256 or GUID bytes, see type +terms = fod_id.terms # address of the terms document it was + # created under, None where it names none + # this package knows # Delegated OWID-level fields and operations. domain = fod_id.domain @@ -130,6 +140,70 @@ from an IAB consent string the caller sent rather than stated by the caller directly. Both are legitimate ways to arrive at a usage and it says nothing about which usage was reached. +### The terms an identifier was created under + +`fod_id.terms` answers with the address of the terms document the +identifier was created under, so the terms travel with the identifier +instead of alongside it and a receiver can tell which document was in +force when the identifier was made. The byte behind it is an index into a +table of terms documents in the +[layout specification](https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md), +and the package turns the index into the address so a caller never handles +the byte. The byte is an index and not a version number, so that a later +document can live at any address rather than only at one the specification +could compose from a number, and an index is never reused or repointed +once published, because an identifier issued under it has to stay readable +years later. + +| Index | Document | `fod_id.terms` | +|------:|----------|----------------| +| `0` | Not stated in the identifier | `None` | +| `1` | Model Terms for Marketing, version 2 | `https://m4ow.uk/mtm/2.txt` | +| any other | One this package cannot name | `None` | + +The answer is `None` where the identifier does not state its terms and +where it states an index added after this package was released, using +absence rather than an empty string. No address is ever built from an +index this package cannot name, because that would name a document nobody +wrote and a receiver would record having accepted terms that do not exist. +A caller therefore cannot tell those two apart, which is deliberate, since +both lead to the same place. This package answers with the address and +never fetches it, because what to do with the document is the receiver's +decision. + +No address does not mean the identifier is unrestricted. It says only that +this identifier does not carry the answer, so the answer has to come from +somewhere else, being the Terms Document Locator in an OpenRTB request or +whatever the surrounding protocol provides. + +A payload with no byte after the match key reads as index 0, so absence +and zero mean the same thing and no presence flag exists to tell them +apart. The Usage and the Terms answer different questions and a +receiver needs both, because the Usage says where an identifier may go and +the Terms says under which document it was created. + +### The payload version + +Bits 4 and 5 of the flags byte say which payload layout the identifier +follows, and this package reads version 0. A payload naming version 1, 2 +or 3 is refused with `FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION`, and +the raising readers name the version they found in the message. + +No field is read under the layout this package knows once the version says +otherwise. A later version exists precisely because a field moved, so +reading such a payload here would answer with values that are wrong rather +than absent, which is worse than refusing. A version that nothing checks +protects nothing. + +The version is not exposed. Either this package read the layout, in which +case the accessors are the answer, or it did not, in which case there is +no identifier to read fields from. + +No identifier already issued is refused by this. The two bits held no +field before the version was defined and every issuer wrote them as zero, +which is version 0, so an identifier from before the field existed reads +exactly as it did. + The raw flags byte, the byte layout constants and the old `hash` names are not part of this package. `fod_id.flags`, `fod_id.hash`, `fod_id.date_minutes`, `FodId.MATCH_KEY_OFFSET` and every other offset and @@ -182,7 +256,7 @@ reason. ### Status meanings The `FodIdParseStatus` vocabulary is the OWID one, member for member and -value for value, plus two members for the payload rules this package +value for value, plus three members for the payload rules this package applies once the envelope has been read. A failure inside the envelope is carried through with the OWID status unchanged, so the reason reads the same whichever language parsed the bytes. @@ -202,20 +276,24 @@ same whichever language parsed the bytes. | `MALFORMED_ENVELOPE` | Malformed in a way none of the above describes | | `PAYLOAD_TOO_SHORT` | The envelope was read but the payload is shorter than the 5 byte header, so the type cannot be read | | `INVALID_TYPE_PAYLOAD_LENGTH` | The header names a type whose match key needs more bytes than the payload holds | +| `UNSUPPORTED_PAYLOAD_VERSION` | Bits 4 and 5 of the flags byte name a payload layout version this package does not know, so no field is read | ### Lower bounds and no upper bound The payload must hold the 5 byte header before the type can be read, and the type then says how many match key bytes must follow, being 16 for `RANDOM` and 32 for `PROBABILISTIC` and `HASHED_EMAIL`, as the payload -layout specification says. `RESERVED` keeps the best-effort reading, being -the header fields and whatever bytes follow. Anything beyond the match key +layout specification says. The Terms byte follows the match key, and a +payload ending at the match key reads as terms that are not stated. +`RESERVED` keeps the best-effort reading, being the header fields and +whatever bytes follow, and because no match key length is defined for it +there is no byte left over to read as its Terms. Anything beyond the Terms is a creator context section whose lengths belong to the cloud, so a longer payload, a longer creator domain (a self-hosted container may sign with one) or a longer envelope is accepted and this package places no upper bound of its own on any of them. An older reader meeting a context -section of a version it does not know still reads the header and the -match key. +section of a version it does not know still reads the header, the match +key and the Terms. `DidClient` refuses text longer than 4096 characters before it parses it, fetches a key or calls the cloud. That figure is client policy, @@ -230,12 +308,14 @@ from the `try_` readers and never an exception. The raising readers, `from_base64`, `from_byte_array`, `from_owid` and the constructor, read through the same logic and keep their documented exceptions for callers who prefer them, being `TypeError` for `None` or a wrong input type, -`ValueError` for `PAYLOAD_TOO_SHORT` and `INVALID_TYPE_PAYLOAD_LENGTH`, -and `OwidError` for every other status, with the message naming the -status. Signature verification against a key that cannot be decoded, a -key list that cannot be fetched, and a cloud answer other than the one -asked for remain exceptions, because they are faults in the surroundings -and not properties of the identifier. +`ValueError` for `PAYLOAD_TOO_SHORT`, `INVALID_TYPE_PAYLOAD_LENGTH` and +`UNSUPPORTED_PAYLOAD_VERSION`, and `OwidError` for every other status, +with the message naming the status. The three raising `ValueError` are +the three the payload rules produce, whilst every other status comes +from the envelope. Signature verification against a key that cannot be +decoded, a key list that cannot be fetched, and a cloud answer other than +the one asked for remain exceptions, because they are faults in the +surroundings and not properties of the identifier. ### Migrating from the removed OWID API @@ -533,9 +613,12 @@ is refreshed by common-ci's `update-example-assets` step. - **No signature verification on parsing.** A parsed 51Did is not known to be genuine. Call `verify(public_key_pem)`, `signature_status(public_key_pem)` or a `DidClient` check when needed. +- **No fetching of a terms document.** `fod_id.terms` answers with the + address and nothing more. What to do with the document is the receiver's + decision. - **No upper bound on the size of an identifier.** The lengths beyond the - header and match key belong to the cloud. The 4096 character figure in - `DidClient` is client policy against obviously malformed text, not a - format limit. + header, match key and Terms belong to the cloud. The 4096 character + figure in `DidClient` is client policy against obviously malformed text, + not a format limit. - **No creation of new 51Dids.** This is a parser; new 51Dids are issued by the 51Degrees cloud / on-premise hashing engines. diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py index ccd920e..4d7373e 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py @@ -27,7 +27,8 @@ OWID form in either alphabet, exposes a typed accessor for every field it carries (the identifier :class:`~fiftyone_pipeline_did.id_type.IdType`, the :class:`~fiftyone_pipeline_did.usage.Usage` it was created for and whether -that usage came from a consent string, the License Id and the match key), +that usage came from a consent string, the License Id, the match key and +the address of the terms document it was created under), and delegates OWID-level concerns to the wrapped envelope. The raw bytes and offsets behind those accessors are not part of this surface, which is specified at diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py index c55ad94..a52e522 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_layout.py @@ -61,3 +61,11 @@ RANDOM_PAYLOAD_LENGTH = HEADER_LENGTH + GUID_LENGTH #: Minimum byte length of a Probabilistic or HashedEmail 51Did payload. PAYLOAD_LENGTH = MATCH_KEY_OFFSET + MATCH_KEY_LENGTH +#: Byte length of the Terms field, which follows the match key. There is no +#: offset constant for it, because the identifier type fixes the match key +#: length and so the type says where the field starts. +TERMS_LENGTH = 1 +#: The payload layout version this package reads, carried in bits 4 and 5 +#: of the flags byte. Any other version is refused rather than read under +#: this layout. +SUPPORTED_PAYLOAD_VERSION = 0 diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py new file mode 100644 index 0000000..2acbf72 --- /dev/null +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/_terms.py @@ -0,0 +1,130 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + + +from enum import Enum +from typing import Optional + + +class Terms(Enum): + """The terms document a 51Did was created under, read from the + identifier through :attr:`~fiftyone_pipeline_did.FodId.terms`, so that + the terms travel with the identifier instead of alongside it and a + receiver can tell which document was in force when the identifier was + made. + + The payload carries an index into a table of terms documents and not a + version number, so that a later document can live at any address rather + than only at one a number could be put into. That table is the whole of + the definition, so a new terms document is a new index there and every + package has to be released to know it, and an index is never reused or + repointed once published. It is specified at + https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md + + :attr:`NOT_STATED` and :attr:`UNKNOWN` are separate members here + because the index they came from is different, and they reach a caller + as one answer because neither names a document. :attr:`NOT_STATED` is + index 0 and says the identifier does not carry the answer, so it has + to come from somewhere else, being the Terms Document Locator in an + OpenRTB request or whatever the surrounding protocol provides, and it + does not mean the identifier is unrestricted. :attr:`UNKNOWN` stands + for an index added after this package was released, so the identifier + does state its terms and this package cannot name them. + + Both answer :attr:`url` with ``None``, which is what + :attr:`~fiftyone_pipeline_did.FodId.terms` hands a caller, so the two + cannot be told apart from outside the package. That is deliberate and + the specification requires it, because both leave a caller in the same + place, being that the identifier does not give them the terms and they + have to look elsewhere. What must never happen is an address composed + from an index the table does not carry, since that would name a + document nobody wrote, which is why :attr:`UNKNOWN` exists as a member + rather than the lookup falling back to :attr:`NOT_STATED`. + + Every member carries the index it is written as in a payload, and + :attr:`UNKNOWN` carries -1 because it stands for every index the table + does not carry and so has none of its own. + + This module is private to the package, as its underscore name says. The + package turns the index into the address that + :attr:`~fiftyone_pipeline_did.FodId.terms` answers with, so a caller + never handles the byte, and the names here are the ones the + specification gives so that every package describes one document the + same way. + + The Usage and the Terms answer different questions and a receiver needs + both, because the Usage says where an identifier may go and the Terms + says under which document it was created.""" + + #: The terms are not stated in the identifier, which is index 0 and + #: also what an identifier whose payload ends at the match key reads + #: as. It names no document, so it has no address. + NOT_STATED = (0, None) + #: Index 1, the Model Terms for Marketing version 2, at + #: https://m4ow.uk/mtm/2.txt + MODEL_TERMS_FOR_MARKETING_2 = (1, "https://m4ow.uk/mtm/2.txt") + #: An index added after this package was released, so the identifier + #: states terms this package cannot name. It is a member of its own + #: rather than :attr:`NOT_STATED` so that the table keeps saying what + #: the payload said, being that terms were stated, and so that no + #: later reader of this module mistakes the index for one the table + #: carries and composes an address from it. + #: Its index is -1 rather than a real one. A Terms index read from a + #: payload is one byte, so it is 0 to 255 and can never be negative, + #: which is what makes -1 safe as the index that is not in the table. + UNKNOWN = (-1, None) + + def __init__(self, index: int, url: Optional[str]) -> None: + self._index = index + self._url = url + + @classmethod + def from_index(cls, index: int) -> "Terms": + """The member for a Terms index, and :attr:`UNKNOWN` for an index + this package does not know.""" + return _BY_INDEX.get(index, cls.UNKNOWN) + + @property + def index(self) -> int: + """The Terms index this member is carried as in a payload, and -1 + for :attr:`UNKNOWN`, which has none of its own.""" + return self._index + + @property + def url(self) -> Optional[str]: + """The address of the terms document, and ``None`` for + :attr:`NOT_STATED` and for :attr:`UNKNOWN`, where there is no + document this package can name. + + The address is answered and never fetched, because what to do with + the document is the receiver's decision.""" + return self._url + + +#: The table above read by index. It is built from the members rather than +#: written out a second time, so a member and its index can never disagree, +#: and a new terms document is one new member above and nothing here. The +#: negative index of :attr:`Terms.UNKNOWN` is left out, because it stands +#: for every index the table does not carry. +_BY_INDEX = { + terms.index: terms for terms in Terms if terms.index >= 0 +} diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 1219756..de7ef53 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -795,9 +795,11 @@ def _date_of(fod_id: FodId) -> datetime: def _payload_length_valid(fod_id: FodId) -> bool: """Whether the payload is at least the base length for its type, being five header bytes plus a 32 byte match key, or 16 for a Random - identifier. Anything beyond the base is a creator context section, - whose exact lengths belong to the cloud, so any longer payload is - accepted here.""" + identifier. Beyond the base come the Terms byte and then a creator + context section, whose exact lengths belong to the cloud, so any + longer payload is accepted here. The base does not include the Terms, + because a payload ending at the match key carries no byte for it and + reads as terms that are not stated.""" match_key_length = GUID_LENGTH if fod_id.type is IdType.RANDOM \ else MATCH_KEY_LENGTH return len(fod_id.payload) >= HEADER_LENGTH + match_key_length diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index 118608e..7257019 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -34,6 +34,8 @@ LICENSE_ID_OFFSET, MATCH_KEY_LENGTH, MATCH_KEY_OFFSET, + SUPPORTED_PAYLOAD_VERSION, + TERMS_LENGTH, ) from ._owid import ( Owid, @@ -45,6 +47,7 @@ ) from .id_type import IdType +from ._terms import Terms from .usage import Usage #: The moment the envelope's date field counts minutes from, being the OWID @@ -58,11 +61,11 @@ class FodIdParseStatus(Enum): """Why reading a 51Did succeeded or failed. The vocabulary is the OWID one, member for member and value for value, - with two members added for the checks this package makes on the payload - once the envelope has been read. A failure in the envelope keeps the - OWID status unchanged, so a caller sees the same reason whichever + with three members added for the checks this package makes on the + payload once the envelope has been read. A failure in the envelope keeps + the OWID status unchanged, so a caller sees the same reason whichever language read the bytes, and a failure in the payload names which of the - two 51Did rules was broken. + three 51Did rules was broken. Every member other than :attr:`PARSED` is an expected outcome for data that arrived from outside, not a fault in the program. A parse that @@ -113,6 +116,12 @@ class FodIdParseStatus(Enum): #: the header for Random and a 32 byte SHA-256 match key for #: Probabilistic and HashedEmail. INVALID_TYPE_PAYLOAD_LENGTH = "InvalidTypePayloadLength" + #: Bits 4 and 5 of the flags byte name a payload layout version this + #: package does not know, so no field is read. A later version exists + #: precisely because a field moved, so reading the payload under the + #: layout this package knows would answer with values that are wrong + #: rather than absent. + UNSUPPORTED_PAYLOAD_VERSION = "UnsupportedPayloadVersion" @classmethod def of(cls, status: ParseStatus) -> "FodIdParseStatus": @@ -166,9 +175,9 @@ class FodId: Payload layout. Every field has a typed accessor here, being :attr:`type`, :attr:`usage`, :attr:`usage_from_consent`, - :attr:`license_id` and :attr:`match_key`, and those accessors are the - supported way to read an identifier. The bytes and offsets behind them - are specified at + :attr:`license_id`, :attr:`match_key` and :attr:`terms`, and those + accessors are the supported way to read an identifier. The bytes and + offsets behind them are specified at https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md and the surface this class offers, which is the same in every 51Did package, at @@ -176,10 +185,21 @@ class FodId: which are the authority for both. In short, the header is shared by every identifier type and the type then fixes the length of the match key that follows, being a 32-byte SHA-256 for Probabilistic and - HashedEmail or 16 GUID bytes for Random. A payload longer than the - header and match key is accepted, because the bytes after the match key - are a creator context section whose lengths belong to the cloud, so - this package places no upper bound on a payload or an envelope. + HashedEmail or 16 GUID bytes for Random, and the Terms byte follows the + match key. A payload longer than that is accepted, because the bytes + after the Terms are a creator context section whose lengths belong to + the cloud, so this package places no upper bound on a payload or an + envelope. A payload that ends at the match key reads as terms that are + not stated. + + Bits 4 and 5 of the flags byte say which payload layout the identifier + follows, and this package reads version 0. A payload naming any other + version is refused with + :attr:`FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION` rather than read + under the layout this package knows, because a later version exists + precisely because a field moved, so reading one here would answer with + values that are wrong rather than absent. The version is not exposed, + because a caller has nothing to decide with it. Reading and verifying are separate steps. :meth:`try_from_base64` and :meth:`try_from_byte_array` read external data without raising and @@ -217,34 +237,36 @@ def __init__(self, owid: Owid) -> None: self._assign(read.owid, *_unpack_or_raise(read.owid.payload)) def _assign(self, owid: Owid, flags: int, license_id: int, - match_key: bytes) -> None: + match_key: bytes, terms_index: int) -> None: self._owid = owid self._flags = flags self._license_id = license_id self._match_key = match_key + self._terms_index = terms_index @classmethod def _build(cls, owid: Owid, flags: int, license_id: int, - match_key: bytes) -> "FodId": + match_key: bytes, terms_index: int) -> "FodId": """An identifier over fields :func:`_read_payload` has already checked, so the constructor's read is not repeated.""" fod_id = cls.__new__(cls) - fod_id._assign(owid, flags, license_id, match_key) + fod_id._assign(owid, flags, license_id, match_key, terms_index) return fod_id @classmethod def _from_read(cls, read: ParseResult) -> FodIdParseResult: """The non-raising reader over an OWID read. Carries an OWID failure - through unchanged, then applies the two 51Did payload rules, and - builds the identifier only when both have passed.""" + through unchanged, then applies the three 51Did payload rules, and + builds the identifier only when all of them have passed.""" if not read.ok: return _failed(FodIdParseStatus.of(read.status)) - status, flags, license_id, match_key = _read_payload( + status, flags, license_id, match_key, terms_index = _read_payload( read.owid.payload) if status is not FodIdParseStatus.PARSED: return _failed(status) return FodIdParseResult( - True, cls._build(read.owid, flags, license_id, match_key), + True, + cls._build(read.owid, flags, license_id, match_key, terms_index), FodIdParseStatus.PARSED) @classmethod @@ -411,6 +433,34 @@ def match_key(self) -> bytes: """ return self._match_key + @property + def terms(self) -> Optional[str]: + """The address of the terms document the identifier was created + under, so the terms travel with the identifier instead of + alongside it. + + The byte after the match key is an index into a table in the + specification and this package turns the index into the address, + so a caller never handles the byte. The address is answered and + never fetched, because what to do with the document is the + receiver's decision. + + ``None`` covers both an index of zero, which says the terms are + not stated in the identifier, and an index added to the table + after this package was released, which it cannot name. A caller + cannot tell those two apart, which is deliberate, because both + lead to the same place, being that the identifier does not say + which terms it was created under and the answer has to come from + somewhere else. No address is ever built from an index this + package does not know, since that would name a document nobody + wrote. + + No address does not mean the identifier is unrestricted. Where an + identifier may go is a separate question :attr:`usage` answers, + which still bars a non-marketing identifier from a demand source. + """ + return Terms.from_index(self._terms_index).url + @property def version(self) -> Version: """The OWID version.""" @@ -496,25 +546,36 @@ def _date_minutes(fod_id: "FodId") -> int: return int((fod_id.date - DATE_EPOCH).total_seconds() // 60) -def _read_payload(payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes]: - """Applies the two 51Did payload rules and unpacks the three fields. +def _read_payload( + payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes, int]: + """Applies the three 51Did payload rules and unpacks the four fields. - The header must be present before the type can be read, and the type - then says how many match key bytes must follow. Anything beyond the - match key is a creator context section whose lengths belong to the - cloud, so a longer payload passes. A Reserved type has no known match - key length and keeps the documented best-effort reading, being the - header fields and whatever bytes follow. + The header must be present before the type can be read, the version it + carries must be one this package reads, and the type then says how many + match key bytes must follow. The Terms byte follows + the match key, and anything beyond it is a creator context section + whose lengths belong to the cloud, so a longer payload passes. A + Reserved type has no known match key length and keeps the documented + best-effort reading, being the header fields and whatever bytes follow. - Returns the status and, on success, the flags, the licence id and the - match key bytes. On failure the three fields are zero and empty. + Returns the status and, on success, the flags, the licence id, the + match key bytes and the Terms index. On failure the four fields are + zero and empty. """ if payload is None or len(payload) < HEADER_LENGTH: - return FodIdParseStatus.PAYLOAD_TOO_SHORT, 0, 0, b"" + return FodIdParseStatus.PAYLOAD_TOO_SHORT, 0, 0, b"", 0 flags = payload[FLAGS_OFFSET] + # The version is read before any field, because a later version exists + # precisely because a field moved. Reading a payload of a version this + # package does not know under the layout it does know would answer with + # values that are wrong rather than absent, which is worse than + # refusing, and a version that nothing checks protects nothing. + if _payload_version(flags) != SUPPORTED_PAYLOAD_VERSION: + return ( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, 0, 0, b"", 0) match_key_length = _match_key_length(IdType.from_flags(flags), payload) if len(payload) < HEADER_LENGTH + match_key_length: - return FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, 0, 0, b"" + return FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, 0, 0, b"", 0 # Little-endian uint32, unsigned (Python ints are unbounded and # non-negative here, so the high bit never becomes negative). license_id = int.from_bytes( @@ -526,16 +587,48 @@ def _read_payload(payload: bytes) -> Tuple[FodIdParseStatus, int, int, bytes]: # to change the underlying payload and no defensive copy is required. match_key = bytes( payload[MATCH_KEY_OFFSET:MATCH_KEY_OFFSET + match_key_length]) - return FodIdParseStatus.PARSED, flags, license_id, match_key + return (FodIdParseStatus.PARSED, flags, license_id, match_key, + _read_terms_index(payload, MATCH_KEY_OFFSET + match_key_length)) -def _unpack_or_raise(payload: bytes) -> Tuple[int, int, bytes]: +def _read_terms_index(payload: bytes, offset: int) -> int: + """The Terms byte at the offset the match key ends at, and the index + that says the terms are not stated where the payload ends there. + + A payload ending at the match key has no byte to read, so absence and + zero mean the same thing and neither has to be told apart from the + other. + + A Reserved type cannot carry a Terms byte this package can find, + because no match key length is defined for that type and so every byte + after the header is its match key. The offset then lands at the end of + the payload, nothing is left to read, and the identifier answers with + the index that says the terms are not stated. That is the right answer + and not a defect, so no special case is written for it. + """ + if len(payload) < offset + TERMS_LENGTH: + # Read off the table rather than written here a second time, so the + # index that says the terms are not stated is recorded once. + return Terms.NOT_STATED.index + return payload[offset] + + +def _unpack_or_raise(payload: bytes) -> Tuple[int, int, bytes, int]: """The payload rules for the raising readers, with the messages they have always given.""" - status, flags, license_id, match_key = _read_payload(payload) + status, flags, license_id, match_key, terms_index = _read_payload(payload) if status is not FodIdParseStatus.PARSED: raise ValueError(_payload_message(status, payload)) - return flags, license_id, match_key + return flags, license_id, match_key, terms_index + + +def _payload_version(flags: int) -> int: + """Bits 4 and 5 of the flags byte, being the version of the payload + layout the identifier follows. The envelope carries a version of its + own at its first byte, which versions the envelope, whilst this one + versions the payload. + """ + return (flags >> 4) & 0b11 def _match_key_length(id_type: IdType, payload: bytes) -> int: @@ -553,6 +646,10 @@ def _payload_message(status: FodIdParseStatus, payload: bytes) -> str: if status is FodIdParseStatus.PAYLOAD_TOO_SHORT: return "51Did payload must be at least {0} bytes; got {1}.".format( HEADER_LENGTH, length) + if status is FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION: + return ( + "51Did payload version {0} is not one this package can " + "read.".format(_payload_version(payload[FLAGS_OFFSET]))) id_type = IdType.from_flags(payload[FLAGS_OFFSET]) return ("51Did payload for the {0} type must be at least {1} bytes; " "got {2}.".format( diff --git a/fiftyone_pipeline_did/tests/envelope.py b/fiftyone_pipeline_did/tests/envelope.py index 6049028..0ab7a09 100644 --- a/fiftyone_pipeline_did/tests/envelope.py +++ b/fiftyone_pipeline_did/tests/envelope.py @@ -33,6 +33,10 @@ from owid import io as owid_io from fiftyone_pipeline_did import FodId +# The named terms value is private to the package and is not exported, so +# the payload builders here reach the private module directly, as the +# package surface page says the package's own tests may. +from fiftyone_pipeline_did._terms import Terms # The byte layout is not part of the package's public surface. These tests # build payloads byte by byte, so they read it from the private module, as # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md @@ -75,10 +79,12 @@ def random_payload(): def context_payload(): - """A Probabilistic payload followed by a creator context - section. How long a section is belongs to the cloud and changes with - the section version, so an arbitrary length is used here.""" - return probabilistic_payload() + bytes([0]) + bytes(range(1, 24)) + """A Probabilistic payload, a Terms byte saying the terms are not + stated in the identifier, and then a creator context section. How long + a section is belongs to the cloud and changes with the section version, + so an arbitrary length is used here.""" + return (probabilistic_payload() + bytes([Terms.NOT_STATED.index]) + + bytes(range(1, 24))) def envelope_bytes(crypto, payload, date=None, version=Version.VERSION3, diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 6536ba8..55592d0 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -37,6 +37,11 @@ SignatureStatus, ) +# The named terms value is private to the package and is not exported, so +# the vocabulary tests below reach the private module directly, as the +# package surface page says the package's own tests may. +from fiftyone_pipeline_did._terms import Terms + # The byte layout is not part of the package's public surface. These tests # build payloads byte by byte, so they read it from the private module, as # https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md @@ -51,12 +56,15 @@ MATCH_KEY_OFFSET, PAYLOAD_LENGTH, RANDOM_PAYLOAD_LENGTH, + SUPPORTED_PAYLOAD_VERSION, + TERMS_LENGTH, ) from .envelope import envelope_bytes, signed_envelope TEST_DOMAIN = "51degrees.com" -# 0xA5: usage bits plus the HashedEmail type tag in bits 6-7. -CANONICAL_FLAGS = 0xA5 +# 0x85: the personalized marketing usage in bits 0-2, the payload version 0 +# in bits 4-5 and the HashedEmail type tag in bits 6-7. +CANONICAL_FLAGS = 0x85 CANONICAL_LICENSE_ID = 0x12345678 CANONICAL_MATCH_KEY = bytes((0x20 + i) for i in range(MATCH_KEY_LENGTH)) @@ -64,6 +72,18 @@ #: self-hosted container may be configured to use. LONG_DOMAIN = "identifiers." + ("a" * 120) + ".example" +#: The address the specification gives for Terms index 1, the Model Terms +#: for Marketing version 2. +MODEL_TERMS_URL = "https://m4ow.uk/mtm/2.txt" +#: The Terms index of the Model Terms for Marketing version 2. +MODEL_TERMS_INDEX = 1 +#: An index the specification has not assigned, standing for one added +#: after this package was released. +UNKNOWN_TERMS_INDEX = 200 +#: A creator context section. How long a section is belongs to the cloud +#: and changes with the section version, so an arbitrary length is used. +CONTEXT_SECTION = bytes(range(1, 24)) + def _write_license_id(payload): # Little-endian: low byte first (0x12345678 -> 78 56 34 12). @@ -73,7 +93,17 @@ def _write_license_id(payload): payload[LICENSE_ID_OFFSET + 3] = 0x12 -def canonical_payload(): +def with_terms(payload, index): + """The payload with a Terms byte after the match key, which is where + the specification puts it.""" + return bytearray(bytes(payload) + bytes([index])) + + +def payload_ending_at_match_key(): + """The canonical payload cut off at the end of the match key, so it + carries no Terms byte. A reader takes that as a Terms of zero, and + this is the fixture for that rule rather than anything an issuer + would write.""" payload = bytearray(PAYLOAD_LENGTH) payload[FLAGS_OFFSET] = CANONICAL_FLAGS _write_license_id(payload) @@ -83,7 +113,16 @@ def canonical_payload(): return bytearray(payload) -def canonical_random_payload(): +def canonical_payload(): + """The canonical payload as an issuer writes one, carrying the + payload version 0 in its flags byte and the Terms byte of the + document a personalized marketing identifier is created under. This + is the creating side, so it writes every field an issuer writes.""" + return with_terms(payload_ending_at_match_key(), MODEL_TERMS_INDEX) + + +def random_payload_ending_at_match_key(): + """The canonical Random payload cut off at the end of its GUID.""" payload = bytearray(RANDOM_PAYLOAD_LENGTH) payload[FLAGS_OFFSET] = (1 << 6) | 0b001 # Random tag + usage bits _write_license_id(payload) @@ -92,6 +131,22 @@ def canonical_random_payload(): return bytearray(payload) +def canonical_random_payload(): + """The canonical Random payload as an issuer writes one, carrying the + zero Terms byte a non-marketing identifier carries.""" + return with_terms( + random_payload_ending_at_match_key(), Terms.NOT_STATED.index) + + +def with_payload_version(payload, version): + """The payload with its version bits set to the given version, leaving + every other bit of the flags byte alone.""" + changed = bytearray(payload) + changed[FLAGS_OFFSET] = ( + (payload[FLAGS_OFFSET] & 0b1100_1111) | (version << 4)) + return changed + + class FodIdTestFactory: """Generates a fresh ECDSA P-256 key pair and signs real OWID envelopes. A Creator is the only way the OWID library brings a new @@ -214,11 +269,15 @@ def test_flags_byte_of_zero_is_read(self): fod = FodId.from_base64(self.factory.signed_owid_base64(payload)) self.assertEqual(0, fod._flags) - def test_flags_byte_of_all_bits_set_is_read(self): + def test_every_flags_bit_outside_the_version_is_read(self): + # Bits 4 and 5 are the payload version and only version 0 is read, + # so every other bit is set and those two are left clear. A payload + # with them set is refused rather than read, which + # FodIdVersionTests covers. payload = canonical_payload() - payload[FLAGS_OFFSET] = 0xFF + payload[FLAGS_OFFSET] = 0xCF fod = FodId.from_base64(self.factory.signed_owid_base64(payload)) - self.assertEqual(255, fod._flags) + self.assertEqual(0xCF, fod._flags) def test_match_key_is_immutable(self): fod = FodId.from_base64( @@ -487,9 +546,9 @@ def assert_canonical(self, fod): # ----- Vocabulary ----- - def test_status_vocabulary_is_the_owid_one_plus_two(self): + def test_status_vocabulary_is_the_owid_one_plus_three(self): # Every OWID status has a member of the same name and value, so an - # OWID failure is carried through unchanged, and the two 51Did + # OWID failure is carried through unchanged, and the three 51Did # payload statuses are the only additions. for status in ParseStatus: member = FodIdParseStatus.of(status) @@ -498,7 +557,12 @@ def test_status_vocabulary_is_the_owid_one_plus_two(self): owid_names = {status.name for status in ParseStatus} extra = {member.name for member in FodIdParseStatus} - owid_names self.assertEqual( - {"PAYLOAD_TOO_SHORT", "INVALID_TYPE_PAYLOAD_LENGTH"}, extra) + { + "PAYLOAD_TOO_SHORT", + "INVALID_TYPE_PAYLOAD_LENGTH", + "UNSUPPORTED_PAYLOAD_VERSION", + }, + extra) def test_result_is_immutable_and_carries_exactly_three_facts(self): result = FodId.try_from_base64( @@ -572,7 +636,7 @@ def test_success_does_not_verify_the_signature(self): fod = self.assert_parsed(FodId.try_from_byte_array(raw)) self.assertFalse(fod.verify(self.factory.public_pem)) - # ----- The two 51Did payload rules ----- + # ----- The three 51Did payload rules ----- def test_short_random_payload_reports_invalid_type_payload_length(self): payload = canonical_random_payload()[:RANDOM_PAYLOAD_LENGTH - 1] @@ -775,5 +839,324 @@ def test_raising_and_non_raising_readers_agree_on_success(self): self.assertEqual(raising.match_key, result.value.match_key) +class FodIdTermsTests(unittest.TestCase): + """The Terms byte, which says which terms document the identifier was + created under so that the terms travel with the identifier. The byte is + an index into a table in the specification and not a version number, + and it follows the match key, so the identifier type fixes where it + sits. The package turns the index into the address, so ``terms`` + answers with the address and a caller never handles the byte. + """ + + def setUp(self): + self.factory = FodIdTestFactory() + + def _read(self, payload): + return FodId.from_base64(self.factory.signed_owid_base64(payload)) + + # ----- A payload that ends at the match key ----- + + def test_payload_ending_at_the_match_key_has_no_address(self): + # There is no byte after the match key to read. A missing byte is + # index 0, which says the terms are not stated in the identifier, + # so absence and zero mean the same thing and no presence flag is + # needed to tell them apart. + for name, payload, length in ( + ("probabilistic", payload_ending_at_match_key(), + MATCH_KEY_LENGTH), + ("random", random_payload_ending_at_match_key(), + GUID_LENGTH)): + with self.subTest(name): + fod = self._read(payload) + self.assertIsNone(fod.terms) + self.assertEqual(length, len(fod.match_key)) + + def test_an_absent_byte_and_a_zero_byte_read_the_same(self): + absent = self._read(payload_ending_at_match_key()) + stated = self._read(with_terms(payload_ending_at_match_key(), + Terms.NOT_STATED.index)) + self.assertEqual(absent.terms, stated.terms) + self.assertIsNone(absent.terms) + self.assertIsNone(stated.terms) + + # ----- An index this package knows ----- + + def test_index_one_answers_with_the_model_terms_address(self): + for name, payload, length in ( + ("probabilistic", payload_ending_at_match_key(), + MATCH_KEY_LENGTH), + ("random", random_payload_ending_at_match_key(), + GUID_LENGTH)): + with self.subTest(name): + fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) + self.assertEqual(MODEL_TERMS_URL, fod.terms) + self.assertEqual(length, len(fod.match_key)) + + def test_the_address_is_the_versioned_document(self): + # The address names the exact document in force when the + # identifier was made, because a receiver has to be able to check + # years later what it agreed to, and an address whose contents can + # be edited cannot answer that. + fod = self._read(with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX)) + self.assertEqual("https://m4ow.uk/mtm/2.txt", fod.terms) + + # ----- An index this package does not know ----- + + def test_an_unknown_index_has_no_address(self): + # No address is ever built from an index this package cannot name, + # because that would name a document nobody wrote and a receiver + # would record having accepted terms that do not exist. + fod = self._read(with_terms(payload_ending_at_match_key(), + UNKNOWN_TERMS_INDEX)) + self.assertIsNone(fod.terms) + + def test_an_unknown_index_answers_as_no_terms_stated_does(self): + # A caller cannot tell the two apart, which is deliberate, since + # both say the identifier does not give the terms and the answer + # has to come from somewhere else. + unknown = self._read(with_terms(payload_ending_at_match_key(), + UNKNOWN_TERMS_INDEX)) + none = self._read(with_terms(payload_ending_at_match_key(), + Terms.NOT_STATED.index)) + self.assertIsNone(none.terms) + self.assertIsNone(unknown.terms) + + def test_every_index_the_package_does_not_know_has_no_address(self): + for index in (2, 3, 127, 128, 255): + with self.subTest(index=index): + fod = self._read( + with_terms(payload_ending_at_match_key(), index)) + self.assertIsNone(fod.terms) + self.assertIs(Terms.UNKNOWN, Terms.from_index(index)) + + # ----- The byte is read at the right offset ----- + + def test_terms_is_read_before_a_creator_context_section(self): + # The bytes after the Terms are a creator context section whose + # lengths belong to the cloud, so the byte has to be read at the + # offset the match key ends at and not at the end of the payload. + for name, payload, key in ( + ("probabilistic", payload_ending_at_match_key(), + CANONICAL_MATCH_KEY), + ("random", random_payload_ending_at_match_key(), + bytes((0x40 + i) for i in range(GUID_LENGTH)))): + with self.subTest(name): + built = (with_terms(payload, MODEL_TERMS_INDEX) + + CONTEXT_SECTION) + fod = self._read(built) + self.assertEqual(key, fod.match_key) + self.assertEqual(MODEL_TERMS_URL, fod.terms) + + def test_a_context_section_alone_does_not_state_terms(self): + # The first byte after the match key is the Terms and not the + # start of the context section, so a section opening with a zero + # reads as terms that are not stated. + built = (with_terms(payload_ending_at_match_key(), + Terms.NOT_STATED.index) + + CONTEXT_SECTION) + fod = self._read(built) + self.assertIsNone(fod.terms) + self.assertEqual(CANONICAL_MATCH_KEY, fod.match_key) + + def test_reserved_type_takes_every_byte_as_its_match_key(self): + # A Reserved type has no defined match key length, so its + # documented best-effort reading takes every byte after the header + # and leaves none to read as the Terms. It therefore states no + # terms. + payload = payload_ending_at_match_key() + payload[FLAGS_OFFSET] = 0b1100_0101 + fod = self._read(with_terms(payload, MODEL_TERMS_INDEX)) + self.assertIs(IdType.RESERVED, fod.type) + self.assertIsNone(fod.terms) + + # ----- The same answer from every reader ----- + + def test_every_reader_reads_the_same_terms(self): + payload = with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX) + base64 = self.factory.signed_owid_base64(payload) + owid = self.factory.signed_owid(payload) + readers = ( + FodId.from_base64(base64), + FodId.from_byte_array(self.factory.signed_bytes(payload)), + FodId.from_owid(owid), + FodId(owid), + FodId.try_from_base64(base64).value, + FodId.try_from_byte_array( + self.factory.signed_bytes(payload)).value, + ) + for fod in readers: + self.assertEqual(MODEL_TERMS_URL, fod.terms) + + def test_base64_roundtrip_preserves_the_terms(self): + first = self._read(with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX)) + for base64 in (first.as_base64(), first.as_base64_url()): + fod = FodId.from_base64(base64) + self.assertEqual(first.terms, fod.terms) + self.assertEqual(MODEL_TERMS_URL, fod.terms) + + def test_terms_does_not_change_how_the_other_fields_read(self): + # The byte must not move any field before it, so an identifier + # reads the same with the byte and without. + without = self._read(payload_ending_at_match_key()) + stated = self._read(with_terms(payload_ending_at_match_key(), + MODEL_TERMS_INDEX)) + self.assertEqual(without.type, stated.type) + self.assertEqual(without.usage, stated.usage) + self.assertEqual(without.usage_from_consent, + stated.usage_from_consent) + self.assertEqual(without.license_id, stated.license_id) + self.assertEqual(without.match_key, stated.match_key) + + def test_the_payload_rules_are_unchanged_by_the_terms(self): + # A payload one byte short of its match key still fails, and a + # Terms byte written onto it only makes up the match key, because + # the type says how many bytes the match key takes and only then + # does the Terms begin. + short = random_payload_ending_at_match_key()[ + :RANDOM_PAYLOAD_LENGTH - 1] + result = FodId.try_from_base64( + self.factory.signed_owid_base64(short)) + self.assertFalse(result.ok) + self.assertIs(FodIdParseStatus.INVALID_TYPE_PAYLOAD_LENGTH, + result.status) + fod = self._read(with_terms(short, MODEL_TERMS_INDEX)) + self.assertEqual(GUID_LENGTH, len(fod.match_key)) + self.assertIsNone(fod.terms) + + +class FodIdVersionTests(unittest.TestCase): + """Bits 4 and 5 of the flags byte, being the payload layout version. + This package reads version 0 and refuses every other version rather + than reading fields that may have moved. + """ + + def setUp(self): + self.factory = FodIdTestFactory() + + def _read(self, payload): + return FodId.try_from_base64( + self.factory.signed_owid_base64(payload)) + + def test_version_zero_reads_every_field(self): + result = self._read(canonical_payload()) + self.assertTrue(result.ok) + fod = result.value + self.assertIs(IdType.HASHED_EMAIL, fod.type) + self.assertIs(Usage.PERSONALIZED, fod.usage) + self.assertEqual(CANONICAL_LICENSE_ID, fod.license_id) + self.assertEqual(CANONICAL_MATCH_KEY, fod.match_key) + self.assertEqual(MODEL_TERMS_URL, fod.terms) + + def test_the_layout_names_the_version_this_package_reads(self): + self.assertEqual(0, SUPPORTED_PAYLOAD_VERSION) + + def test_an_unassigned_version_is_refused(self): + for version in (1, 2, 3): + with self.subTest(version=version): + result = self._read( + with_payload_version(canonical_payload(), version)) + self.assertFalse(result.ok) + self.assertIs( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + result.status) + # Nothing is handed back, rather than a value with some + # fields filled in, because there is no identifier to + # expose fields for when the layout was not understood. + self.assertIsNone(result.value) + + def test_the_raising_readers_name_the_version(self): + for version in (1, 2, 3): + with self.subTest(version=version): + base64 = self.factory.signed_owid_base64( + with_payload_version(canonical_payload(), version)) + with self.assertRaises(ValueError) as caught: + FodId.from_base64(base64) + self.assertIn("version {0}".format(version), + str(caught.exception)) + + def test_the_version_is_read_apart_from_the_usage_and_type_bits(self): + # A reader masking the wrong bits would refuse a version 0 + # identifier or let a later version through, so every combination + # is tried. + for usage in (0b000, 0b001, 0b011, 0b111): + for id_type in (0b00, 0b10, 0b11): + flags = (id_type << 6) | usage + payload = payload_ending_at_match_key() + payload[FLAGS_OFFSET] = flags + with self.subTest(flags=flags): + self.assertTrue(self._read(payload).ok) + for version in (1, 2, 3): + refused = self._read( + with_payload_version(payload, version)) + self.assertIs( + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + refused.status) + self.assertIsNone(refused.value) + + +class TermsTests(unittest.TestCase): + """The Terms vocabulary on its own, without an identifier around it. + It is private to the package, so these tests reach it directly.""" + + def test_the_layout_gives_the_terms_one_byte(self): + self.assertEqual(1, TERMS_LENGTH) + + def test_from_index_names_the_indexes_the_package_knows(self): + self.assertIs(Terms.NOT_STATED, Terms.from_index(0)) + self.assertIs(Terms.MODEL_TERMS_FOR_MARKETING_2, + Terms.from_index(1)) + + def test_from_index_names_every_other_index_unknown(self): + for index in range(2, 256): + self.assertIs(Terms.UNKNOWN, Terms.from_index(index)) + + def test_only_a_named_document_has_an_address(self): + self.assertIsNone(Terms.NOT_STATED.url) + self.assertIsNone(Terms.UNKNOWN.url) + self.assertEqual("https://m4ow.uk/mtm/2.txt", + Terms.MODEL_TERMS_FOR_MARKETING_2.url) + + def test_every_member_agrees_with_the_table(self): + # Each member carries its own index and address, and the index to + # member map is built from the members, so this fails if a member + # does not read back from its own index or if one that names a + # document was added without an address. + for member in Terms: + if member is Terms.UNKNOWN: + # Stands for every index the table does not carry, so it + # has no index of its own and no address. + self.assertEqual(-1, member.index) + self.assertIsNone(member.url) + continue + self.assertIs( + member, Terms.from_index(member.index), + "{0} does not read back from its own index".format(member)) + if member is Terms.NOT_STATED: + # Names no document, so it has no address. + self.assertEqual(0, member.index) + self.assertIsNone(member.url) + else: + self.assertIsNotNone( + member.url, + "{0} names a document with no address".format(member)) + self.assertTrue(member.url.startswith("https://")) + + def test_no_two_members_share_an_index(self): + # One index stands for one document, so which document an + # identifier was created under never depends on the order the + # members happen to be written in. + indexes = [m.index for m in Terms if m.index >= 0] + self.assertEqual(len(indexes), len(set(indexes))) + + def test_the_named_value_is_not_part_of_the_package_surface(self): + # The address on FodId is the whole of what a caller reads, so the + # named value is not exported from the package. + import fiftyone_pipeline_did as package + self.assertFalse(hasattr(package, "Terms")) + + if __name__ == "__main__": unittest.main()