diff --git a/fodid/README.md b/fodid/README.md index 074285e..189a548 100644 --- a/fodid/README.md +++ b/fodid/README.md @@ -49,6 +49,47 @@ which determines the length and meaning of the match key: - `IdType::Random` carries a 16-byte server-generated GUID. - `IdType::Reserved` is not yet assigned and is parsed best effort. +## The terms the identifier was created under + +The byte after the match key says which terms document the 51Did was created +under, so that the terms travel with the identifier rather than beside it. It +is an index into a table published in the +[specification](https://github.com/51Degrees/specifications/blob/main/did-specification/identifier-layout.md) +and is not a version number. `FodId::terms` answers with the address of the +document, so a caller never handles the byte. + +| Index | Document | `FodId::terms` | +| --- | --- | --- | +| `0` | Not stated in the identifier | `None` | +| `1` | Model Terms for Marketing, version 2 | `Some("https://m4ow.uk/mtm/2.txt")` | +| any other | One this crate cannot name | `None` | + +An identifier whose payload ends at the match key carries no terms byte, and +a missing byte is index 0, which answers with no address. An index added to +the specification after this release answers with no address as well, and no +address is ever built from an index this crate cannot name, since that would +name a document nobody wrote. A caller therefore cannot tell an index of +zero from an index this crate cannot name, which is deliberate, because both +lead to the same place. This crate answers with the address and never +fetches it. + +## The payload version + +Bits 4 and 5 of the flags byte say which payload layout the identifier +follows, and this crate reads version 0. A payload naming version 1, 2 or 3 +is refused with `Error::UnsupportedPayloadVersion`, which names the version +it found. + +No field is read under the layout this crate knows once the version says +otherwise. A later version exists precisely because a field moved, so +reading such a payload here would answer with values that are wrong rather +than absent, which is worse than refusing. A version that nothing checks +protects nothing. + +The version is not exposed. Either this crate read the layout, in which case +the accessors are the answer, or it did not, in which case there is no +identifier to read fields from. + ## Payload layout The payload is a five byte header, being a flags byte and a four byte little @@ -62,10 +103,17 @@ and the accessors every 51Did package offers at [package-surface.md](https://github.com/51Degrees/specifications/blob/main/did-specification/package-surface.md), and those two pages are the authority rather than any summary here. +A terms byte follows the match key, read through `FodId::terms`, which +answers with the address of the document the identifier was created under. +Where it sits depends on the match key length the type requires, which is +one reason the offsets stay internal. + The lengths given there are lower bounds. The payload must hold the header before the type can be read, and then the value the type requires, being 16 GUID bytes for a random identifier and 32 hash bytes for a probabilistic or -hashed email one. A payload may carry more bytes after the value, which this +hashed email one. The terms byte follows the value, so where it sits depends +on the value length the type requires, and a payload that ends at the value +carries none. A payload may carry more bytes after the terms, which this crate accepts and leaves in place. There is no upper bound on a 51Did in this crate, so a reader built today keeps reading identifiers issued in a newer, longer shape. @@ -95,6 +143,10 @@ fn read(base64_from_cloud_service: &str, public_pem: &str) -> Result<(), fodid:: let license_id = fod_id.license_id(); // u32 let match_key = fod_id.match_key(); // the match key bytes (SHA-256 or GUID) + // The terms the identifier was created under, and the address of that + // document where this crate knows the index. + let terms = fod_id.terms(); // Option<&'static str> + // Inherited OWID level fields and operations, available through Deref. let domain = fod_id.domain(); let round_trip = fod_id.as_base64()?; @@ -105,6 +157,7 @@ fn read(base64_from_cloud_service: &str, public_pem: &str) -> Result<(), fodid:: let _ = (usage, from_consent, id_type, license_id, match_key); let _ = (domain, round_trip, genuine); + let _ = terms; Ok(()) } ``` diff --git a/fodid/src/error.rs b/fodid/src/error.rs index e92464d..f69b67c 100644 --- a/fodid/src/error.rs +++ b/fodid/src/error.rs @@ -36,8 +36,9 @@ pub type Result = std::result::Result; /// outcome. Each one is a named status a caller can branch on directly, /// without matching on message text, and together they are the 51Did status /// vocabulary, being the OWID one (carried unchanged inside -/// [`Error::Parse`]) plus the two 51Did statuses [`Error::PayloadTooShort`] -/// and [`Error::InvalidTypePayloadLength`]. +/// [`Error::Parse`]) plus the three 51Did statuses +/// [`Error::PayloadTooShort`], [`Error::InvalidTypePayloadLength`] and +/// [`Error::UnsupportedPayloadVersion`]. /// /// A successful read says nothing about the signature. Whether the bytes /// are a 51Did and whether the signature is genuine are two questions with @@ -81,6 +82,18 @@ pub enum Error { /// The number of payload bytes actually present. actual: usize, }, + /// Bits 4 and 5 of the flags byte name a payload layout version this + /// crate does not know, so no field is read. + /// + /// A later version exists precisely because a field moved, so reading + /// the payload under the layout this crate knows would answer with + /// values that are wrong rather than absent, which is worse than + /// refusing. + UnsupportedPayloadVersion { + /// The version the payload named, being 1, 2 or 3, since 0 is the + /// layout this crate reads. + version: u8, + }, /// An OWID operation other than a read failed, for example serialising /// the envelope again or verifying its signature. Wraps the error type of /// the OWID library compiled into this crate, re-exported as @@ -108,6 +121,11 @@ impl fmt::Display for Error { "InvalidTypePayloadLength: a {id_type:?} 51Did needs at least \ {expected} payload bytes and {actual} are present" ), + Error::UnsupportedPayloadVersion { version } => write!( + f, + "UnsupportedPayloadVersion: 51Did payload version {version} \ + is not one this crate can read" + ), Error::Owid(e) => write!(f, "OWID operation failed because {e}"), } } @@ -118,7 +136,9 @@ impl std::error::Error for Error { match self { Error::Parse(e) => Some(e), Error::Owid(e) => Some(e), - Error::PayloadTooShort { .. } | Error::InvalidTypePayloadLength { .. } => None, + Error::PayloadTooShort { .. } + | Error::InvalidTypePayloadLength { .. } + | Error::UnsupportedPayloadVersion { .. } => None, } } } diff --git a/fodid/src/fodid.rs b/fodid/src/fodid.rs index 352e0ec..6d474c0 100644 --- a/fodid/src/fodid.rs +++ b/fodid/src/fodid.rs @@ -73,6 +73,17 @@ pub(crate) const HEADER_LENGTH: usize = MATCH_KEY_OFFSET; /// identifiers. pub(crate) const GUID_LENGTH: usize = 16; +/// The payload layout version this crate reads, carried in bits 4 and 5 of +/// the flags byte. Any other version is refused with +/// [`Error::UnsupportedPayloadVersion`] rather than read under this layout. +pub(crate) const SUPPORTED_PAYLOAD_VERSION: u8 = 0; + +/// The Terms index that says the terms are not stated in the identifier. A +/// payload ending at the match key reads as this, so absence and a zero +/// byte mean the same thing and nothing has to tell them apart. It is not a +/// row in [`TERMS_TABLE`] because it names no document. +const NOT_STATED_INDEX: u8 = 0; + /// The identifier type carried in bits 6-7 of the 51Did flags byte. /// /// Existing identifiers were issued with those bits zeroed, so they decode as @@ -165,6 +176,116 @@ impl IdType { } } +/// The terms document a 51Did was created under, carried in the byte that +/// follows the match key, so that the terms travel with the identifier +/// rather than beside it. +/// +/// The byte is an index into the table below and is not a version number. +/// An index is used so that a later document can live at any address, rather +/// than only at an address the specification could compose from a number. +/// +/// | Index | Document | Address | +/// |------:|--------------------------------------|-----------------------------| +/// | 0 | Not stated in the identifier | None | +/// | 1 | Model Terms for Marketing, version 2 | `https://m4ow.uk/mtm/2.txt` | +/// +/// A new terms document is a new index in that table, and every package has +/// to be released to know it, which is the cost of a receiver being able to +/// trust what it reads. An index is never reused or repointed once +/// published, because repointing one would rewrite what an identifier +/// already issued says it agreed to. +/// +/// An identifier whose payload ends at the match key carries no terms byte, +/// and a missing byte is read as index 0, so absence and zero say the same +/// thing and neither has to be told apart from the other. +/// +/// The table is published at +/// , +/// which is the authority rather than this summary. +/// +/// This enumeration is not public, and neither is the index behind it. The +/// crate turns the index into the address that [`FodId::terms`] answers +/// with, so a caller never handles the byte, and the names here are the ones +/// the specification gives so that every package describes one document the +/// same way. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum Terms { + /// Index 0. The terms are not stated in the identifier, which is also + /// how an identifier whose payload ends at the match key reads. + /// + /// This does not mean the identifier is unrestricted. It means the + /// identifier does not carry the answer, so the answer has to come from + /// what accompanies it, being the Terms Document Locator in an OpenRTB + /// request or whatever the surrounding protocol provides. The usage + /// says where an identifier may go and the terms say which document it + /// was created under, and a receiver needs both. + NotStated, + /// Index 1. The Model Terms for Marketing, version 2, whose address is + /// answered by [`FodId::terms`]. + ModelTermsForMarketing2, + /// An index added to the specification after this release, which this + /// crate cannot name. + /// + /// It answers with no address, as [`Terms::NotStated`] does, because + /// no address may be built from an index this crate cannot name, since + /// that would name a document nobody wrote. + Unknown, +} + +/// The terms table from the specification, which is the whole of the +/// definition of which index is which document. It is published at +/// +/// and this is the only place in the shipped code that carries it. The +/// tests write the address out again on purpose, so that a test never +/// compares the reader with itself. +/// +/// One row per terms document, holding the index the payload carries, the +/// name for it and the address it stands for. A new terms document is one +/// new row here and one new variant of [`Terms`], and nothing else in the +/// crate changes. That is the point of the byte being an index rather than +/// a version number, so the cost of a new document is a row and not a +/// search for every place a number was written down. +/// +/// [`Terms::NotStated`] and [`Terms::Unknown`] are deliberately absent. +/// Neither names a document, so neither has an address, and a lookup that +/// finds no row is the answer for both. +/// +/// Each address names an exact version rather than a landing page, because +/// a document at an unversioned address can be edited afterwards and a +/// receiver has to know the document that was in force when the identifier +/// was made. +const TERMS_TABLE: &[(u8, Terms, &str)] = &[( + 1, + Terms::ModelTermsForMarketing2, + "https://m4ow.uk/mtm/2.txt", +)]; + +impl Terms { + /// Decode the terms from the index byte that follows the match key. An + /// index this crate does not know decodes as [`Terms::Unknown`] and + /// never as [`Terms::NotStated`]. + fn from_index(index: u8) -> Terms { + if index == NOT_STATED_INDEX { + return Terms::NotStated; + } + TERMS_TABLE + .iter() + .find(|(row_index, _, _)| *row_index == index) + .map_or(Terms::Unknown, |(_, terms, _)| *terms) + } + + /// The address of the terms document, or `None` where there is none to + /// give, being index 0 and an index this crate does not know. The + /// address is answered and never fetched, and the caller decides what + /// to do with it. + fn url(self) -> Option<&'static str> { + TERMS_TABLE + .iter() + .find(|(_, terms, _)| *terms == self) + .map(|(_, _, url)| *url) + } +} + /// A parsed 51Did: an [`Owid`] envelope whose payload encodes the fields of a /// 51Degrees identifier. /// @@ -184,6 +305,20 @@ impl IdType { /// , /// and those two pages are the authority rather than any summary here. /// +/// The terms byte follows the match key, so where it sits depends on the +/// match key length the type requires. It is read through +/// [`terms`](FodId::terms), which answers with the address of the document, +/// and a payload that ends at the match key carries no terms byte and +/// answers with no address. +/// +/// Bits 4 and 5 of the flags byte say which payload layout the identifier +/// follows, and this crate reads version 0. A payload naming any other +/// version is refused with [`Error::UnsupportedPayloadVersion`] rather than +/// read under the layout this crate knows, because a later version exists +/// precisely because a field moved, so reading one here would answer with +/// values that are wrong rather than absent. The version is not exposed, +/// because a caller has nothing to decide with it. +/// /// Those lengths are minimums. A payload may carry more bytes after the match /// key, which this reader accepts and leaves in place, reachable through /// [`payload`](Owid::payload). There is no upper bound in this crate. @@ -203,6 +338,7 @@ pub struct FodId { flags: u8, license_id: u32, match_key: Vec, + terms_index: u8, } impl FodId { @@ -250,8 +386,10 @@ impl FodId { /// [`IdType::Random`] and a 32 byte hash for [`IdType::Probabilistic`] /// and [`IdType::HashedEmail`]. A /// [`IdType::Reserved`] payload has no defined value length and is read - /// best effort. Bytes after the value are accepted and left in the - /// payload, because a longer payload is a newer shape rather than a fault. + /// best effort, taking every byte after the header as its value, so a + /// reserved identifier states no terms until that length is assigned. + /// Bytes after the value are accepted and left in the payload, because a + /// longer payload is a newer shape rather than a fault. /// /// # Errors /// @@ -267,6 +405,18 @@ impl FodId { }); } let flags = payload[FLAGS_OFFSET]; + // The version is read before any field, because a later version + // exists precisely because a field moved. Reading a payload of a + // version this crate does not know under the layout it does know + // would answer with values that are wrong rather than absent, which + // is worse than refusing, and a version that nothing checks + // protects nothing. + let payload_version = (flags >> 4) & 0b11; + if payload_version != SUPPORTED_PAYLOAD_VERSION { + return Err(Error::UnsupportedPayloadVersion { + version: payload_version, + }); + } let license_id = u32::from_le_bytes( payload[LICENSE_ID_OFFSET..LICENSE_ID_OFFSET + LICENSE_ID_LENGTH] .try_into() @@ -288,11 +438,26 @@ impl FodId { }); } let match_key = payload[MATCH_KEY_OFFSET..MATCH_KEY_OFFSET + value_length].to_vec(); + // The terms index is the byte after the match key, so where it sits + // follows the match key length the type selects. A missing byte is + // index 0, which says the terms are not stated in the identifier, so + // absence and zero are one answer. + // + // A reserved type has no assigned match key length, so its value is + // every byte after the header and there is no byte left for the + // terms to be taken from. Such an identifier reads as index 0, which + // is correct rather than a fault, and it stops being a special case + // as soon as a reserved type is assigned a length. + let terms_index = payload + .get(MATCH_KEY_OFFSET + value_length) + .copied() + .unwrap_or(NOT_STATED_INDEX); Ok(FodId { owid, flags, license_id, match_key, + terms_index, }) } @@ -332,6 +497,30 @@ impl FodId { &self.match_key } + /// The address of the terms document the identifier was created under, + /// read from the byte after the match key. + /// + /// The byte is an index into a table in the specification and this + /// crate turns the index into the address, so a caller never handles + /// the byte. The address is answered and never fetched, and it is never + /// an empty string and never built from the index, so `Some` means this + /// crate knows the document and the caller can rely on the address it + /// holds. + /// + /// `None` covers both an index of zero, which says the terms are not + /// stated in the identifier, and an index added to the table after this + /// release, which this crate cannot name. A caller cannot tell those + /// two apart, which is deliberate, because both lead to the same place, + /// being that the identifier does not say which terms it was created + /// under and the answer has to come from somewhere else. + /// + /// No address does not mean the identifier is unrestricted. Where an + /// identifier may go is a separate question [`usage`](FodId::usage) + /// answers, which still bars a non-marketing identifier from a demand + /// source. + pub fn terms(&self) -> Option<&'static str> { + Terms::from_index(self.terms_index).url() + } /// A reference to the underlying OWID envelope. pub fn owid(&self) -> &Owid { &self.owid @@ -374,6 +563,66 @@ impl FromStr for FodId { FodId::from_base64(s) } } +#[cfg(test)] +mod terms_table_tests { + use super::{Terms, NOT_STATED_INDEX, TERMS_TABLE}; + + /// Every row reads back through the pair of lookups it feeds, so a row + /// whose index and variant were mistyped apart is caught here rather + /// than by a caller reading no address for a document this crate is + /// meant to know. + #[test] + fn every_row_round_trips_through_the_lookups() { + for (index, terms, url) in TERMS_TABLE { + assert_eq!(Terms::from_index(*index), *terms, "index {index}"); + assert_eq!(terms.url(), Some(*url), "{terms:?}"); + assert!(url.starts_with("https://"), "{url} is not https"); + } + } + + /// No row may claim index 0. That index says the terms are not stated, + /// which names no document, so a row there would give an address to an + /// identifier that states none. + #[test] + fn no_row_claims_the_not_stated_index() { + assert!(TERMS_TABLE + .iter() + .all(|(index, _, _)| *index != NOT_STATED_INDEX)); + assert_eq!(Terms::from_index(NOT_STATED_INDEX), Terms::NotStated); + assert_eq!(Terms::NotStated.url(), None); + } + + /// One index may stand for one document only. Two rows sharing an index + /// would make which document an identifier was created under depend on + /// the order the table happens to be written in. + #[test] + fn no_index_appears_twice() { + for (position, (index, _, _)) in TERMS_TABLE.iter().enumerate() { + assert!( + !TERMS_TABLE[position + 1..] + .iter() + .any(|(later, _, _)| later == index), + "index {index} appears more than once" + ); + } + } + + /// An index the table does not carry is Unknown and never NotStated, + /// and it answers with no address rather than one built from the + /// number, since that would name a document nobody wrote. + #[test] + fn an_index_outside_the_table_is_unknown_with_no_address() { + for index in 0..=u8::MAX { + let known = + index == NOT_STATED_INDEX || TERMS_TABLE.iter().any(|(row, _, _)| *row == index); + if known { + continue; + } + assert_eq!(Terms::from_index(index), Terms::Unknown, "index {index}"); + assert_eq!(Terms::from_index(index).url(), None, "index {index}"); + } + } +} /// The layout constants are internal, so a consumer cannot read them and the /// tests that build payloads byte by byte carry their own copy of the layout diff --git a/fodid/src/lib.rs b/fodid/src/lib.rs index 76231a3..4f07e1c 100644 --- a/fodid/src/lib.rs +++ b/fodid/src/lib.rs @@ -58,6 +58,35 @@ //! - [`IdType::Random`] carries a 16-byte server-generated GUID. //! - [`IdType::Reserved`] is not yet assigned and is parsed best effort. //! +//! ## The terms the identifier was created under +//! +//! The byte after the match key says which terms document the 51Did was +//! created under, so that the terms travel with the identifier rather than +//! beside it. It is an index into a table published in the specification and +//! is not a version number, and [`FodId::terms`] answers with the address of +//! the document, so a caller never handles the byte. +//! +//! An identifier whose payload ends at the match key carries no terms byte, +//! and a missing byte is index 0, which answers with no address. An index +//! added to the specification after this release answers with no address as +//! well, and no address is ever built from an index this crate cannot name, +//! since that would name a document nobody wrote. A caller therefore cannot +//! tell an index of zero from an index this crate cannot name, which is +//! deliberate, because both lead to the same place. This crate answers with +//! the address and never fetches it. +//! +//! ## The payload version +//! +//! Bits 4 and 5 of the flags byte say which payload layout the identifier +//! follows, and this crate reads version 0. A payload naming version 1, 2 or +//! 3 is refused with [`Error::UnsupportedPayloadVersion`], which names the +//! version it found. No field is read under the layout this crate knows once +//! the version says otherwise, because a later version exists precisely +//! because a field moved, so reading such a payload here would answer with +//! values that are wrong rather than absent. The version is not exposed, +//! because either this crate read the layout or there is no identifier to +//! read fields from. +//! //! ## Payload layout //! //! The payload is a five byte header, being a flags byte and a four byte @@ -70,13 +99,20 @@ //! , //! and those two pages are the authority rather than any summary here. //! +//! A terms byte follows the match key, read through [`FodId::terms`], +//! which answers with the address of the document the identifier was +//! created under. Where it sits depends on the match key length the type +//! requires, which is a reason the offsets stay internal. +//! //! The lengths given there are lower bounds. The payload must hold the header //! before the type can be read, and then the value the type requires, being //! 16 GUID bytes for a random identifier and 32 hash bytes for a -//! probabilistic or hashed email one. A payload may carry more bytes after -//! the value, and this crate accepts them and leaves them in place. There -//! is no upper bound on a 51Did in this crate, so a reader built today keeps -//! reading identifiers issued in a newer, longer shape. +//! probabilistic or hashed email one. The terms byte follows the value, so +//! where it sits depends on the value length the type requires, and a +//! payload that ends at the value carries none. A payload may carry more +//! bytes after the terms, and this crate accepts them and leaves them in +//! place. There is no upper bound on a 51Did in this crate, so a reader +//! built today keeps reading identifiers issued in a newer, longer shape. //! //! [`FodId`] [`Deref`](std::ops::Deref)s to the underlying [`Owid`], so //! a `FodId` can be used directly for all OWID level concerns (domain, date, @@ -146,6 +182,10 @@ //! let license_id: u32 = fod_id.license_id(); //! let match_key: &[u8] = fod_id.match_key(); // the match key to compare (32 or 16 bytes) //! +//! // The address of the terms document the identifier was created under, +//! // and None where it names none this crate knows. +//! let terms: Option<&str> = fod_id.terms(); +//! //! // Inherited OWID level fields and operations, available through Deref. //! let domain = fod_id.domain(); //! let round_trip = fod_id.as_base64()?; @@ -155,6 +195,7 @@ //! let genuine = status == SignatureStatus::Valid; //! # let _ = (usage, from_consent, id_type, license_id, match_key); //! # let _ = (domain, round_trip, genuine); +//! # let _ = terms; //! # Ok(()) //! # } //! ``` diff --git a/fodid/tests/cloud_51did.rs b/fodid/tests/cloud_51did.rs index f500c9c..c83215e 100644 --- a/fodid/tests/cloud_51did.rs +++ b/fodid/tests/cloud_51did.rs @@ -59,7 +59,7 @@ //! when it does not, so this test starts covering them automatically once a //! paid key is expanded for marketing. -use fodid::FodId; +use fodid::{FodId, IdType, Usage}; mod layout; @@ -84,30 +84,63 @@ const USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) /// reserved for documentation (RFC 5737). const CLIENT_IP: &str = "203.0.113.42"; -/// A cloud `id.usage` level and whether every resource key must return a 51Did -/// for it. -struct Usage { +/// A cloud `id.usage` level, whether every resource key must return a 51Did +/// for it, and what the identifier must then say about itself. +struct UsageCase { /// The `id.usage` request value. name: &'static str, /// Whether a 51Did is required for this usage. Only `non-marketing` is /// required today; the marketing usages are validated when returned. required: bool, + /// The usage the reader must answer with. + expected: Usage, + /// The terms address the identifier must carry, or `None` where it must + /// state none. A non-marketing identifier may not reach a demand source + /// at all, so there is nothing for a receiver to agree to. + terms: Option<&'static str>, } +/// The versioned Model Terms for Marketing document a marketing 51Did is +/// created under. +/// +/// Written out here rather than read from the crate, because a test that +/// asked the crate what it expects would agree with itself whatever the +/// crate said. The literal is what a receiver has to be able to fetch. +const MODEL_TERMS_FOR_MARKETING_2: &str = "https://m4ow.uk/mtm/2.txt"; + +/// IAB TCF v2 consent strings, and the usage the service must derive from +/// each without the caller stating one. +/// +/// The first sets all twelve purposes, which is personalized. The second +/// sets the Appendix 1 standard set, being purposes 1, 2, 7, 8 and 11, which +/// is standard. Both are the strings the cloud's own IabTcfElement tests use, +/// repeated here rather than shared, for the same reason as the address +/// above. +const CONSENT_STRINGS: &[(&str, Usage)] = &[ + ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage::Personalized), + ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage::Standard), +]; + /// The usage levels checked for the resource key. Ordered with the required /// `non-marketing` usage first. -const USAGES: &[Usage] = &[ - Usage { +const USAGES: &[UsageCase] = &[ + UsageCase { name: "non-marketing", required: true, + expected: Usage::NonMarketing, + terms: None, }, - Usage { + UsageCase { name: "standard", required: false, + expected: Usage::Standard, + terms: Some(MODEL_TERMS_FOR_MARKETING_2), }, - Usage { + UsageCase { name: "personalized", required: false, + expected: Usage::Personalized, + terms: Some(MODEL_TERMS_FOR_MARKETING_2), }, ]; @@ -125,21 +158,96 @@ fn resource_key() -> Option { /// Calls the cloud JSON endpoint for the given `id.usage` and returns the /// parsed response body. fn request_usage(resource_key: &str, usage: &str) -> serde_json::Value { + request_with(resource_key, "id.usage", usage) +} + +/// Calls the cloud JSON endpoint with one extra query parameter and returns +/// the parsed response body. +fn request_with(resource_key: &str, name: &str, value: &str) -> serde_json::Value { let body = ureq::get(CLOUD_JSON_URL) .query("resource", resource_key) .query("user-agent", USER_AGENT) .query("client-ip", CLIENT_IP) - .query("id.usage", usage) + .query(name, value) .call() - .unwrap_or_else(|e| panic!("cloud request for id.usage={usage} should succeed: {e}")) + .unwrap_or_else(|e| panic!("cloud request for {name}={value} should succeed: {e}")) .into_string() .expect("cloud response should be readable"); serde_json::from_str(&body).expect("cloud response should be JSON") } +/// A consent management platform sends an IAB TCF consent string and no usage +/// of its own. The service decodes the string, decides the usage from the +/// purposes it grants, and records in the identifier that it did so, which is +/// bit 3 of the flags byte. +/// +/// This is the half a caller cannot state for itself. An identifier whose +/// usage was stated in the request and one whose usage was decoded from a +/// consent string are both legitimate, and they are different assertions +/// about how the permission was obtained, so a receiver has to be able to +/// tell them apart. The service signs the answer, and this proves the two +/// ends agree about which bit it is and which way round it reads. +/// +/// Marked `#[ignore]` for the same reason as the test above. +#[test] +#[ignore = "live cloud test: set 51DEGREES_RESOURCE_KEY and run with `--include-ignored` (see module docs)"] +fn consent_string_sets_the_usage_from_consent_bit() { + let Some(resource_key) = resource_key() else { + panic!( + "no resource key found for the live cloud 51Did test. See the message on resource_key_returns_51did_for_supported_usages for how to set one." + ); + }; + + let mut proven = 0; + for (tc_string, expected) in CONSENT_STRINGS { + // No id.usage is sent. A stated usage wins over a consent string, so + // sending one would leave the bit clear and this would prove the + // opposite of what it says. + let response = request_with(&resource_key, "tcstring", tc_string); + + let Some(fodid) = response.get("fodid") else { + eprintln!( + "consent string granting {expected:?}: no 'fodid' element returned, so this key is not entitled to that marketing usage" + ); + continue; + }; + + for name in ["idprobglobal", "idproblic"] { + if let Some(value) = string_field(fodid, name) { + // A consent string granting a marketing usage produces a + // marketing identifier, so the terms travel with it too. + assert_valid_51did( + &format!("consent/{expected:?}/{name}"), + value, + Some(MODEL_TERMS_FOR_MARKETING_2), + *expected, + true, + ); + proven += 1; + } + } + } + + // Rust has no inconclusive result, so this says plainly what the run did + // rather than leaving a pass to be read as proof. + if proven == 0 { + eprintln!( + "NOTHING PROVEN: this resource key returned no identifier for either consent string, so the usage-from-consent bit was never read. Use a key entitled to the standard or personalized usage." + ); + } else { + eprintln!("Usage-from-consent read on {proven} identifier(s)."); + } +} + /// Asserts that `base64` is a real 51Did: a signed OWID envelope whose payload /// carries the three 51Did fields, including the 32-byte probabilistic hash. -fn assert_valid_51did(label: &str, base64: &str) { +fn assert_valid_51did( + label: &str, + base64: &str, + expected_terms: Option<&str>, + expected_usage: Usage, + from_consent: bool, +) { assert!(!base64.is_empty(), "{label} should not be empty"); let fod_id = FodId::from_base64(base64) @@ -173,6 +281,60 @@ fn assert_valid_51did(label: &str, base64: &str) { "{label}: hash should survive a base64 round trip" ); + // The terms travel with the identifier, so a receiver can read what it + // was created under without asking anyone. A payload that stops at the + // match key reads as no terms, which is why this is the assertion that + // fails where the service has not been updated to write the byte. + assert_eq!( + fod_id.terms(), + expected_terms, + "{label}: expected the terms to be {expected_terms:?} and the \ + identifier carries {:?}. Where this reads None for a marketing \ + usage the service that answered is older than the release that \ + writes the Terms byte.", + fod_id.terms() + ); + assert_eq!( + reparsed.terms(), + expected_terms, + "{label}: terms should survive a base64 round trip" + ); + + // The flags byte, read through the accessors rather than by masking. + // The usage values are cumulative, being 001, 011 and 111, so a caller + // masking the byte for the non-marketing bit reads every marketing + // identifier as non-marketing. These assertions are the alignment + // between what the service wrote and what this crate answers. + assert_eq!( + fod_id.usage(), + expected_usage, + "{label}: the service was asked for a {expected_usage:?} identifier \ + and this reads as {:?}", + fod_id.usage() + ); + assert_eq!( + fod_id.usage_from_consent(), + from_consent, + "{label}: expected the usage to be recorded as {}, and it reads as {}", + if from_consent { + "derived from a consent string" + } else { + "stated by the caller" + }, + if fod_id.usage_from_consent() { + "derived from a consent string" + } else { + "stated by the caller" + } + ); + assert_eq!( + fod_id.id_type(), + IdType::Probabilistic, + "{label}: an idprob* value must be a probabilistic identifier and \ + this reads as {:?}", + fod_id.id_type() + ); + let hash_hex: String = fod_id .match_key() .iter() @@ -252,9 +414,13 @@ fn resource_key_returns_51did_for_supported_usages() { // idprobglobal is the global 51Did for this usage. It is required for // non-marketing and validated when a marketing usage returns it. match string_field(fodid, "idprobglobal") { - Some(idprobglobal) => { - assert_valid_51did(&format!("{}/idprobglobal", usage.name), idprobglobal) - } + Some(idprobglobal) => assert_valid_51did( + &format!("{}/idprobglobal", usage.name), + idprobglobal, + usage.terms, + usage.expected, + false, + ), None if usage.required => { panic!( "id.usage={}: no idprobglobal returned. fodid element: {fodid}", @@ -271,7 +437,13 @@ fn resource_key_returns_51did_for_supported_usages() { // idproblic is scoped to the caller's license and is validated whenever // it is returned. if let Some(idproblic) = string_field(fodid, "idproblic") { - assert_valid_51did(&format!("{}/idproblic", usage.name), idproblic); + assert_valid_51did( + &format!("{}/idproblic", usage.name), + idproblic, + usage.terms, + usage.expected, + false, + ); } } } diff --git a/fodid/tests/fodid_tests.rs b/fodid/tests/fodid_tests.rs index 88739af..75612ba 100644 --- a/fodid/tests/fodid_tests.rs +++ b/fodid/tests/fodid_tests.rs @@ -34,7 +34,10 @@ mod layout; const TEST_DOMAIN: &str = "51degrees.com"; -const CANONICAL_FLAGS: u8 = 0b1010_0101; +/// Bits 4 and 5 are the payload version and are left clear, because this +/// crate reads version 0 and refuses any other, so a fixture with them +/// set would be refused rather than read. +const CANONICAL_FLAGS: u8 = 0b1000_0101; const CANONICAL_LICENSE_ID: u32 = 0x1234_5678; /// Flags bytes whose bits 6-7 select each identifier type. The lower usage @@ -54,9 +57,11 @@ fn canonical_hash() -> [u8; layout::MATCH_KEY_LENGTH] { hash } -/// A canonical 37-byte 51Did payload with flags = 0xA5, -/// licenseId = 0x12345678 (little endian) and the canonical hash. -fn canonical_payload() -> Vec { +/// A 51Did payload cut off at the end of the match key, so it carries no +/// terms byte. A reader takes a missing byte as a terms index of zero, +/// and this is the fixture for that rule rather than anything an issuer +/// would write. +fn payload_ending_at_match_key() -> Vec { let mut payload = vec![0u8; layout::PAYLOAD_LENGTH]; payload[layout::FLAGS_OFFSET] = CANONICAL_FLAGS; payload[layout::LICENSE_ID_OFFSET..layout::LICENSE_ID_OFFSET + layout::LICENSE_ID_LENGTH] @@ -282,19 +287,21 @@ fn a_flags_byte_of_zero_reads_as_no_usage_and_the_default_type() { } #[test] -fn a_flags_byte_with_every_bit_set_reads_as_the_highest_usage_and_reserved() { +fn a_flags_byte_with_every_other_bit_set_reads_as_the_highest_usage_and_reserved() { let fixture = Fixture::new(); let mut payload = canonical_payload(); - payload[layout::FLAGS_OFFSET] = 0xFF; + // Every bit except 4 and 5, which are the payload version. A byte of + // 0xFF names version 3, which this crate refuses rather than reads, + // and that refusal is covered by the payload version tests below. + payload[layout::FLAGS_OFFSET] = 0xCF; let fod_id = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap(); assert_eq!(Usage::Personalized, fod_id.usage()); assert!(fod_id.usage_from_consent()); assert_eq!(IdType::Reserved, fod_id.id_type()); - // The byte itself is still reachable through the envelope payload, and - // the unused bits 4 and 5 change none of the answers above. - assert_eq!(0xFF, fod_id.payload()[layout::FLAGS_OFFSET]); + // The byte itself is still reachable through the envelope payload. + assert_eq!(0xCF, fod_id.payload()[layout::FLAGS_OFFSET]); } #[test] @@ -873,3 +880,304 @@ fn reserved_type_exposes_remaining_payload_best_effort() { let fod_id = assert_parsed(&result); assert!(fod_id.match_key().is_empty()); } + +/// The canonical payload as an issuer writes one, carrying the payload +/// version 0 in its flags byte and the terms byte of the document a +/// personalized marketing identifier is created under. +fn canonical_payload() -> Vec { + payload_with_terms(MODEL_TERMS_INDEX) +} + +/// The payload with its version bits set to the given version, leaving every +/// other bit of the flags byte alone. +fn with_payload_version(payload: &[u8], version: u8) -> Vec { + let mut changed = payload.to_vec(); + changed[layout::FLAGS_OFFSET] = (payload[layout::FLAGS_OFFSET] & 0b1100_1111) | (version << 4); + changed +} + +/// The terms index the specification gives to the Model Terms for Marketing, +/// version 2, and the address that index stands for. Both are written out +/// here rather than taken from the crate, so the test checks the crate +/// against the specification rather than against itself. +const MODEL_TERMS_INDEX: u8 = 1; + +const MODEL_TERMS_URL: &str = "https://m4ow.uk/mtm/2.txt"; + +/// An index no release of this crate knows, standing in for one added to the +/// specification after this one. +const UNKNOWN_TERMS_INDEX: u8 = 200; + +/// The canonical payload with a terms index byte after the match key, which +/// is the shape an issuer writes. The byte is appended by hand, because the +/// offset it lands at is what these tests are checking. +fn payload_with_terms(index: u8) -> Vec { + let mut payload = payload_ending_at_match_key(); + payload.push(index); + payload +} + +#[test] +fn a_payload_ending_at_the_match_key_has_no_terms_address() { + // There is no byte after the match key to read. A missing byte is index + // 0, which says the terms are not stated in the identifier, so such an + // identifier answers with no address and every other field reads as it + // does with the byte present. + let fixture = Fixture::new(); + let result = FodId::from_base64(&fixture.signed_owid_base64(payload_ending_at_match_key())); + let fod_id = assert_parsed(&result); + + assert_eq!(fod_id.terms(), None); + + // Every other field reads as it does with the byte present. + assert_canonical_flags(fod_id); + assert_eq!(fod_id.license_id(), CANONICAL_LICENSE_ID); + assert_eq!(fod_id.match_key(), &canonical_hash()); +} + +#[test] +fn an_explicit_zero_reads_the_same_as_a_missing_terms_byte() { + // Absence and zero say the same thing, so nothing has to tell them + // apart and no presence flag is needed. + let fixture = Fixture::new(); + let absent = + FodId::from_base64(&fixture.signed_owid_base64(payload_ending_at_match_key())).unwrap(); + let zero = FodId::from_base64(&fixture.signed_owid_base64(payload_with_terms(0))).unwrap(); + + assert_eq!(absent.terms(), zero.terms()); + assert_eq!(zero.terms(), None); + + // The payloads still differ by the byte, which stays in place. + assert_eq!(absent.payload().len() + 1, zero.payload().len()); +} + +#[test] +fn index_one_is_the_model_terms_for_marketing_and_carries_its_address() { + let fixture = Fixture::new(); + let payload = payload_with_terms(MODEL_TERMS_INDEX); + let result = FodId::from_base64(&fixture.signed_owid_base64(payload)); + let fod_id = assert_parsed(&result); + + assert_eq!(fod_id.terms(), Some(MODEL_TERMS_URL)); + assert_eq!(fod_id.match_key(), &canonical_hash()); +} + +#[test] +fn an_index_this_crate_does_not_know_has_no_address() { + // No address is ever built from an index this crate cannot name, + // because that would name a document nobody wrote and a receiver would + // record having accepted terms that do not exist. + let fixture = Fixture::new(); + for index in [2u8, 127, UNKNOWN_TERMS_INDEX, 255] { + let payload = payload_with_terms(index); + let result = FodId::from_base64(&fixture.signed_owid_base64(payload)); + let fod_id = assert_parsed(&result); + + assert_eq!(fod_id.terms(), None, "index {index}"); + } +} + +#[test] +fn no_terms_stated_and_an_unknown_index_both_have_no_address() { + // A caller cannot tell the two apart, which is deliberate, since both + // say the identifier does not give the terms and the answer has to come + // from somewhere else. + let fixture = Fixture::new(); + let stated_none = + FodId::from_base64(&fixture.signed_owid_base64(payload_with_terms(0))).unwrap(); + let unknown = + FodId::from_base64(&fixture.signed_owid_base64(payload_with_terms(UNKNOWN_TERMS_INDEX))) + .unwrap(); + + assert_eq!(stated_none.terms(), None); + assert_eq!(unknown.terms(), None); +} + +#[test] +fn every_terms_index_decodes_as_the_specification_publishes_it() { + // The published table holds two indexes today. Everything else is an + // index added later, whatever its value, including the one immediately + // after the last published index and the largest a byte can hold. + let fixture = Fixture::new(); + let cases = [ + (0u8, None), + (1, Some(MODEL_TERMS_URL)), + (2, None), + (UNKNOWN_TERMS_INDEX, None), + (255, None), + ]; + for (index, expected_url) in cases { + let payload = payload_with_terms(index); + let fod_id = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap(); + + assert_eq!(fod_id.terms(), expected_url, "index {index}"); + } +} + +#[test] +fn the_terms_byte_is_read_after_the_match_key_for_both_match_key_lengths() { + // The terms byte follows the match key, so where it sits moves with the + // length the identifier type requires. Reading it at a fixed offset + // would take a hash byte for a random identifier. + let fixture = Fixture::new(); + let cases = [ + ( + PROBABILISTIC_FLAGS, + IdType::Probabilistic, + layout::MATCH_KEY_LENGTH, + ), + (RANDOM_FLAGS, IdType::Random, layout::GUID_LENGTH), + ( + HASHED_EMAIL_FLAGS, + IdType::HashedEmail, + layout::MATCH_KEY_LENGTH, + ), + ]; + for (flags, id_type, value_len) in cases { + let mut payload = typed_payload(flags, value_len); + payload.push(MODEL_TERMS_INDEX); + let fod_id = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap(); + + assert_eq!(fod_id.id_type(), id_type); + assert_eq!(fod_id.match_key().len(), value_len, "{id_type:?}"); + assert_eq!(fod_id.match_key()[0], 0x50, "{id_type:?}"); + assert_eq!( + fod_id.match_key()[value_len - 1], + 0x50 + (value_len as u8 - 1), + "{id_type:?}" + ); + assert_eq!(fod_id.terms(), Some(MODEL_TERMS_URL), "{id_type:?}"); + } +} + +#[test] +fn a_creator_context_after_the_terms_leaves_both_the_match_key_and_terms_read() { + // The terms sit between the match key and the creator context, so a + // payload carrying a context proves the byte is taken from the right + // offset rather than from the end of the payload. The context bytes are + // arbitrary here, because their meaning belongs to the issuer and this + // crate leaves them in the payload untouched. + let fixture = Fixture::new(); + for context_len in [1usize, 40, 300] { + let mut payload = payload_with_terms(MODEL_TERMS_INDEX); + payload.extend((0..context_len).map(|i| 0xC0 | (i as u8 & 0x0F))); + let expected_payload = payload.clone(); + + let result = FodId::from_base64(&fixture.signed_owid_base64(payload)); + let fod_id = assert_parsed(&result); + + assert_eq!(fod_id.match_key(), &canonical_hash(), "{context_len}"); + assert_eq!(fod_id.terms(), Some(MODEL_TERMS_URL), "{context_len}"); + assert_eq!(fod_id.license_id(), CANONICAL_LICENSE_ID, "{context_len}"); + assert_eq!( + fod_id.payload(), + expected_payload.as_slice(), + "{context_len}" + ); + } +} + +#[test] +fn a_reserved_identifier_states_no_terms() { + // A reserved type has no defined match key length, so every byte after + // the header is read as its value best effort and there is no byte the + // terms could be taken from. Index 0 is the answer, being the same one + // an identifier that ends at its match key gives. + let fixture = Fixture::new(); + let mut payload = typed_payload(RESERVED_FLAGS, 8); + payload.push(MODEL_TERMS_INDEX); + let fod_id = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap(); + + assert_eq!(fod_id.id_type(), IdType::Reserved); + assert_eq!(fod_id.terms(), None); +} + +#[test] +fn the_terms_survive_a_base64_round_trip() { + let fixture = Fixture::new(); + let payload = payload_with_terms(MODEL_TERMS_INDEX); + let fod_id = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap(); + + let round_tripped = FodId::from_base64(&fod_id.as_base64().unwrap()).unwrap(); + + assert_eq!(round_tripped.terms(), fod_id.terms()); + assert_eq!(round_tripped.terms(), Some(MODEL_TERMS_URL)); +} + +#[test] +fn payload_version_zero_reads_every_field() { + // Bits 4 and 5 clear is version 0, which is the layout this crate + // reads, so every field reads as it does on the canonical payload. + let fixture = Fixture::new(); + let fod_id = FodId::from_base64(&fixture.signed_owid_base64(canonical_payload())).unwrap(); + + assert_canonical_flags(&fod_id); + assert_eq!(fod_id.license_id(), CANONICAL_LICENSE_ID); + assert_eq!(fod_id.match_key(), &canonical_hash()); + assert_eq!(fod_id.terms(), Some(MODEL_TERMS_URL)); +} + +#[test] +fn an_unassigned_payload_version_is_refused_and_names_the_version() { + // Versions 1, 2 and 3 are not assigned, so a payload naming one is + // refused rather than read under the layout this crate knows, and + // nothing is handed back because there is no identifier to expose + // fields for when the layout was not understood. + let fixture = Fixture::new(); + for version in [1u8, 2, 3] { + let payload = with_payload_version(&canonical_payload(), version); + let result = FodId::from_base64(&fixture.signed_owid_base64(payload)); + + match result { + Err(Error::UnsupportedPayloadVersion { version: found }) => { + assert_eq!(found, version); + } + other => panic!("version {version} was not refused: {other:?}"), + } + } +} + +#[test] +fn a_refused_payload_version_names_itself_in_the_message() { + let fixture = Fixture::new(); + for version in [1u8, 2, 3] { + let payload = with_payload_version(&canonical_payload(), version); + let error = FodId::from_base64(&fixture.signed_owid_base64(payload)).unwrap_err(); + + let message = error.to_string(); + assert!( + message.contains(&format!("version {version}")), + "message did not name the version: {message}" + ); + } +} + +#[test] +fn the_payload_version_is_read_apart_from_the_usage_and_type_bits() { + // A reader masking the wrong bits would refuse a version 0 identifier + // or let a later version through, so every combination of the usage and + // type bits is tried. + let fixture = Fixture::new(); + for usage in [0b000u8, 0b001, 0b011, 0b111] { + for id_type in [0b00u8, 0b10, 0b11] { + let flags = (id_type << 6) | usage; + let mut payload = payload_ending_at_match_key(); + payload[layout::FLAGS_OFFSET] = flags; + + assert!( + FodId::from_base64(&fixture.signed_owid_base64(payload.clone())).is_ok(), + "flags {flags}" + ); + + for version in [1u8, 2, 3] { + let refused = FodId::from_base64( + &fixture.signed_owid_base64(with_payload_version(&payload, version)), + ); + assert!( + matches!(refused, Err(Error::UnsupportedPayloadVersion { .. })), + "flags {flags} version {version}" + ); + } + } + } +}