diff --git a/conformance/licenseterm_canonical_disjoint_test.go b/conformance/licenseterm_canonical_disjoint_test.go new file mode 100644 index 00000000..0805b8e0 --- /dev/null +++ b/conformance/licenseterm_canonical_disjoint_test.go @@ -0,0 +1,185 @@ +package conformance + +// Drift guard: the post-fold disjointness rule the SDKs enforce IS a rule the +// contract publishes. +// +// Disjointness is asserted twice, at two tiers, over two different readings of +// the same field. The wire tier's half is a protovalidate rule and needs no +// guard — it is IN the descriptor. The ingest tier's half is SDK code, and it +// cannot be a CEL rule for the reason the neighbouring ingest-tier checks give: +// the fold resolves aliases out of a generated vocabulary, and a CEL expression +// that re-listed that vocabulary would drift from it. +// +// So the rule exists twice — as SDK code, which is what the reference +// implementations run, and as prose in ramp.proto, which is what a third-party +// implementor reads. An implementor who read only the contract would otherwise +// build an Exchange that stores the term this rule exists to refuse, and every +// suite in every language would stay green. Nothing made the two agree; this +// file does. +// +// The halves work differently, as in regschema_rule_test.go. That the rule +// FIRES is read out of the committed vectors — conformance imports nothing from +// sdk/, so the ids arrive as data. That the contract STATES it is checked by +// reading the .proto SOURCE, because a comment does not survive into the +// descriptor and the comment is the thing an implementor actually reads. +// +// The phrases below are clauses only this rule's sentences can produce. A guard +// that matched a bare word would pass on text that never states the rule: the +// word "canonicalised" already occurs in several unrelated paragraphs of this +// file, and "disjoint" occurs in the wire rule's own id. + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "buf.build/go/protovalidate" + "google.golang.org/protobuf/reflect/protoreflect" +) + +const ( + rampProtoSource = "../proto/ramp/v1/ramp.proto" + + // The two ids. They are deliberately different strings: one name per rule, which + // licenseterm_rule_ids_test.go enforces across the whole ingest-tier id set. + canonicalDisjointRuleID = "restriction.canonical_disjoint" + wireDisjointRuleID = "restriction.permitted_prohibited_disjoint" +) + +// canonicalDisjointFires reports whether the committed license-term vectors record +// the ingest-tier rule at all. Read as data: this package is the guard tier BELOW +// the SDKs and imports none of them. +func canonicalDisjointFires(t *testing.T) bool { + t.Helper() + b, err := os.ReadFile(licenseTermVectors) + if err != nil { + t.Fatalf("read %s: %v", licenseTermVectors, err) + } + var doc struct { + Validate []struct { + Violation *struct { + Rule string `json:"rule"` + } `json:"violation"` + } `json:"validate"` + } + if err := json.Unmarshal(b, &doc); err != nil { + t.Fatalf("decode %s: %v", licenseTermVectors, err) + } + for _, v := range doc.Validate { + if v.Violation != nil && v.Violation.Rule == canonicalDisjointRuleID { + return true + } + } + return false +} + +// messageBody returns the source text between a declaration's opening brace and the +// first closing brace in column 1 — the whole body of a top-level message or service, +// including the comments inside it. +func messageBody(t *testing.T, src, decl string) string { + t.Helper() + start := strings.Index(src, decl) + if start < 0 { + t.Fatalf("%s: %q not found — the guard is reading the wrong file or the declaration was renamed", rampProtoSource, decl) + } + rest := src[start:] + end := strings.Index(rest, "\n}\n") + if end < 0 { + t.Fatalf("%s: %q is never closed in column 1", rampProtoSource, decl) + } + return rest[:end] +} + +// docComment returns the run of // lines immediately above a declaration. +func docComment(t *testing.T, src, decl string) string { + t.Helper() + at := strings.Index(src, decl) + if at < 0 { + t.Fatalf("%s: %q not found", rampProtoSource, decl) + } + lines := strings.Split(src[:at], "\n") + // The declaration starts a line, so the split ends with the empty string before it. + // Dropping it is what makes the walk below start on the last comment line rather + // than stopping instantly on a non-comment. + if len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { + lines = lines[:len(lines)-1] + } + var out []string + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if !strings.HasPrefix(line, "//") { + break + } + out = append([]string{line}, out...) + } + return strings.Join(out, "\n") +} + +func TestCanonicalDisjointRuleIsStatedByTheContract(t *testing.T) { + if !canonicalDisjointFires(t) { + t.Fatalf("no vector records %q — either the rule is gone or the corpus stopped covering it, "+ + "and this guard would be holding the contract to a rule nothing runs", canonicalDisjointRuleID) + } + + b, err := os.ReadFile(rampProtoSource) + if err != nil { + t.Fatalf("read %s: %v", rampProtoSource, err) + } + src := string(b) + + // Where the rule is declared, the contract must say that the declared rule reads + // the tokens as written and that the property is asserted again after the fold. + // Both halves are load-bearing: an implementor who takes the CEL rule for the + // whole rule builds the gap this exists to close. + restriction := messageBody(t, src, "message Restriction {") + for _, want := range []string{ + canonicalDisjointRuleID, + "compares the tokens AS WRITTEN", + "two spellings of ONE token pass this rule", + } { + if !strings.Contains(restriction, want) { + t.Errorf("message Restriction does not state %q — the SDK refuses a term the contract does not say it refuses", want) + } + } + + // And the service comment, which is where a publisher reads what the Exchange + // will run, must carry the rule in its ingest-tier list. + catalog := docComment(t, src, "service CatalogService {") + for _, want := range []string{ + canonicalDisjointRuleID, + "BOTH tiers assert", + } { + if !strings.Contains(catalog, want) { + t.Errorf("the CatalogService comment does not state %q — its two-tier description is incomplete", want) + } + } +} + +// The two rules must keep two names. The ingest-tier id set is held to that as a +// whole elsewhere; this pins the specific pair, because these two are the ones a +// reader is most likely to collapse into one. +func TestCanonicalDisjointDoesNotDisplaceTheWireRule(t *testing.T) { + if canonicalDisjointRuleID == wireDisjointRuleID { + t.Fatal("the ingest-tier and wire-tier ids are the same string — one name for two rules") + } + var found bool + EachMessage(func(md protoreflect.MessageDescriptor) { + if string(md.Name()) != "Restriction" { + return + } + mr, err := protovalidate.ResolveMessageRules(md) + if err != nil || mr == nil { + return + } + for _, r := range mr.GetCel() { + if r.GetId() == wireDisjointRuleID { + found = true + } + } + }) + if !found { + t.Errorf("Restriction no longer declares %q — the ingest-tier rule reads the canonicalised tokens "+ + "and was never a replacement for the boundary check over the tokens as received", wireDisjointRuleID) + } +} diff --git a/docs/design-history.md b/docs/design-history.md index 13f2b75d..7740fc13 100644 --- a/docs/design-history.md +++ b/docs/design-history.md @@ -1090,7 +1090,8 @@ A pushed entry passes two tiers at the Exchange. The wire tier is protovalidate: applied to the request exactly as received. The ingest tier runs afterwards, over the canonicalised terms: restriction tokens are folded and alias-resolved to their registered form, then a bare `Pricing.unit` or `Quota.metric` that is not a registered -token is rejected, while an unregistered restriction token and an `OBLIGATION_KIND_OTHER` +token is rejected, as is a restriction whose permitted and prohibited lists name one +token once folded, while an unregistered restriction token and an `OBLIGATION_KIND_OTHER` obligation without detail are accepted and reported in `PushResourcesResponse.warnings`. The first tier was always in the contract. The second lived only inside the reference Exchange, so a publisher learned what it would say by pushing and reading the refusal. @@ -1150,6 +1151,54 @@ what it does pin, at three different strengths. The two JSON ports walk every me reachable from an entry explicitly for the cross-field rules, because their composed schemas attach per message and a top-level parse would never reach `terms[1].pricing`. +The consequence of that order took a while to surface, and it is the reason the tier +boundary is not simply a matter of where a check is cheapest to run. A rule evaluated +before the fold is a rule about SPELLINGS. `restriction.permitted_prohibited_disjoint` +is `this.permitted.all(p, !(p in this.prohibited))` — a string comparison standing in +for a statement about tokens — and the vocabulary it reads has ten aliases and folds +ASCII case on every axis. So a term naming `scrape` under permitted and `crawl` under +prohibited passed the rule, because as written the two lists shared nothing, and the +fold then collapsed both into one token sitting in both lists. The wire tier had +cleared the very violation the ingest tier went on to create. + +What made it expensive was where it landed. The stored term rides on offers and an +Exchange validates its own responses, so the refusal arrived at a different party, on +a different RPC, an unbounded time later: discovery of that resource failed with an +internal error, while the push that caused it had been answered as accepted. A check +that runs on the wrong values does not merely miss things; it can certify the thing it +was written to prevent. + +The property is now asserted at both tiers, and that is the general rule this records: +a rule that compares two token-valued fields is only correct on canonical values, so +when an axis gains canonicalisation, every rule that reads it must be re-examined — +either the rule moves to the tier that sees canonical values, or canonicalisation moves +ahead of the tier that carries the rule. Here the rule moved, because the fold resolves +aliases out of a generated vocabulary and a CEL expression that re-listed that +vocabulary would drift from it, which is the same reason membership is not a descriptor +rule. + +Three details of the second half are deliberate. It has its own id +(`restriction.canonical_disjoint`) rather than reusing the first's, because one name for +two rules is what the ingest-tier id guard exists to prevent, and these two really are +different rules: they read different values and can disagree. It canonicalises what it +compares instead of trusting its caller to have folded first — both callers do fold, but +a rule written to close an ordering gap that opened its own would be the same mistake +one tier down, and the fold is a fixed point, so the cost is a pass over tokens already +canonical. And neither rule suppresses the other: a term that fails both is reported by +both, matching the tier the contract already describes as collecting every violation +rather than stopping at the first. Suppression was considered and rejected on a concrete +ground — the server binding defaults to validation off, so on a mount that never asks +for the wire tier the ingest check is the only one a pushed term meets, and a rule that +stayed silent because "the boundary already caught it" would catch nothing there. + +One boundary of that argument is worth stating, because the contract's own sentence +invites the stronger reading. Both disjointness rules are scoped to a single +`Restriction`; what makes one restriction the whole of an axis is +`license_term.one_restriction_per_kind`, which is a wire-tier rule. Split the two lists +across two restrictions of one kind and neither disjointness rule sees a collision. So a +mount without the wire tier gains the second READING of the rule, not the per-axis +property — the property has always been held jointly, and this change does not move it. + ## A catalog client is its own constructor The usage-report decision recorded above settled a rule that generalises: a client's diff --git a/docs/sdk-parity-matrix.md b/docs/sdk-parity-matrix.md index 9d1002f2..109ea63b 100644 --- a/docs/sdk-parity-matrix.md +++ b/docs/sdk-parity-matrix.md @@ -12,7 +12,7 @@ Go is the oracle (`sdk/go/{helpers,resolvers,core,connect,connectserver}`); Python and TS mirror it. This document is **generated** from the same two artifacts CI already enforces against the code, so it cannot drift from the real surface — a mismatch fails the API-surface gate or the corpus-completeness gate before it can reach this file. -**At a glance:** 133 symbols at cross-language parity · 16 documented divergences · 183 Go-idiomatic exclusions · 35 conformance corpora, each tri-replayed. +**At a glance:** 134 symbols at cross-language parity · 16 documented divergences · 183 Go-idiomatic exclusions · 35 conformance corpora, each tri-replayed. Layering (L1 pure trust core vs L2 I/O resolvers), the SSRF transport-wiring invariant, and naming conventions are recorded in [`design-history.md`](./design-history.md). @@ -85,6 +85,7 @@ Legend: a name = the public face in that language · `—` = intentionally none | `RuleObligationOtherRequiresDetail` | `RULE_OBLIGATION_OTHER_REQUIRES_DETAIL` | `RULE_OBLIGATION_OTHER_REQUIRES_DETAIL` | | `RulePricingUnitRegistered` | `RULE_PRICING_UNIT_REGISTERED` | `RULE_PRICING_UNIT_REGISTERED` | | `RuleQuotaMetricRegistered` | `RULE_QUOTA_METRIC_REGISTERED` | `RULE_QUOTA_METRIC_REGISTERED` | +| `RuleRestrictionCanonicalDisjoint` | `RULE_RESTRICTION_CANONICAL_DISJOINT` | `RULE_RESTRICTION_CANONICAL_DISJOINT` | | `RuleRestrictionTokenRegistered` | `RULE_RESTRICTION_TOKEN_REGISTERED` | `RULE_RESTRICTION_TOKEN_REGISTERED` | | `RuleViolation` | `RuleViolation` | `RuleViolation` | | `RuleWarning` | `RuleWarning` | `RuleWarning` | diff --git a/gen/descriptor.binpb b/gen/descriptor.binpb index f14f4a8a..4fc4136a 100644 Binary files a/gen/descriptor.binpb and b/gen/descriptor.binpb differ diff --git a/proto/CHANGELOG.md b/proto/CHANGELOG.md index 1e31c204..6497f2df 100644 --- a/proto/CHANGELOG.md +++ b/proto/CHANGELOG.md @@ -41,6 +41,30 @@ request-signing key already resolved. Holder binding for delegations still matches the wire signer, so a Broker may project a delegated request only when the agent has delegated to the Broker's key. +**A term is now checked for permitted/prohibited disjointness a second time, over +the canonicalised tokens (`restriction.canonical_disjoint`; SDK behaviour change +plus a comment clarification, no wire change).** `restriction.permitted_prohibited_disjoint` +runs at the wire tier, over the request exactly as received, so it compares token +SPELLINGS. Ten registered aliases resolve to eight distinct restriction tokens — +`scrape` is a registered alias of `crawl`, `adapt` and `derivative` both mean +`modify`, `personal` means `individual` — and every axis also folds ASCII case. A term naming +one spelling under `permitted` and another under `prohibited` therefore passed the +boundary check and became, once the ingest tier folded it, a stored term with the +same token in both lists. + +Nothing looked at it again, and the failure surfaced elsewhere: the term rides on +offers, an Exchange validates its own responses, and so every discovery request +returning that resource answered with an internal error while the push had looked +clean. The ingest tier now asserts the same property over the canonical values, +under its own rule id, in `ValidateLicenseTerm` and its Python and TypeScript twins. +The boundary rule is unchanged. + +Disjointness is now the one property both tiers assert, deliberately: they read +different values, neither suppresses the other, and a deployment that does not mount +the wire tier still gets the second. The `Restriction` and `CatalogService` comments +say so, and a conformance guard holds the contract's statement to the rule the SDKs +run. + **Signed delivery URLs are documented as Ed25519 signed by the Exchange and verified with its published public key, not HMAC-SHA256 over a shared secret (documentation correction; no wire change).** Since the initial public snapshot diff --git a/proto/ramp/v1/ramp.proto b/proto/ramp/v1/ramp.proto index f823f803..e8384c24 100644 --- a/proto/ramp/v1/ramp.proto +++ b/proto/ramp/v1/ramp.proto @@ -1231,6 +1231,24 @@ message License { // USER_TYPE — RAMP user/organization categories message Restriction { // A token cannot be both permitted and prohibited on the same axis. + // + // Read that as a property of the AXIS, held by two rules together: this one is + // scoped to a single Restriction, and one restriction per kind — the rule on + // LicenseTerm below — is what makes one Restriction the whole of an axis. Split + // the two lists across two restrictions of the same kind and neither disjointness + // rule sees a collision; that shape is refused by the one-per-kind rule instead, + // at the wire tier. + // + // This rule compares the tokens AS WRITTEN, because it runs at the wire tier, + // over the request exactly as received. Several tokens have more than one + // accepted spelling — a registered alias beside its canonical form, and either + // one in any ASCII case — so two spellings of ONE token pass this rule and + // become the same token when the ingest tier folds them. The ingest tier + // therefore asserts the same property a second time, over the canonicalised + // tokens, under the SDK rule id restriction.canonical_disjoint; see + // CatalogService. Both refuse the term, and a term that fails both is reported + // by both — the second is not a fallback for the first, it reads different + // values. option (buf.validate.message).cel = { id: "restriction.permitted_prohibited_disjoint" message: "a token cannot appear in both permitted and prohibited" @@ -2281,16 +2299,26 @@ enum IngestionSource { // GEOGRAPHY; a non-ASCII byte is never folded) and alias-resolved to their // registered form — the aliases are authored beside the tokens, on the // RestrictionKind values — after which a bare (non-namespaced) Pricing.unit or -// Quota.metric that is not a registered token is rejected, while an -// unregistered restriction token and an OBLIGATION_KIND_OTHER obligation -// without detail are accepted and reported in PushResourcesResponse.warnings. +// Quota.metric that is not a registered token is rejected, as is a restriction +// whose permitted and prohibited lists name one token once folded +// (restriction.canonical_disjoint), while an unregistered restriction token and +// an OBLIGATION_KIND_OTHER obligation without detail are accepted and reported +// in PushResourcesResponse.warnings. +// +// Disjointness is the one property BOTH tiers assert, and deliberately so: the +// wire tier reads the tokens as written and the ingest tier reads what the fold +// produced, so a term the boundary clears can still be refused here. A deployment +// that does not mount the wire tier still gets the second of those two readings — +// it does not thereby get the one-per-kind rule that makes a restriction the whole +// of its axis, which stays a wire-tier rule. +// // The SDK ships both tiers as a publisher-side pre-check; the Exchange's own // run of them is the deciding one. // // A push is ALL-OR-NOTHING, at both tiers. A hard rejection anywhere in the // submission — an envelope or term rule at the wire tier, an unregistered bare -// unit or metric at the ingest tier — refuses the entire submission and persists -// nothing; the publisher fixes and resubmits the whole set. The refusal names +// unit or metric or a restriction that is not disjoint once folded at the ingest +// tier — refuses the entire submission and persists nothing; the publisher fixes and resubmits the whole set. The refusal names // the entries that failed and why, and that per-entry detail is REPORTING, never // partial acceptance: no entry of a refused submission is stored. Warnings do // not refuse anything. diff --git a/sdk/go/README.md b/sdk/go/README.md index 243e13fd..c1cddc18 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -48,8 +48,12 @@ vr, err = helpers.VerifyRequestResolved(ctx, req, body, resolver, helpers.Verify **License-term pre-check** — the two tiers an Exchange applies to a pushed entry, runnable by a publisher before signing: the wire rules over the entry as given, then canonicalisation (RFC 8259 trim, ASCII-only fold, alias resolution through the generated -vocabulary) and registry membership over a copy of its terms. Warning messages are the -exact `PushResourcesResponse.warnings` strings; rule ids share the CEL-id namespace: +vocabulary), then registry membership and canonical disjointness over a copy of its terms. +The second of those is why the tier order matters: `permitted: ["scrape"]` with +`prohibited: ["crawl"]` clears the boundary rule, which compares the tokens as written, and +names one token once folded. Warning messages are the exact +`PushResourcesResponse.warnings` strings; rule ids share the CEL-id namespace +(`pricing.unit.registered`, `restriction.canonical_disjoint`, …): ```go verdict := helpers.ValidateResourceEntry(entry) // never modifies entry diff --git a/sdk/go/helpers/gen_licenseterm_vectors_test.go b/sdk/go/helpers/gen_licenseterm_vectors_test.go index 76b4e93f..96289241 100644 --- a/sdk/go/helpers/gen_licenseterm_vectors_test.go +++ b/sdk/go/helpers/gen_licenseterm_vectors_test.go @@ -17,8 +17,11 @@ package helpers import ( "encoding/json" "errors" + "fmt" "os" "path/filepath" + "sort" + "strings" "testing" "google.golang.org/protobuf/encoding/protojson" @@ -26,6 +29,9 @@ import ( "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" rampv1 "github.com/RAMP-Protocol/protocol/gen/go/ramp/v1" + "github.com/RAMP-Protocol/protocol/gen/go/vocab/functiontokens" + "github.com/RAMP-Protocol/protocol/gen/go/vocab/geographytokens" + "github.com/RAMP-Protocol/protocol/gen/go/vocab/usertypes" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -68,7 +74,17 @@ type ltFinding struct { Message string `json:"message"` } -// ltValidateVector is one ValidateLicenseTerm case over an already-canonical term. +// ltValidateVector is one ValidateLicenseTerm case. +// +// Its term is canonical wherever the case is ACCEPTED, because every check but one +// reads it that way. The disjointness rejections are the exception and carry the term +// as a publisher authored it — an alias beside its registered form, or two spellings +// differing in case — because that check folds what it compares and exists precisely +// to catch what the fold produces. A case that is accepted must stay canonical: read +// as written, a bare alias earns a warning that the fold takes away, and this list +// would then record a warning no Exchange sends. Such a case belongs in the entry +// list, where a copy is folded first. + type ltValidateVector struct { Name string `json:"name"` Term json.RawMessage `json:"term"` @@ -106,6 +122,19 @@ type ltEntryVector struct { Warnings []ltFinding `json:"warnings"` } +// ltIngestRejectRuleIDs is the set of rule ids ValidateLicenseTerm can return as a +// VIOLATION. It is the one authored copy of that classification: the emitter uses it +// to sort an entry's violations into columns, and the three replays derive the same +// set from the corpus this emitter writes, so a new ingest-tier reject is registered +// here and nowhere else. +// +// It cannot be derived on this side — the emitter produces the corpus the others read. +var ltIngestRejectRuleIDs = map[string]bool{ + RulePricingUnitRegistered: true, + RuleQuotaMetricRegistered: true, + RuleRestrictionCanonicalDisjoint: true, +} + // ltCrossFieldRuleIDs is every message-level CEL id the contract declares, read // from the generated descriptor rather than listed here. Listing them would be a // fourth copy of a set the proto already owns, and the copy that drifts is the one @@ -236,6 +265,23 @@ func buildLTFoldVectors() []ltFoldVector { return out } +// ltSortedAliases returns an axis's alias spellings in a fixed order, so the +// derived case list is stable across runs — Go randomises map iteration and the +// corpus is compared byte for byte. +func ltSortedAliases(aliases map[string]string) []string { + out := make([]string, 0, len(aliases)) + for alias := range aliases { + out = append(out, alias) + } + sort.Strings(out) + return out +} + +// ltCaseToken renders a token as a case-name segment. Only the hyphen needs +// folding: every registered spelling is otherwise lowercase alphanumerics and +// underscores, which the vocabulary generator enforces. +func ltCaseToken(token string) string { return strings.ReplaceAll(token, "-", "_") } + func ltRestriction(kind rampv1.RestrictionKind, permitted, prohibited []string) *rampv1.Restriction { return &rampv1.Restriction{Kind: kind, Permitted: permitted, Prohibited: prohibited} } @@ -368,10 +414,11 @@ func buildLTValidateVectors(t *testing.T) []ltValidateVector { licenseURI := func() *rampv1.License { return &rampv1.License{Uri: proto.String("https://example.com/license.txt"), UriDigest: proto.String("sha256:" + ltRepeatHex(64))} } - cases := []struct { + type validateCase struct { name string term *rampv1.LicenseTerm - }{ + } + cases := []validateCase{ {"reference_only_with_restrictions_accepted", &rampv1.LicenseTerm{ Semantics: rampv1.TermSemantics_TERM_SEMANTICS_REFERENCE_ONLY, License: licenseURI(), Pricing: ltPerUnitPricing("accesses"), Restrictions: []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"ai-train"}, nil)}, @@ -456,7 +503,93 @@ func buildLTValidateVectors(t *testing.T) []ltValidateVector { x.Restrictions = []*rampv1.Restriction{{Permitted: []string{"flibbertigibbet"}}} })}, {"empty_term_accepted", &rampv1.LicenseTerm{}}, + + // The canonical-disjointness reject. The alias pairs are derived below; these + // are the shapes a table of aliases cannot express. + // + // Two ALIASES of one token, neither of them the registered spelling — the + // pairing the derived cases never produce, because they always pair an alias + // with its canonical form. + {"restriction_two_aliases_of_one_token_collide_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"adapt"}, []string{"derivative"})} + })}, + // Case folding alone, with no alias in sight: the axis that folds DOWN, and + // the axis that folds UP and registers no aliases at all. + {"restriction_function_case_collides_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"CRAWL"}, []string{"crawl"})} + })}, + {"restriction_geography_case_collides_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindGeography, []string{"us"}, []string{"US"})} + })}, + // An axis with no canonicalisation rule at all. The check still runs there: + // the fold returns the token unchanged and the comparison is plain equality, + // which is the only tier a term meets on a server that never asked for the + // wire tier. + {"restriction_other_axis_exact_duplicate_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindOther, []string{"custom-use"}, []string{"custom-use"})} + })}, + // The RFC 8259 trim is part of the fold, so it can collide two spellings too. + // Only a direct caller can reach this: the wire pattern forbids whitespace in + // a token, so an entry carrying it never gets as far as the ingest tier. + {"restriction_padded_token_collides_after_trim_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{" crawl "}, []string{"crawl"})} + })}, + // The other side of the rule: folding must not invent a collision. The + // alias-bearing form of this case is an ENTRY vector, not one of these — + // see alias_disjoint_after_fold_accepted for why. + {"restriction_namespaced_tokens_disjoint_accepted", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindOther, []string{"acme:read"}, []string{"acme:write"})} + })}, + // Scan order, both across restrictions and within one. A term with more than + // one collision reports exactly one, and which one is not left to chance. + {"restriction_second_restriction_collides_rejected", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ + ltRestriction(ltKindFunction, []string{"ai-train"}, []string{"search"}), + ltRestriction(ltKindUserType, []string{"personal"}, []string{"individual"}), + } + })}, + {"restriction_collision_reports_first_permitted_site", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, + []string{"tdm", "scrape"}, []string{"crawl", "text-and-data-mining"})} + })}, + // Precedence against the checks that run before it. + {"quota_metric_checked_before_restriction_collision", ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Quotas = []*rampv1.Quota{{Metric: "frobnications", Limit: 10, Window: rampv1.QuotaWindow_QUOTA_WINDOW_DAILY}} + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"scrape"}, []string{"crawl"})} + })}, + } + + // One case per REGISTERED ALIAS, derived from the generated vocabulary rather + // than listed, so an alias added to the proto brings its vector with it and the + // obligation to cover every alias pair cannot quietly rot. Each pairs the alias + // under permitted with its registered form under prohibited: two strings the + // wire tier reads as disjoint and the fold reads as one token. + // + // Every axis that CAN carry aliases is listed, not every axis that does today: + // GEOGRAPHY registers none, so it contributes nothing now and contributes a + // vector the day it registers one. Listing only the axes with aliases would have + // made the sentence above false for the axis nobody was thinking about, which is + // how this rule's own defect reached the catalog. + for _, axis := range []struct { + name string + kind rampv1.RestrictionKind + aliases map[string]string + }{ + {"function", ltKindFunction, functiontokens.Aliases}, + {"geography", ltKindGeography, geographytokens.Aliases}, + {"user_type", ltKindUserType, usertypes.Aliases}, + } { + for _, alias := range ltSortedAliases(axis.aliases) { + canonical := axis.aliases[alias] + cases = append(cases, validateCase{ + fmt.Sprintf("restriction_alias_%s_%s_collides_rejected", axis.name, ltCaseToken(alias)), + ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(axis.kind, []string{alias}, []string{canonical})} + }), + }) + } } + out := make([]ltValidateVector, 0, len(cases)) for _, c := range cases { warnings, err := ValidateLicenseTerm(c.term) // REAL face @@ -495,6 +628,19 @@ func buildLTEntryVectors(t *testing.T) []ltEntryVector { {"alias_resolved_before_membership_no_warning", entry(func(e *rampv1.ResourceEntry) { e.Terms[0].Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"Generative-AI"}, nil)} })}, + // Folding must not invent a collision, recorded here rather than in the + // per-term list because only the composed face answers what an Exchange + // answers. A bare alias earns an unregistered-token warning when it is read + // AS WRITTEN, and earns none once the fold resolves it to a registered token; + // the entry face folds a copy first, exactly as the Exchange does, so the + // warning this records is the one a publisher is really sent. Its neighbour + // above pins the same asymmetry for membership; this one pins it for + // disjointness. + {"alias_disjoint_after_fold_accepted", entry(func(e *rampv1.ResourceEntry) { + e.Terms[0].Restrictions = []*rampv1.Restriction{ + ltRestriction(ltKindFunction, []string{"scrape"}, []string{"ai-train"}), + } + })}, {"unknown_token_warns_with_entry_path", entry(func(e *rampv1.ResourceEntry) { e.Terms = append(e.Terms, ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"ai-train", "flibbertigibbet"}, nil)} @@ -596,6 +742,14 @@ func buildLTEntryVectors(t *testing.T) []ltEntryVector { e.Path = "x" e.Terms[0] = ltEnumerated(ltPerUnitPricing("frobnications"), nil) })}, + // The ingest tier catching what the wire tier cleared. The entry is + // wire-conformant — two different strings — so the boundary rule reports + // nothing and the reject arrives with an entry-relative path. + {"term_reject_restriction_canonical_disjoint", entry(func(e *rampv1.ResourceEntry) { + e.Terms[0] = ltEnumerated(ltFreePricing(), func(x *rampv1.LicenseTerm) { + x.Restrictions = []*rampv1.Restriction{ltRestriction(ltKindFunction, []string{"scrape"}, []string{"crawl"})} + }) + })}, {"no_terms_accepted", entry(func(e *rampv1.ResourceEntry) { e.Terms = nil })}, } celIDs := ltCrossFieldRuleIDs(t) @@ -611,7 +765,7 @@ func buildLTEntryVectors(t *testing.T) []ltEntryVector { CrossFieldRules: []string{}, TermRules: []ltFinding{}, Warnings: ltWarningsOf(verdict.Warnings)} for _, viol := range verdict.Violations { switch { - case viol.Rule == RulePricingUnitRegistered || viol.Rule == RuleQuotaMetricRegistered: + case ltIngestRejectRuleIDs[viol.Rule]: v.TermRules = append(v.TermRules, ltFindingOf(viol.Rule, viol.Path, viol.Token, viol.Message)) case celIDs[viol.Rule]: v.CrossFieldRules = append(v.CrossFieldRules, viol.Rule) diff --git a/sdk/go/helpers/licenseterm.go b/sdk/go/helpers/licenseterm.go index de8ee922..cc090eb6 100644 --- a/sdk/go/helpers/licenseterm.go +++ b/sdk/go/helpers/licenseterm.go @@ -23,7 +23,8 @@ import ( // structural and cross-field rules, applied to the request as received. The // ingest tier runs over the CANONICALISED terms: restriction tokens are folded // and alias-resolved to their registered form, then a bare Pricing.unit or -// Quota.metric that is not a registered token is rejected, while an +// Quota.metric that is not a registered token is rejected, as is a restriction +// whose permitted and prohibited lists name one token once folded, while an // unregistered restriction token and an OBLIGATION_KIND_OTHER obligation // without detail are accepted with a warning that reaches // PushResourcesResponse.warnings. @@ -55,6 +56,13 @@ const ( // RuleQuotaMetricRegistered rejects a bare Quota.metric that is not a // registered quota token. RuleQuotaMetricRegistered = "quota.metric.registered" + // RuleRestrictionCanonicalDisjoint rejects a restriction whose permitted and + // prohibited lists name the same token once both are canonicalised. The wire + // tier's rule compares the tokens AS WRITTEN, so two accepted spellings of one + // token — an alias beside its registered form, or two spellings differing only + // in ASCII case — pass it and collide only after the fold. This is that + // property read on the values the fold produces. + RuleRestrictionCanonicalDisjoint = "restriction.canonical_disjoint" // RuleRestrictionTokenRegistered warns about a bare restriction token that is // not registered on its axis. The term is accepted: the restriction // vocabulary is open and forward-compatible, and under scope-only projection @@ -150,7 +158,9 @@ func KnownRestrictionToken(kind rampv1.RestrictionKind, token string) bool { // form in place, on every axis that carries a canonicalisation rule. It touches // nothing else — Pricing.unit and Quota.metric are exact registry values, scopes // are matched verbatim — and is nil-safe and idempotent. Run it before -// ValidateLicenseTerm, which assumes canonical tokens. +// ValidateLicenseTerm, whose checks read canonical tokens — all but the +// disjointness check, which folds what it compares and so reaches the same +// verdict on either form. func NormalizeLicenseTerm(term *rampv1.LicenseTerm) { for _, r := range term.GetRestrictions() { kind := r.GetKind() @@ -174,18 +184,29 @@ func NormalizeResourceEntry(entry *rampv1.ResourceEntry) { } } -// ValidateLicenseTerm runs the ingest-tier checks over one already-canonical -// term. It returns a *RuleViolation as the error for a hard reject — a bare -// Pricing.unit or Quota.metric that is not a registered token, pricing checked -// first and the first offending quota winning — and, when the term is accepted, -// the warnings it would carry: one per unregistered bare restriction token, in +// ValidateLicenseTerm runs the ingest-tier checks over one term. It returns a +// *RuleViolation as the error for a hard reject, in a fixed order: a bare +// Pricing.unit that is not a registered token, then the first offending quota +// metric, then the first restriction whose permitted and prohibited lists name +// one token once canonicalised. When the term is accepted it returns the +// warnings it would carry: one per unregistered bare restriction token, in // restriction order with permitted before prohibited, then one per // OBLIGATION_KIND_OTHER obligation without detail. Empty and vendor-namespaced // (containing ":") tokens are never membership-checked. // +// Every check but one reads the term as ALREADY CANONICAL, which is what +// NormalizeLicenseTerm produces. The exception is the disjointness check, which +// folds what it compares: it exists because a rule that compares spellings +// answers a question about tokens, so a rule written to close that gap must not +// in turn assume its own caller folded first. +// // The wire tier is not re-run here: token format, PER_UNIT⇒unit, FREE⇒rate 0, -// one restriction per kind, permitted∩prohibited and the presence rules are -// protovalidate's, and ValidateResourceEntry composes the two tiers. +// one restriction per kind and the presence rules are protovalidate's, and +// ValidateResourceEntry composes the two tiers. Disjointness is the one property +// BOTH tiers assert, over different values — permitted∩prohibited over the tokens +// as written, and the rule below over the tokens the fold produces — so a term +// the first clears can still fail the second, and a term that fails both is +// reported by both. func ValidateLicenseTerm(term *rampv1.LicenseTerm) ([]RuleWarning, error) { if unit := term.GetPricing().GetUnit(); bareUnregistered(unit, pricingunits.IsRegistered) { return nil, &RuleViolation{ @@ -205,6 +226,11 @@ func ValidateLicenseTerm(term *rampv1.LicenseTerm) ([]RuleWarning, error) { } } } + for i, r := range term.GetRestrictions() { + if v := canonicalDisjointViolation(i, r); v != nil { + return nil, v + } + } var warnings []RuleWarning for i, r := range term.GetRestrictions() { kind := r.GetKind() @@ -284,6 +310,57 @@ func ValidateResourceEntry(entry *rampv1.ResourceEntry) EntryVerdict { return verdict } +// canonicalDisjointViolation returns the violation for the first permitted token +// of r whose canonical form is also the canonical form of one of its prohibited +// tokens, or nil when the two lists name no token in common. The permitted list +// decides the order, so a restriction carrying several collisions always answers +// the same way. +// +// It canonicalises what it compares rather than trusting r to have been +// normalised, and it runs on EVERY axis. On an axis with no canonicalisation +// rule the fold returns the token unchanged, so the check degenerates to plain +// equality — which is what it must do on a server that never asked for the wire +// tier, where this is the only tier a pushed term meets. +// +// The finding names the CANONICAL token and not the spellings that produced it. +// Naming the spellings would make the message depend on whether the caller had +// folded first — the Exchange has, a direct caller has not — and the message is +// wire-visible and pinned byte-for-byte across the three SDKs. Path locates the +// permitted element; the prohibited one is the entry that folds to the same token. +// +// The prohibited list is folded once into a set, so the cost is one fold per token +// rather than one per pair — linear in the tokens present, where a pairwise walk +// would have been their product. The bound on how many arrive is NOT this rule's: +// where the wire tier runs it is the max_items cap on each list, and on the mount +// named above, which has no wire tier, it is the size of the request the server +// agreed to read. Stating the cap alone would have claimed a bound that the one +// deployment this check exists for does not have. +func canonicalDisjointViolation(i int, r *rampv1.Restriction) *RuleViolation { + prohibited := r.GetProhibited() + permitted := r.GetPermitted() + if len(prohibited) == 0 || len(permitted) == 0 { + return nil + } + kind := r.GetKind() + banned := make(map[string]struct{}, len(prohibited)) + for _, tok := range prohibited { + banned[CanonicalRestrictionToken(kind, tok)] = struct{}{} + } + for j, tok := range permitted { + canon := CanonicalRestrictionToken(kind, tok) + if _, ok := banned[canon]; !ok { + continue + } + return &RuleViolation{ + Rule: RuleRestrictionCanonicalDisjoint, + Path: fmt.Sprintf("restrictions[%d].permitted[%d]", i, j), + Token: canon, + Message: fmt.Sprintf("restriction token \"%s\" is both permitted and prohibited after canonicalisation", canon), + } + } + return nil +} + // restrictionTokenWarning returns the warning for one restriction token, or // false when the token needs none (empty, vendor-namespaced, or registered). func restrictionTokenWarning(kind rampv1.RestrictionKind, tok, path string) (RuleWarning, bool) { diff --git a/sdk/go/helpers/licenseterm_corpus_test.go b/sdk/go/helpers/licenseterm_corpus_test.go index a96a4d8a..6a469b9d 100644 --- a/sdk/go/helpers/licenseterm_corpus_test.go +++ b/sdk/go/helpers/licenseterm_corpus_test.go @@ -211,9 +211,36 @@ func TestLicenseTermCorpus_Validate(t *testing.T) { } } +// corpusTermRuleIDs is the set of ids ValidateLicenseTerm can return as a violation, +// read out of the corpus's own per-term list rather than listed here: an ingest-tier +// reject IS an id that list can carry as a violation. Reading them beats restating +// them, the same trade the cross-field set already makes — a classification written +// down twice is a classification that can disagree with itself. +func corpusTermRuleIDs(t *testing.T, c licenseTermCorpus) map[string]bool { + t.Helper() + ids := map[string]bool{} + for _, v := range c.Validate { + if v.Violation != nil { + ids[v.Violation.Rule] = true + } + } + // Guard the guard. A derivation that stopped reading the column would leave an + // empty set here and quietly move every term reject into the structural bucket, + // where the entry assertions would still pass. Both long-standing ids are + // reachable from that list, so requiring them pins that it is still being read. + for _, want := range []string{helpers.RulePricingUnitRegistered, helpers.RuleQuotaMetricRegistered} { + if !ids[want] { + t.Fatalf("the per-term list carries no %q — the column this classification is derived from has moved", want) + } + } + return ids +} + func TestLicenseTermCorpus_Entry(t *testing.T) { + corpus := loadLicenseTermCorpus(t) celIDs := corpusCrossFieldRuleIDs(t) - for _, v := range loadLicenseTermCorpus(t).Entry { + termRuleIDs := corpusTermRuleIDs(t, corpus) + for _, v := range corpus.Entry { var entry rampv1.ResourceEntry if err := protojson.Unmarshal(v.Entry, &entry); err != nil { t.Fatalf("%s: decode entry: %v", v.Name, err) @@ -230,7 +257,7 @@ func TestLicenseTermCorpus_Entry(t *testing.T) { termRules := []corpusFinding{} for _, viol := range verdict.Violations { switch { - case viol.Rule == helpers.RulePricingUnitRegistered || viol.Rule == helpers.RuleQuotaMetricRegistered: + case termRuleIDs[viol.Rule]: termRules = append(termRules, corpusFinding{Rule: viol.Rule, Path: viol.Path, Token: viol.Token, Message: viol.Message}) case celIDs[viol.Rule]: crossField = append(crossField, viol.Rule) diff --git a/sdk/go/helpers/licenseterm_test.go b/sdk/go/helpers/licenseterm_test.go index fde987d7..f14f32da 100644 --- a/sdk/go/helpers/licenseterm_test.go +++ b/sdk/go/helpers/licenseterm_test.go @@ -109,3 +109,64 @@ func TestValidateResourceEntry_ReportsBothTiersWithEntryPaths(t *testing.T) { t.Errorf("want both the wire-tier path violation and the ingest-tier term violation, got %+v", verdict.Violations) } } + +// The disjointness check is the one ingest-tier rule that does not read its input +// as already canonical, so the property worth pinning here is that folding first +// changes nothing: the Exchange normalises in place before it validates, a +// publisher's pre-check does not, and both must reach the same verdict. The value +// table for the rule lives in the shared corpus. +func TestValidateLicenseTerm_CanonicalDisjointIsIndifferentToFoldingFirst(t *testing.T) { + raw := func() *rampv1.LicenseTerm { + return &rampv1.LicenseTerm{Restrictions: []*rampv1.Restriction{{ + Kind: rampv1.RestrictionKind_RESTRICTION_KIND_FUNCTION, + Permitted: []string{"scrape"}, + Prohibited: []string{"crawl"}, + }}} + } + before := raw() + _, errBefore := helpers.ValidateLicenseTerm(before) + normalized := raw() + helpers.NormalizeLicenseTerm(normalized) + _, errAfter := helpers.ValidateLicenseTerm(normalized) + + var rv *helpers.RuleViolation + if !errors.As(errBefore, &rv) { + t.Fatalf("unfolded term: error is %T, want *helpers.RuleViolation", errBefore) + } + if rv.Rule != helpers.RuleRestrictionCanonicalDisjoint || rv.Token != "crawl" { + t.Errorf("violation = %+v", *rv) + } + if errAfter == nil { + t.Fatal("folded term: expected the same violation, got none") + } + if rv.Rule != helpers.RuleRestrictionCanonicalDisjoint { + t.Errorf("rule = %q", rv.Rule) + } + if errBefore.Error() == errAfter.Error() { + return + } + t.Errorf("the verdict depends on whether the caller folded first:\n raw: %s\n folded: %s", + errBefore.Error(), errAfter.Error()) +} + +// Folding rewrites restriction TOKENS and nothing else, so the sibling rule that +// counts restrictions per kind cannot be created or destroyed by it. That is +// assumed by the split between the tiers; assert it rather than believe it. +func TestNormalizeLicenseTerm_LeavesRestrictionKindAlone(t *testing.T) { + term := &rampv1.LicenseTerm{Restrictions: []*rampv1.Restriction{ + {Kind: rampv1.RestrictionKind_RESTRICTION_KIND_FUNCTION, Permitted: []string{"SCRAPE"}}, + {Kind: rampv1.RestrictionKind_RESTRICTION_KIND_GEOGRAPHY, Permitted: []string{"us"}}, + {Kind: rampv1.RestrictionKind_RESTRICTION_KIND_USER_TYPE, Prohibited: []string{"Personal"}}, + {Kind: rampv1.RestrictionKind_RESTRICTION_KIND_OTHER, Permitted: []string{"Left-Alone"}}, + }} + want := make([]rampv1.RestrictionKind, len(term.Restrictions)) + for i, r := range term.Restrictions { + want[i] = r.GetKind() + } + helpers.NormalizeLicenseTerm(term) + for i, r := range term.Restrictions { + if r.GetKind() != want[i] { + t.Errorf("restrictions[%d].kind = %v, want %v", i, r.GetKind(), want[i]) + } + } +} diff --git a/sdk/go/helpers/testdata/licenseterm-vectors.json b/sdk/go/helpers/testdata/licenseterm-vectors.json index 8edb29a2..7b753b0a 100644 --- a/sdk/go/helpers/testdata/licenseterm-vectors.json +++ b/sdk/go/helpers/testdata/licenseterm-vectors.json @@ -71,6 +71,38 @@ "term_rules": [], "warnings": [] }, + { + "name": "alias_disjoint_after_fold_accepted", + "entry": { + "domain": "publisher.example", + "path": "/premium/article-42.html", + "terms": [ + { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "scrape" + ], + "prohibited": [ + "ai-train" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + } + ] + }, + "ok": true, + "structural": false, + "cross_field_rules": [], + "term_rules": [], + "warnings": [] + }, { "name": "unknown_token_warns_with_entry_path", "entry": { @@ -285,7 +317,14 @@ "cross_field_rules": [ "restriction.permitted_prohibited_disjoint" ], - "term_rules": [], + "term_rules": [ + { + "rule": "restriction.canonical_disjoint", + "path": "terms[0].restrictions[0].permitted[0]", + "token": "ai-train", + "message": "restriction token \"ai-train\" is both permitted and prohibited after canonicalisation" + } + ], "warnings": [] }, { @@ -943,6 +982,45 @@ ], "warnings": [] }, + { + "name": "term_reject_restriction_canonical_disjoint", + "entry": { + "domain": "publisher.example", + "path": "/premium/article-42.html", + "terms": [ + { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "scrape" + ], + "prohibited": [ + "crawl" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + } + ] + }, + "ok": false, + "structural": false, + "cross_field_rules": [], + "term_rules": [ + { + "rule": "restriction.canonical_disjoint", + "path": "terms[0].restrictions[0].permitted[0]", + "token": "crawl", + "message": "restriction token \"crawl\" is both permitted and prohibited after canonicalisation" + } + ], + "warnings": [] + }, { "name": "no_terms_accepted", "entry": { @@ -2262,6 +2340,551 @@ "term": {}, "violation": null, "warnings": [] + }, + { + "name": "restriction_two_aliases_of_one_token_collide_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "adapt" + ], + "prohibited": [ + "derivative" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "modify", + "message": "restriction token \"modify\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_function_case_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "CRAWL" + ], + "prohibited": [ + "crawl" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "crawl", + "message": "restriction token \"crawl\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_geography_case_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_GEOGRAPHY", + "permitted": [ + "us" + ], + "prohibited": [ + "US" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "US", + "message": "restriction token \"US\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_other_axis_exact_duplicate_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_OTHER", + "permitted": [ + "custom-use" + ], + "prohibited": [ + "custom-use" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "custom-use", + "message": "restriction token \"custom-use\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_padded_token_collides_after_trim_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + " crawl " + ], + "prohibited": [ + "crawl" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "crawl", + "message": "restriction token \"crawl\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_namespaced_tokens_disjoint_accepted", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_OTHER", + "permitted": [ + "acme:read" + ], + "prohibited": [ + "acme:write" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": null, + "warnings": [] + }, + { + "name": "restriction_second_restriction_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "ai-train" + ], + "prohibited": [ + "search" + ] + }, + { + "kind": "RESTRICTION_KIND_USER_TYPE", + "permitted": [ + "personal" + ], + "prohibited": [ + "individual" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[1].permitted[0]", + "token": "individual", + "message": "restriction token \"individual\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_collision_reports_first_permitted_site", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "tdm", + "scrape" + ], + "prohibited": [ + "crawl", + "text-and-data-mining" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "text-and-data-mining", + "message": "restriction token \"text-and-data-mining\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "quota_metric_checked_before_restriction_collision", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "quotas": [ + { + "limit": "10", + "metric": "frobnications", + "window": "QUOTA_WINDOW_DAILY" + } + ], + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "scrape" + ], + "prohibited": [ + "crawl" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "quota.metric.registered", + "path": "quotas[0].metric", + "token": "frobnications", + "message": "quota metric \"frobnications\" is not a registered quota token" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_adapt_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "adapt" + ], + "prohibited": [ + "modify" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "modify", + "message": "restriction token \"modify\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_copy_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "copy" + ], + "prohibited": [ + "reproduce" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "reproduce", + "message": "restriction token \"reproduce\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_derivative_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "derivative" + ], + "prohibited": [ + "modify" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "modify", + "message": "restriction token \"modify\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_generative_ai_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "generative-ai" + ], + "prohibited": [ + "ai-input" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "ai-input", + "message": "restriction token \"ai-input\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_scrape_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "scrape" + ], + "prohibited": [ + "crawl" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "crawl", + "message": "restriction token \"crawl\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_tdm_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "tdm" + ], + "prohibited": [ + "text-and-data-mining" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "text-and-data-mining", + "message": "restriction token \"text-and-data-mining\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_function_train_ai_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_FUNCTION", + "permitted": [ + "train-ai" + ], + "prohibited": [ + "ai-train" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "ai-train", + "message": "restriction token \"ai-train\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_user_type_business_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_USER_TYPE", + "permitted": [ + "business" + ], + "prohibited": [ + "commercial_entity" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "commercial_entity", + "message": "restriction token \"commercial_entity\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_user_type_enterprise_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_USER_TYPE", + "permitted": [ + "enterprise" + ], + "prohibited": [ + "commercial_entity" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "commercial_entity", + "message": "restriction token \"commercial_entity\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] + }, + { + "name": "restriction_alias_user_type_personal_collides_rejected", + "term": { + "pricing": { + "model": "PRICING_MODEL_FREE", + "rate": "0" + }, + "restrictions": [ + { + "kind": "RESTRICTION_KIND_USER_TYPE", + "permitted": [ + "personal" + ], + "prohibited": [ + "individual" + ] + } + ], + "semantics": "TERM_SEMANTICS_ENUMERATED" + }, + "violation": { + "rule": "restriction.canonical_disjoint", + "path": "restrictions[0].permitted[0]", + "token": "individual", + "message": "restriction token \"individual\" is both permitted and prohibited after canonicalisation" + }, + "warnings": [] } ] } diff --git a/sdk/parity/symbol-map.json b/sdk/parity/symbol-map.json index 1b519595..c452a43c 100644 --- a/sdk/parity/symbol-map.json +++ b/sdk/parity/symbol-map.json @@ -654,6 +654,11 @@ "python": "RULE_QUOTA_METRIC_REGISTERED", "ts": "RULE_QUOTA_METRIC_REGISTERED" }, + "helpers.RuleRestrictionCanonicalDisjoint": { + "allowlist_reason": null, + "python": "RULE_RESTRICTION_CANONICAL_DISJOINT", + "ts": "RULE_RESTRICTION_CANONICAL_DISJOINT" + }, "helpers.RuleRestrictionTokenRegistered": { "allowlist_reason": null, "python": "RULE_RESTRICTION_TOKEN_REGISTERED", diff --git a/sdk/python/ramp_sdk/__init__.py b/sdk/python/ramp_sdk/__init__.py index cc087268..e59522a9 100644 --- a/sdk/python/ramp_sdk/__init__.py +++ b/sdk/python/ramp_sdk/__init__.py @@ -98,6 +98,7 @@ RULE_OBLIGATION_OTHER_REQUIRES_DETAIL, RULE_PRICING_UNIT_REGISTERED, RULE_QUOTA_METRIC_REGISTERED, + RULE_RESTRICTION_CANONICAL_DISJOINT, RULE_RESTRICTION_TOKEN_REGISTERED, EntryVerdict, RuleViolation, @@ -207,6 +208,7 @@ "RULE_OBLIGATION_OTHER_REQUIRES_DETAIL", "RULE_PRICING_UNIT_REGISTERED", "RULE_QUOTA_METRIC_REGISTERED", + "RULE_RESTRICTION_CANONICAL_DISJOINT", "RULE_RESTRICTION_TOKEN_REGISTERED", "RegistrationDataVerdict", "RegistrationSchema", diff --git a/sdk/python/ramp_sdk/licenseterm.py b/sdk/python/ramp_sdk/licenseterm.py index cf2a9b7c..3b3e087f 100644 --- a/sdk/python/ramp_sdk/licenseterm.py +++ b/sdk/python/ramp_sdk/licenseterm.py @@ -6,7 +6,8 @@ field-level model plus the cross-field (message-CEL) rules, applied to the entry as received. The ingest tier runs over the CANONICALISED terms: restriction tokens are folded and alias-resolved to their registered form, then a bare -``Pricing.unit`` or ``Quota.metric`` that is not a registered token is rejected, +``Pricing.unit`` or ``Quota.metric`` that is not a registered token is rejected, as +is a restriction whose permitted and prohibited lists name one token once folded, while an unregistered restriction token and an ``OBLIGATION_KIND_OTHER`` obligation without detail are accepted with a warning that reaches ``PushResourcesResponse.warnings``. The Exchange's own run is the deciding one; a @@ -36,6 +37,11 @@ RULE_PRICING_UNIT_REGISTERED = "pricing.unit.registered" #: Rejects a bare Quota.metric that is not a registered quota token. RULE_QUOTA_METRIC_REGISTERED = "quota.metric.registered" +#: Rejects a restriction whose permitted and prohibited lists name the same token once both +#: are canonicalised. The wire tier's rule compares the tokens AS WRITTEN, so two accepted +#: spellings of one token — an alias beside its registered form, or two spellings differing +#: only in ASCII case — pass it and collide only after the fold. +RULE_RESTRICTION_CANONICAL_DISJOINT = "restriction.canonical_disjoint" #: Warns about a bare restriction token not registered on its axis; the term is accepted. RULE_RESTRICTION_TOKEN_REGISTERED = "restriction.token.registered" #: Warns about an OBLIGATION_KIND_OTHER obligation carrying no detail. @@ -82,7 +88,11 @@ class RuleWarning: @dataclass(frozen=True) class TermVerdict: - """What :func:`validate_license_term` reports for one already-canonical term.""" + """What :func:`validate_license_term` reports for one term. + + Every check but the disjointness one reads the term as already canonical; that one + folds what it compares, so it is correct on a term as authored too. + """ violation: RuleViolation | None warnings: list[RuleWarning] = field(default_factory=list) @@ -232,14 +242,57 @@ def _restriction_token_warning(kind: str, tok: str, path: str) -> RuleWarning | ) +def _canonical_disjoint_violation(i: int, restriction: dict[str, Any]) -> RuleViolation | None: + """Return the violation for the first permitted token of ``restriction`` whose canonical + form is also the canonical form of one of its prohibited tokens, or ``None``. + + It canonicalises what it compares rather than trusting the term to have been + normalised, and it runs on every axis — where the fold is a no-op the check is plain + equality, which is what a server that never asked for the wire tier needs. The finding + names the canonical token, not the spellings that produced it, so the message does not + depend on whether the caller folded first. + + An element that is not a string is SKIPPED rather than coerced. Coercing it to ``""`` + would report two ill-typed elements as a collision on the empty token, which names the + wrong fault; the wire tier already refuses a non-string where the schema says string, + the same division this file applies to an empty or namespaced token. Two genuinely + empty strings still collide, which is what the Go oracle answers. + """ + permitted = _as_list(restriction.get("permitted")) + prohibited = _as_list(restriction.get("prohibited")) + if not permitted or not prohibited: + return None + kind = _str(restriction.get("kind")) + banned = { + canonical_restriction_token(kind, tok) for tok in prohibited if isinstance(tok, str) + } + for j, tok in enumerate(permitted): + if not isinstance(tok, str): + continue + canon = canonical_restriction_token(kind, tok) + if canon not in banned: + continue + return RuleViolation( + rule=RULE_RESTRICTION_CANONICAL_DISJOINT, + path=f"restrictions[{i}].permitted[{j}]", + token=canon, + message=f'restriction token "{canon}" is both permitted and prohibited after canonicalisation', + ) + return None + + def validate_license_term(term: dict[str, Any]) -> TermVerdict: - """Run the ingest-tier checks over one already-canonical term. + """Run the ingest-tier checks over one term, in a fixed order. + + A bare ``pricing.unit`` that is not registered, then the first offending + ``quotas[].metric``, then the first restriction whose permitted and prohibited + lists name one token once canonicalised. An accepted term carries one warning + per unregistered bare restriction token — restriction order, permitted before + prohibited — then one per ``OBLIGATION_KIND_OTHER`` obligation without detail. - A bare ``pricing.unit`` or ``quotas[].metric`` that is not registered is a - violation (pricing first, the first offending quota wins); an accepted term - carries one warning per unregistered bare restriction token — restriction - order, permitted before prohibited — then one per ``OBLIGATION_KIND_OTHER`` - obligation without detail. The wire tier is not re-run here. + Every check but the disjointness one reads the term as already canonical. The + wire tier is not re-run here; disjointness is the one property both tiers assert, + over different values, so a term the boundary clears can still fail here. """ pricing = _as_obj(term.get("pricing")) or {} unit = _str(pricing.get("unit")) @@ -263,6 +316,13 @@ def validate_license_term(term: dict[str, Any]) -> TermVerdict: message=f'quota metric "{metric}" is not a registered quota token', ) ) + for i, r in enumerate(_as_list(term.get("restrictions"))): + restriction = _as_obj(r) + if restriction is None: + continue + violation = _canonical_disjoint_violation(i, restriction) + if violation is not None: + return TermVerdict(violation=violation) warnings: list[RuleWarning] = [] for i, r in enumerate(_as_list(term.get("restrictions"))): restriction = _as_obj(r) diff --git a/sdk/python/tests/test_licenseterm_non_object.py b/sdk/python/tests/test_licenseterm_non_object.py index 31cf6f84..eb783212 100644 --- a/sdk/python/tests/test_licenseterm_non_object.py +++ b/sdk/python/tests/test_licenseterm_non_object.py @@ -1,8 +1,10 @@ -"""A value that is not an entry gets a VERDICT, not an exception. +"""An ill-typed value gets a VERDICT, not an exception, and is not mistaken for a +well-typed one. -This one has no shared vector and cannot get one from the Go oracle: the Go face takes -a ``*rampv1.ResourceEntry``, so "a string where an entry should be" is not a value it -can construct, let alone marshal into a corpus. The behaviour is a property of the two +Neither case here has a shared vector, and neither can get one from the Go oracle: the +Go face takes a ``*rampv1.ResourceEntry``, so "a string where an entry should be" is not +a value it can construct, and ``Restriction.permitted`` is ``[]string``, so "a number +where a token should be" is not either — let alone marshal into a corpus. The behaviour is a property of the two JSON ports, whose faces take whatever a caller parsed. It is reachable the ordinary way. The pre-check exists for feeds, a JSONL line is parsed @@ -26,7 +28,11 @@ import pytest -from ramp_sdk.licenseterm import validate_resource_entry +from ramp_sdk.licenseterm import ( + RULE_RESTRICTION_CANONICAL_DISJOINT, + validate_license_term, + validate_resource_entry, +) @pytest.mark.parametrize( @@ -43,3 +49,39 @@ def test_refuses_a_non_object_with_a_verdict(label: str, value: Any) -> None: def test_a_real_entry_is_still_valid() -> None: """The guard must not have swallowed the walk it stands in front of.""" assert validate_resource_entry({"domain": "e.co", "path": "/a"}).ok + + +# The same division one level down, on the restriction token lists. +# +# Every other check in the module skips an empty token, on the reasoning that an empty +# string is the wire tier's business. The canonical-disjointness check cannot skip it — +# two empty strings really are one token named on both sides, which is what the Go +# oracle answers — so it has to tell an EMPTY token from an ILL-TYPED one. Coercing the +# latter to "" would report a collision on the empty token and name the wrong fault, +# while the real fault, "this element is not a string", is what the wire tier already +# reports beside it. +def _term(permitted: list[Any], prohibited: list[Any]) -> dict[str, Any]: + return { + "restrictions": [ + {"kind": "RESTRICTION_KIND_FUNCTION", "permitted": permitted, "prohibited": prohibited} + ] + } + + +def test_ill_typed_elements_are_not_a_collision_on_the_empty_token() -> None: + assert validate_license_term(_term(["ai-train", 5], ["search", None])).violation is None + + +def test_a_token_named_on_both_sides_is_still_refused() -> None: + violation = validate_license_term(_term(["scrape", 5], ["crawl", None])).violation + assert violation is not None + assert violation.rule == RULE_RESTRICTION_CANONICAL_DISJOINT + assert violation.token == "crawl" + + +def test_the_empty_token_still_collides() -> None: + """Two empty strings are one token named on both sides; the Go oracle refuses them.""" + violation = validate_license_term(_term([""], [""])).violation + assert violation is not None + assert violation.rule == RULE_RESTRICTION_CANONICAL_DISJOINT + assert violation.token == "" diff --git a/sdk/python/tests/test_licenseterm_parity.py b/sdk/python/tests/test_licenseterm_parity.py index 641ad837..7dc2081c 100644 --- a/sdk/python/tests/test_licenseterm_parity.py +++ b/sdk/python/tests/test_licenseterm_parity.py @@ -37,7 +37,18 @@ _KNOWN = _VECTORS["known"] _VALIDATE = _VECTORS["validate"] _ENTRY = _VECTORS["entry"] -_TERM_RULES = {RULE_PRICING_UNIT_REGISTERED, RULE_QUOTA_METRIC_REGISTERED} +# The ingest-tier reject ids, read out of the corpus's own per-term list — an +# ingest-tier reject IS an id that list can carry as a violation. Reading them beats +# listing them here, the same trade the cross-field set below already makes: a +# classification written down in four places is one that can disagree with itself. +_TERM_RULES = {case["violation"]["rule"] for case in _VALIDATE if case["violation"] is not None} + +# Guard the guard. A derivation that stopped reading the column would leave an empty +# set and quietly move every term reject into the structural bucket, where the entry +# assertions would still pass. Both long-standing ids are reachable from that list, so +# requiring them pins that it is still being read. +for _id in (RULE_PRICING_UNIT_REGISTERED, RULE_QUOTA_METRIC_REGISTERED): + assert _id in _TERM_RULES, f"the per-term list carries no {_id} — the column this classification is derived from has moved" # The registered cross-field rule ids, read from the generated cross-field corpus — # corpusgen emits one mutant per message-level CEL rule, so its ``rules`` are the diff --git a/sdk/ts/src/licenseterm.ts b/sdk/ts/src/licenseterm.ts index 7990f6ac..b24d7d2d 100644 --- a/sdk/ts/src/licenseterm.ts +++ b/sdk/ts/src/licenseterm.ts @@ -7,7 +7,8 @@ // applied to the entry as received. The ingest tier runs over the CANONICALISED // terms: restriction tokens are folded and alias-resolved to their registered // form, then a bare Pricing.unit or Quota.metric that is not a registered token -// is rejected, while an unregistered restriction token and an +// is rejected, as is a restriction whose permitted and prohibited lists name one +// token once folded, while an unregistered restriction token and an // OBLIGATION_KIND_OTHER obligation without detail are accepted with a warning // that reaches PushResourcesResponse.warnings. The Exchange's own run is the // deciding one; a client-side verdict is advice about what that run will say. @@ -34,6 +35,14 @@ import { crossFieldRuleIds } from "./crossfield.ts"; export const RULE_PRICING_UNIT_REGISTERED = "pricing.unit.registered"; /** Rejects a bare Quota.metric that is not a registered quota token. */ export const RULE_QUOTA_METRIC_REGISTERED = "quota.metric.registered"; +/** + * Rejects a restriction whose permitted and prohibited lists name the same token + * once both are canonicalised. The wire tier's rule compares the tokens AS + * WRITTEN, so two accepted spellings of one token — an alias beside its + * registered form, or two spellings differing only in ASCII case — pass it and + * collide only after the fold. + */ +export const RULE_RESTRICTION_CANONICAL_DISJOINT = "restriction.canonical_disjoint"; /** Warns about a bare restriction token not registered on its axis; the term is accepted. */ export const RULE_RESTRICTION_TOKEN_REGISTERED = "restriction.token.registered"; /** Warns about an OBLIGATION_KIND_OTHER obligation carrying no detail. */ @@ -61,7 +70,11 @@ export interface RuleWarning { message: string; } -/** What validateLicenseTerm reports for one already-canonical term. */ +/** + * What validateLicenseTerm reports for one term. Every check but the disjointness + * one reads the term as already canonical; that one folds what it compares, so it + * is correct on a term as authored too. + */ export interface TermVerdict { violation: RuleViolation | null; warnings: RuleWarning[]; @@ -235,12 +248,56 @@ function restrictionTokenWarning(kind: string, tok: string, path: string): RuleW } /** - * validateLicenseTerm runs the ingest-tier checks over one already-canonical - * term: a bare Pricing.unit or Quota.metric that is not registered is a - * violation (pricing first, first offending quota wins), and an accepted term - * carries one warning per unregistered bare restriction token — restriction - * order, permitted before prohibited — then one per OBLIGATION_KIND_OTHER - * obligation without detail. The wire tier is not re-run here. + * canonicalDisjointViolation returns the violation for the first permitted token + * of r whose canonical form is also the canonical form of one of its prohibited + * tokens, or undefined when the two lists name no token in common. It + * canonicalises what it compares rather than trusting the term to have been + * normalised, and it runs on every axis — where the fold is a no-op the check is + * plain equality, which is what a server that never asked for the wire tier + * needs. The finding names the canonical token, not the spellings that produced + * it, so the message does not depend on whether the caller folded first. + * + * An element that is not a string is SKIPPED rather than coerced. Coercing it to "" + * would report two ill-typed elements as a collision on the empty token, which names + * the wrong fault; the wire tier already refuses a non-string where the schema says + * string, the same division this file applies to an empty or namespaced token. Two + * genuinely empty strings still collide, which is what the Go oracle answers. + */ +function canonicalDisjointViolation(i: number, r: Obj): RuleViolation | undefined { + const permitted = asArr(r["permitted"]); + const prohibited = asArr(r["prohibited"]); + if (permitted.length === 0 || prohibited.length === 0) return undefined; + const kind = str(r["kind"]); + const banned = new Set( + prohibited.filter((t) => typeof t === "string").map((t) => canonicalRestrictionToken(kind, t)), + ); + for (let j = 0; j < permitted.length; j++) { + const tok = permitted[j]; + if (typeof tok !== "string") continue; + const canon = canonicalRestrictionToken(kind, tok); + if (!banned.has(canon)) continue; + return { + rule: RULE_RESTRICTION_CANONICAL_DISJOINT, + path: `restrictions[${i}].permitted[${j}]`, + token: canon, + message: `restriction token "${canon}" is both permitted and prohibited after canonicalisation`, + }; + } + return undefined; +} + +/** + * validateLicenseTerm runs the ingest-tier checks over one term, in a fixed + * order: a bare Pricing.unit that is not registered, then the first offending + * quota metric, then the first restriction whose permitted and prohibited lists + * name one token once canonicalised. An accepted term carries one warning per + * unregistered bare restriction token — restriction order, permitted before + * prohibited — then one per OBLIGATION_KIND_OTHER obligation without detail. + * + * Every check but the disjointness one reads the term as already canonical. The + * wire tier is not re-run here; disjointness is the one property both tiers + * assert, over different values, so a term the boundary clears can still fail + * here. */ export function validateLicenseTerm(term: Obj): TermVerdict { const unit = str(asObj(term["pricing"])?.["unit"]); @@ -270,8 +327,14 @@ export function validateLicenseTerm(term: Obj): TermVerdict { }; } } - const warnings: RuleWarning[] = []; const restrictions = asArr(term["restrictions"]); + for (let i = 0; i < restrictions.length; i++) { + const r = asObj(restrictions[i]); + if (!r) continue; + const violation = canonicalDisjointViolation(i, r); + if (violation) return { violation, warnings: [] }; + } + const warnings: RuleWarning[] = []; for (let i = 0; i < restrictions.length; i++) { const r = asObj(restrictions[i]); if (!r) continue; diff --git a/sdk/ts/tests/licenseterm-non-object.test.ts b/sdk/ts/tests/licenseterm-non-object.test.ts index 3a94d085..c7ecb983 100644 --- a/sdk/ts/tests/licenseterm-non-object.test.ts +++ b/sdk/ts/tests/licenseterm-non-object.test.ts @@ -1,8 +1,10 @@ -// A value that is not an entry gets a VERDICT, not an exception. +// An ill-typed value gets a VERDICT, not an exception, and is not mistaken for a +// well-typed one. // -// This one has no shared vector and cannot get one from the Go oracle: the Go face -// takes a *rampv1.ResourceEntry, so "a string where an entry should be" is not a -// value it can construct, let alone marshal into a corpus. The behaviour is a +// Neither case here has a shared vector, and neither can get one from the Go oracle: +// the Go face takes a *rampv1.ResourceEntry, so "a string where an entry should be" is +// not a value it can construct, and Restriction.permitted is []string, so "a number +// where a token should be" is not either — let alone marshal into a corpus. The behaviour is a // property of the two JSON ports, whose faces take whatever a caller parsed. // // It is reachable the ordinary way. The pre-check exists for feeds, a JSONL line is @@ -20,7 +22,11 @@ // Mirrors sdk/python/tests/test_licenseterm_non_object.py. import { describe, expect, it } from "vitest"; -import { validateResourceEntry } from "../src/licenseterm.ts"; +import { + RULE_RESTRICTION_CANONICAL_DISJOINT, + validateLicenseTerm, + validateResourceEntry, +} from "../src/licenseterm.ts"; describe("validateResourceEntry on a value that is not an object", () => { for (const [label, value] of [ @@ -43,3 +49,38 @@ describe("validateResourceEntry on a value that is not an object", () => { expect(verdict.ok).toBe(true); }); }); + +// The same division one level down, on the restriction token lists. +// +// Every other check in the file skips an empty token, on the reasoning that an empty +// string is the wire tier's business. The canonical-disjointness check cannot skip it — +// two empty strings really are one token named on both sides, which is what the Go +// oracle answers — so it has to tell an EMPTY token from an ILL-TYPED one. Coercing +// the latter to "" would report a collision on the empty token and name the wrong +// fault, while the real fault, "this element is not a string", is what the wire tier +// already reports beside it. +describe("canonical disjointness on ill-typed restriction tokens", () => { + const term = (permitted: unknown[], prohibited: unknown[]) => ({ + restrictions: [{ kind: "RESTRICTION_KIND_FUNCTION", permitted, prohibited }], + }); + + it("does not read two ill-typed elements as a collision on the empty token", () => { + // biome-ignore lint/suspicious/noExplicitAny: the point is the untyped caller + const verdict = validateLicenseTerm(term(["ai-train", 5], ["search", null]) as any); + expect(verdict.violation).toBeNull(); + }); + + it("still refuses a token genuinely named on both sides", () => { + // biome-ignore lint/suspicious/noExplicitAny: the point is the untyped caller + const verdict = validateLicenseTerm(term(["scrape", 5], ["crawl", null]) as any); + expect(verdict.violation?.rule).toBe(RULE_RESTRICTION_CANONICAL_DISJOINT); + expect(verdict.violation?.token).toBe("crawl"); + }); + + it("keeps the empty token colliding, which is what the Go oracle answers", () => { + // biome-ignore lint/suspicious/noExplicitAny: the point is the untyped caller + const verdict = validateLicenseTerm(term([""], [""]) as any); + expect(verdict.violation?.rule).toBe(RULE_RESTRICTION_CANONICAL_DISJOINT); + expect(verdict.violation?.token).toBe(""); + }); +}); diff --git a/sdk/ts/tests/licenseterm.parity.test.ts b/sdk/ts/tests/licenseterm.parity.test.ts index e9221170..2840323f 100644 --- a/sdk/ts/tests/licenseterm.parity.test.ts +++ b/sdk/ts/tests/licenseterm.parity.test.ts @@ -43,7 +43,23 @@ type Vectors = { }; const vectors = vectorsFile as Vectors; -const TERM_RULES = new Set([RULE_PRICING_UNIT_REGISTERED, RULE_QUOTA_METRIC_REGISTERED]); +// The ingest-tier reject ids, read out of the corpus's own per-term list — an +// ingest-tier reject IS an id that list can carry as a violation. Reading them beats +// listing them here, the same trade the cross-field set below already makes: a +// classification written down in four places is one that can disagree with itself. +const TERM_RULES = new Set( + vectors.validate.flatMap((v) => (v.violation ? [v.violation.rule] : [])), +); + +// Guard the guard. A derivation that stopped reading the column would leave an empty +// set and quietly move every term reject into the structural bucket, where the entry +// assertions would still pass. Both long-standing ids are reachable from that list, +// so requiring them pins that it is still being read. +for (const id of [RULE_PRICING_UNIT_REGISTERED, RULE_QUOTA_METRIC_REGISTERED]) { + if (!TERM_RULES.has(id)) { + throw new Error(`the per-term list carries no ${id} — the column this classification is derived from has moved`); + } +} // The registered cross-field rule ids, read from the generated cross-field corpus — // corpusgen emits one mutant per message-level CEL rule, so its `rules` are the diff --git a/website/src/content/docs/components/content-ingestion/sources.mdx b/website/src/content/docs/components/content-ingestion/sources.mdx index aa0d0fd6..cc89d82c 100644 --- a/website/src/content/docs/components/content-ingestion/sources.mdx +++ b/website/src/content/docs/components/content-ingestion/sources.mdx @@ -298,7 +298,7 @@ async with CatalogClient(ClientConfig(base_url=catalog_endpoint, signer=contribu }) ``` -`ver` is stamped when the caller leaves it empty; no idempotency key is minted, because the catalog messages carry none — a push is an upsert. A request whose `exchange` is empty or not a bare domain — the shape the wire rule admits, which is narrower than a usable host — is refused before it is signed. The pre-check is advice about what the Exchange will say; the Exchange re-runs both tiers on every push and its verdict decides. The JSONL feed, a sitemap, an RSL file, a crawl or a CMS plugin are inputs that converge on the same `ResourceEntry`; the SDK owns the shape, its validation and the push, never the source. +`ver` is stamped when the caller leaves it empty; no idempotency key is minted, because the catalog messages carry none — a push is an upsert. A request whose `exchange` is empty or not a bare domain — the shape the wire rule admits, which is narrower than a usable host — is refused before it is signed. The pre-check is advice about what the Exchange will say; the Exchange re-runs both tiers on every push and its verdict decides — including the ingest tier's check that a term is still disjoint after its tokens are canonicalised, which is the one refusal a feed written against the registry's alias spellings is most likely to hit. The JSONL feed, a sitemap, an RSL file, a crawl or a CMS plugin are inputs that converge on the same `ResourceEntry`; the SDK owns the shape, its validation and the push, never the source. ### Third-Party Content Intelligence and Verification Vendors (v1.0) diff --git a/website/src/content/docs/components/exchange/request-flows.mdx b/website/src/content/docs/components/exchange/request-flows.mdx index e039b62d..618e944d 100644 --- a/website/src/content/docs/components/exchange/request-flows.mdx +++ b/website/src/content/docs/components/exchange/request-flows.mdx @@ -778,7 +778,7 @@ func (h *ExchangeHandler) ReportUsage( ## Attestation Verification at Catalog Push -Before the handler below runs, the wire tier has already refused a malformed push: the `ResourceEntry` envelope rules (bare-host `domain`, absolute `path`, bounded fields, at most 32 terms, and at most 256 entries in the submission) and the `LicenseTerm` rules (including at most 8 restrictions, and at most 64 quotas or obligations, per term) are protovalidate rules applied at the boundary. That tier is something a deployment turns on: an Exchange built on the Go SDK mounts its handler with `connectserver.WithValidation(connect.ValidationStrict)`, which is what composes protovalidate onto it — the option is not the default, and the reference implementation passes it on every mount. The handler then canonicalises the terms and applies registry membership — the ingest tier — before the attestation checks. Both tiers ship in the SDK as a publisher pre-check, so a conforming feed rarely reaches this flow with a refusable entry. +Before the handler below runs, the wire tier has already refused a malformed push: the `ResourceEntry` envelope rules (bare-host `domain`, absolute `path`, bounded fields, at most 32 terms, and at most 256 entries in the submission) and the `LicenseTerm` rules (including at most 8 restrictions, and at most 64 quotas or obligations, per term) are protovalidate rules applied at the boundary. That tier is something a deployment turns on: an Exchange built on the Go SDK mounts its handler with `connectserver.WithValidation(connect.ValidationStrict)`, which is what composes protovalidate onto it — the option is not the default, and the reference implementation passes it on every mount. The handler then canonicalises the terms and applies registry membership — the ingest tier — before the attestation checks. The ingest tier also re-asserts that no token is both permitted and prohibited (`restriction.canonical_disjoint`), because the boundary rule compares the tokens as written and two spellings of one token become one token only once folded. That is the check a mount without the wire tier still gets, and the one that keeps a self-contradictory term out of the catalog — a stored term that violates the contract would fail validation again on the way out, on every offer that carries it. Both tiers ship in the SDK as a publisher pre-check, so a conforming feed rarely reaches this flow with a refusable entry. When `CatalogService.PushResources` includes attestations, the Exchange performs additional verification before accepting entries into the resource catalog. diff --git a/website/src/content/docs/getting-started/publisher-onboarding.mdx b/website/src/content/docs/getting-started/publisher-onboarding.mdx index 76196b22..1574152c 100644 --- a/website/src/content/docs/getting-started/publisher-onboarding.mdx +++ b/website/src/content/docs/getting-started/publisher-onboarding.mdx @@ -168,7 +168,7 @@ The `catalog_contributors` field authorizes third parties to push attestations a Whatever your source material — a JSONL feed, a sitemap, an RSL file, a crawl, a CMS plugin — it converges on one `ResourceEntry` per resource, and the SDK owns the rest: the shape, the Exchange's own validation run for you to call before you push, and the push itself. 1. Build a `ResourceEntry` per resource: `domain` (your bare host), `path` (an absolute URL path), the metadata you have, and one or more `terms` — every term carries a `pricing` (`model: FREE` is stated, never implied). -2. Pre-check it with `validate_resource_entry` (Python), `validateResourceEntry` (TypeScript) or `helpers.ValidateResourceEntry` (Go). The verdict lists every rule the Exchange would refuse the entry on, with its field path, and the warnings the accepted terms would carry — the exact strings that come back in `PushResourcesResponse.warnings`. Token spellings and aliases (`generative-ai` → `ai-input`, `de` → `DE`) are canonicalised for you. +2. Pre-check it with `validate_resource_entry` (Python), `validateResourceEntry` (TypeScript) or `helpers.ValidateResourceEntry` (Go). The verdict lists every rule the Exchange would refuse the entry on, with its field path, and the warnings the accepted terms would carry — the exact strings that come back in `PushResourcesResponse.warnings`. Token spellings and aliases (`generative-ai` → `ai-input`, `de` → `DE`) are canonicalised for you — which is also why a term must stay disjoint **after** canonicalisation: `permitted: ["scrape"]` with `prohibited: ["crawl"]` names one token on both sides and is refused. 3. Push with the catalog client — `ramp_sdk.client.CatalogClient`, `createCatalogClient`, `connect.NewCatalogClient` — built against the CatalogService address you have configured for that Exchange, and your contributor signing key. The address is configuration, the way the agent client's home Exchange is: nobody but you named it, so the client dials only what you gave it. An Exchange MAY advertise the address as `catalog_endpoint` in its `/.well-known/ramp.json`, and a deployment that reads it from there MUST itself check the binding that field states — same host and port as the manifest, or a subdomain of that host on that port, no userinfo — before dialling it; a manifest naming an unrelated host would otherwise redirect a signed push to a party the signature never covered. Set `exchange` to the Exchange's bare domain, `tenant_id` to your tenant and `caller_id` to your domain; the client stamps `ver`, signs the request with your key and refuses before signing a request whose `exchange` is not a bare domain — the shape the wire rule admits, which is narrower than a usable host. A push is all-or-nothing: an entry that fails a hard rule refuses the whole submission with the offending entry named, and nothing is persisted. Warnings never block. The pre-check is advice about what the Exchange will say; the Exchange re-runs both tiers on every push and its verdict decides. Worked examples in all three languages are in [Source 7: CatalogService API Push](/components/content-ingestion/sources/#source-7-catalogservice-api-push); the feed format is in [JSONL Ingestion Feed](/protocol/jsonl-ingestion/). diff --git a/website/src/content/docs/protocol/jsonl-ingestion.mdx b/website/src/content/docs/protocol/jsonl-ingestion.mdx index 1c686116..0acba815 100644 --- a/website/src/content/docs/protocol/jsonl-ingestion.mdx +++ b/website/src/content/docs/protocol/jsonl-ingestion.mdx @@ -65,10 +65,11 @@ Rules (enforced by protovalidate at the RPC boundary): - `per_unit` REQUIRES `pricing.unit`; `flat` / `free` carry none. - A `reference_only` term MUST carry `license.uri`; any `license.uri` REQUIRES `license.uri_digest`. - Tokens are canonical registry values or `vendor:namespaced`. Restriction tokens are canonicalised at ingest (whitespace trimmed, ASCII case folded, aliases such as `generative-ai` → `ai-input` resolved) and an unregistered bare token is a **warning**, not a reject; a bare unregistered `pricing.unit` or `quotas[].metric` IS a reject. +- A token cannot be both permitted and prohibited on one axis. This is checked at the boundary over the tokens as written, and again at ingest over the canonicalised tokens (`restriction.canonical_disjoint`) — `permitted: ["scrape"]` with `prohibited: ["crawl"]` names one token twice, and only the second check can see it. ## Pre-validate with the SDK -Both tiers the Exchange applies ship in the SDK, so a feed can be checked before anything is signed — `ramp-ingest` runs them for you, and a program that builds its own entries calls them itself; the catalog client does not. `validate_resource_entry` (Python), `validateResourceEntry` (TypeScript) and `helpers.ValidateResourceEntry` (Go) run the wire rules over the entry as given, then canonicalise a copy of its terms and run registry membership and the coherence lints; the verdict lists every violation with its rule id and field path, and the warnings the accepted terms would carry — the exact strings the Exchange puts in `PushResourcesResponse.warnings`. The normalising faces produce the canonical form the Exchange will store — `Generative-AI` becomes `ai-input`, `de` becomes `DE` — and they differ by language on purpose. `normalize_resource_entry` (Python) and `normalizeResourceEntry` (TypeScript) take a proto-JSON object and **return a new one**, leaving the input untouched. `helpers.NormalizeResourceEntry` (Go) takes a `*rampv1.ResourceEntry` and **rewrites it in place**, returning nothing: the Exchange normalises the entry it is about to persist, and an in-place face is what lets the canonical tokens reach the stored row and the offer projection without a second copy. Each is its language's idiom; the shared conformance corpus pins the output, which is the part that must agree. +Both tiers the Exchange applies ship in the SDK, so a feed can be checked before anything is signed — `ramp-ingest` runs them for you, and a program that builds its own entries calls them itself; the catalog client does not. `validate_resource_entry` (Python), `validateResourceEntry` (TypeScript) and `helpers.ValidateResourceEntry` (Go) run the wire rules over the entry as given, then canonicalise a copy of its terms and run registry membership and the coherence lints; the verdict lists every violation with its rule id and field path — including the ones only the canonicalised form reveals — and the warnings the accepted terms would carry — the exact strings the Exchange puts in `PushResourcesResponse.warnings`. The normalising faces produce the canonical form the Exchange will store — `Generative-AI` becomes `ai-input`, `de` becomes `DE` — and they differ by language on purpose. `normalize_resource_entry` (Python) and `normalizeResourceEntry` (TypeScript) take a proto-JSON object and **return a new one**, leaving the input untouched. `helpers.NormalizeResourceEntry` (Go) takes a `*rampv1.ResourceEntry` and **rewrites it in place**, returning nothing: the Exchange normalises the entry it is about to persist, and an in-place face is what lets the canonical tokens reach the stored row and the offer projection without a second copy. Each is its language's idiom; the shared conformance corpus pins the output, which is the part that must agree. ```python from ramp_sdk import validate_resource_entry, normalize_resource_entry @@ -86,7 +87,7 @@ The check is advice about what the Exchange will say, not a substitute for it: t ## All-or-nothing -`PushResources` is atomic, at **both** tiers: if any entry fails a hard rule — the envelope rules above, a term's shape or cross-field rule, or a bare unregistered `pricing.unit` or `quotas[].metric` — the **whole submission is rejected** (`InvalidArgument`, naming the offending entries) and **nothing is persisted**. The publisher fixes and resubmits the full set. What the two tiers change is *when* the refusal happens, not what survives it: the wire tier refuses at the RPC boundary, before any entry is classified, while the ingest tier refuses after canonicalising the terms. Either way no entry of a refused submission is stored — naming the entries that failed is how the refusal tells you what to fix, not a sign that the rest went through. Membership / lint issues are returned in `PushResourcesResponse.warnings[]` and do not block. +`PushResources` is atomic, at **both** tiers: if any entry fails a hard rule — the envelope rules above, a term's shape or cross-field rule, a bare unregistered `pricing.unit` or `quotas[].metric`, or a restriction that is not disjoint once its tokens are canonicalised — the **whole submission is rejected** (`InvalidArgument`, naming the offending entries) and **nothing is persisted**. The publisher fixes and resubmits the full set. What the two tiers change is *when* the refusal happens, not what survives it: the wire tier refuses at the RPC boundary, before any entry is classified, while the ingest tier refuses after canonicalising the terms. Either way no entry of a refused submission is stored — naming the entries that failed is how the refusal tells you what to fix, not a sign that the rest went through. Membership / lint issues are returned in `PushResourcesResponse.warnings[]` and do not block. ## Extension profiles in the feed diff --git a/website/src/content/docs/protocol/licensing-terms.mdx b/website/src/content/docs/protocol/licensing-terms.mdx index 862b5bb9..dfb5f44a 100644 --- a/website/src/content/docs/protocol/licensing-terms.mdx +++ b/website/src/content/docs/protocol/licensing-terms.mdx @@ -55,6 +55,8 @@ Each restriction carries: - `prohibited[]` — tokens blocked on this axis. Takes precedence over `permitted[]`. - `advisory` — when `false` (the **default**), the restriction is **binding**: an agent that cannot evaluate every token in it (including an unknown vendor token) MUST decline the term. Set `advisory: true` to downgrade an unverifiable restriction to non-blocking. (Fail-closed by default — a forgotten flag fails safe.) +**A token cannot be both permitted and prohibited** on one axis, and that is checked twice, over two readings of the same lists. At the RPC boundary `restriction.permitted_prohibited_disjoint` compares the tokens **as written**, because it runs over the request exactly as received. At ingest, after canonicalisation, `restriction.canonical_disjoint` compares what the fold produced. The second is not a repetition: several tokens have more than one accepted spelling, so `permitted: ["scrape"]` with `prohibited: ["crawl"]` — or `["us"]` against `["US"]` — names one token on both sides while passing the first check. Both refuse the term. + **Reading a restriction**: a value is in-scope when it matches at least one `permitted[]` token AND matches none of the `prohibited[]` tokens. Multiple restrictions on the same term are AND-combined — the agent must satisfy all of them. The agent applies this itself when self-selecting a term; the Exchange does not pre-filter terms against requester attributes. **Standard function tokens** (rendered from the proto at build, not hand-listed): @@ -69,7 +71,7 @@ Each restriction carries: ::proto-vocab{axis=user-type} -Unknown tokens during ingest produce a `PushResourcesResponse.warnings[]` entry — the term is accepted but flagged. This is forward-compatible: new vocab tokens can be added to the registry and used immediately without breaking existing Exchange deployments. Tokens are canonicalised before the check — RFC 8259 whitespace trimmed, ASCII case folded, and the accepted aliases listed in each table above resolved to their registered token — so a spelling difference is never an unknown token; only a non-ASCII byte is left as it came, because a Unicode fold would turn a homograph into a registered token. +Unknown tokens during ingest produce a `PushResourcesResponse.warnings[]` entry — the term is accepted but flagged. This is forward-compatible: new vocab tokens can be added to the registry and used immediately without breaking existing Exchange deployments. Tokens are canonicalised before the check — RFC 8259 whitespace trimmed, ASCII case folded, and the accepted aliases listed in each table above resolved to their registered token — so a spelling difference is never an unknown token; only a non-ASCII byte is left as it came, because a Unicode fold would turn a homograph into a registered token. Canonicalisation is also what the disjointness check above is measured against at this tier: the fold is what turns two spellings into the one token the term may not carry on both sides. ## Quotas diff --git a/website/src/content/docs/reference/changelog.mdx b/website/src/content/docs/reference/changelog.mdx index c69f288a..9de8a564 100644 --- a/website/src/content/docs/reference/changelog.mdx +++ b/website/src/content/docs/reference/changelog.mdx @@ -29,6 +29,30 @@ item list is capped at 256 entries, the same ceiling a discovery query's `uris` list carries, and verifiers bound their own work to that cap before rendering the payload to canonical form. +**A term is now checked for permitted/prohibited disjointness a second time, over +the canonicalised tokens (`restriction.canonical_disjoint`; SDK behaviour change +plus a comment clarification, no wire change).** `restriction.permitted_prohibited_disjoint` +runs at the wire tier, over the request exactly as received, so it compares token +SPELLINGS. Ten registered aliases resolve to eight distinct restriction tokens — +`scrape` is a registered alias of `crawl`, `adapt` and `derivative` both mean +`modify`, `personal` means `individual` — and every axis also folds ASCII case. A term naming +one spelling under `permitted` and another under `prohibited` therefore passed the +boundary check and became, once the ingest tier folded it, a stored term with the +same token in both lists. + +Nothing looked at it again, and the failure surfaced elsewhere: the term rides on +offers, an Exchange validates its own responses, and so every discovery request +returning that resource answered with an internal error while the push had looked +clean. The ingest tier now asserts the same property over the canonical values, +under its own rule id, in `ValidateLicenseTerm` and its Python and TypeScript twins. +The boundary rule is unchanged. + +Disjointness is now the one property both tiers assert, deliberately: they read +different values, neither suppresses the other, and a deployment that does not mount +the wire tier still gets the second. The `Restriction` and `CatalogService` comments +say so, and a conformance guard holds the contract's statement to the rule the SDKs +run. + **Signed delivery URLs are documented as Ed25519 signed by the Exchange and verified with its published public key, not HMAC-SHA256 over a shared secret (documentation correction; no wire change).** Since the initial public snapshot diff --git a/website/src/content/docs/reference/proto-ramp.mdx b/website/src/content/docs/reference/proto-ramp.mdx index b5cfbf7c..85c99a6e 100644 --- a/website/src/content/docs/reference/proto-ramp.mdx +++ b/website/src/content/docs/reference/proto-ramp.mdx @@ -381,7 +381,7 @@ Broker returns to Agent (Step 6). Discovery-only: `offer_groups` (field 4, one ` Messages for the optional CatalogService RPC — the publisher role's surface: push, remove and refresh the catalog entries a publisher, or a contributor it authorised, supplies to an Exchange. -Every pushed entry is checked in two tiers, and both are stated so a publisher can run them before sending. The **wire tier** is protovalidate: the `ResourceEntry` envelope rules below and the `LicenseTerm` rules, applied to the request exactly as received. The **ingest tier** runs over the canonicalised terms: restriction tokens are trimmed of RFC 8259 whitespace, ASCII-case-folded (lower for `RESTRICTION_KIND_FUNCTION` and `RESTRICTION_KIND_USER_TYPE`, upper for `RESTRICTION_KIND_GEOGRAPHY`; a non-ASCII byte is never folded) and alias-resolved to their registered token; then a bare (non-namespaced) `Pricing.unit` or `Quota.metric` that is not a registered token is rejected, while an unregistered restriction token and an `OBLIGATION_KIND_OTHER` obligation without detail are accepted and reported in `PushResourcesResponse.warnings`. The SDK ships both tiers (`ValidateResourceEntry` and its Python/TypeScript twins) and a catalog client (`NewCatalogClient` / `createCatalogClient` / `CatalogClient`) in all three languages; the Exchange's own run of the checks is the deciding one. +Every pushed entry is checked in two tiers, and both are stated so a publisher can run them before sending. The **wire tier** is protovalidate: the `ResourceEntry` envelope rules below and the `LicenseTerm` rules, applied to the request exactly as received. The **ingest tier** runs over the canonicalised terms: restriction tokens are trimmed of RFC 8259 whitespace, ASCII-case-folded (lower for `RESTRICTION_KIND_FUNCTION` and `RESTRICTION_KIND_USER_TYPE`, upper for `RESTRICTION_KIND_GEOGRAPHY`; a non-ASCII byte is never folded) and alias-resolved to their registered token; then a bare (non-namespaced) `Pricing.unit` or `Quota.metric` that is not a registered token is rejected, as is a restriction whose `permitted` and `prohibited` lists name one token once folded (`restriction.canonical_disjoint`), while an unregistered restriction token and an `OBLIGATION_KIND_OTHER` obligation without detail are accepted and reported in `PushResourcesResponse.warnings`. Disjointness is the one property both tiers assert, over different values, so a term the boundary clears can still be refused at ingest. The SDK ships both tiers (`ValidateResourceEntry` and its Python/TypeScript twins) and a catalog client (`NewCatalogClient` / `createCatalogClient` / `CatalogClient`) in all three languages; the Exchange's own run of the checks is the deciding one. ### PushResourcesRequest @@ -442,7 +442,7 @@ code is the cross-field CEL rule `id:` that enforces the rule (e.g. `license_ter - `pricing` MUST be present on **every** term, any semantics — absent Pricing is a validation error. `model = FREE` must be stated explicitly — absent Pricing is not free. - `semantics` MUST be set — `TERM_SEMANTICS_UNSPECIFIED` is rejected (the field's `enum.not_in:[0]` rule). - `REFERENCE_ONLY` requires `license.uri` to be non-empty (`license_term.reference_only.requires_uri`); a `License` with a `uri` requires a `uri_digest` (`license.digest_required_with_uri`). -- At most one `Restriction` per `kind` (`license_term.one_restriction_per_kind`); a token cannot be both permitted and prohibited (`restriction.permitted_prohibited_disjoint`). +- At most one `Restriction` per `kind` (`license_term.one_restriction_per_kind`); a token cannot be both permitted and prohibited (`restriction.permitted_prohibited_disjoint`). That rule compares the tokens **as written**, because it runs over the request as received — so two accepted spellings of one token (an alias beside its registered form, or either in another ASCII case) clear it and collide once folded. The ingest tier asserts the same property over the canonicalised tokens as `restriction.canonical_disjoint`; both refuse the term, and a term that fails both is reported by both. Both are scoped to one `Restriction`: what makes one restriction the whole of an axis is `license_term.one_restriction_per_kind`, so the per-axis reading is held by the two rules together. - `quotas` and `obligations` each carry at most 64 items, the bound every per-message list in the contract carries when no rule walks it more than once. They bound what one term can carry, not the cost of checking it. `restrictions` carries at most **8**, and like the other two this bounds the document, not the cost of checking it. Only one restriction per axis is valid and `Restriction.kind` is defined-only, so four is the longest conformant list and eight leaves room for an axis this version does not have. The tighter bound is deliberate for a second reason: this is the one list a message rule walks against itself, so the cap is also the threshold of the size test the one-per-kind rule carries, and a conformance guard holds the two equal. The disjointness rule on each element is quadratic only in that element's two token lists, both capped at 64, so its cost is bounded per restriction and linear across the list. - Unknown tokens in `restrictions[].permitted` / `prohibited` produce `PushResourcesResponse.warnings[]` but do NOT cause hard rejection (ingest-time, not CEL). Tokens are canonicalised first — RFC 8259 whitespace trimmed, ASCII case folded, and the aliases authored beside the tokens resolved (`train-ai` → `ai-train`, `tdm` → `text-and-data-mining`, `personal` → `individual`, …) — so `Generative-AI` is the registered `ai-input`, not an unknown token. - A bare (non-namespaced) `pricing.unit` or `quotas[].metric` that is not a registered token IS a hard rejection at ingest (registry membership, not CEL: a rule that re-listed the vocabulary would drift from it). A `vendor:token` value bypasses membership on every axis.