From 881ffb5e76c66bdd8503960faaf64933b8dc8a69 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 04:59:48 +0900 Subject: [PATCH 1/4] Add sourced provider endpoint variants to schema v1 --- .github/CODEOWNERS | 2 + catalogue/README.md | 12 +++++ catalogue/schemas/models.schema.json | 71 ++++++++++++++++++++++++++++ scripts/check.py | 8 ++++ tests/test_check.py | 17 +++++++ 5 files changed, 110 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..4cf5deb --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Catalogue and tooling changes require maintainer review. +* @softmarshmallow diff --git a/catalogue/README.md b/catalogue/README.md index 99b869b..3f4686a 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -100,3 +100,15 @@ From the repository root: uv run scripts/check.py # schema + invariants uv run pytest # the checker rejects what it should ``` + +## Provider endpoints + +A provider binding may list `endpoints`: exact call IDs with `input` (`text`, +`image`, `video`, `audio`, or `task`) and a first-party `source_url`. These identify +input variants, not a routing preference. The existing binding `id` remains its +primary call ID; `references` and pricing retain their existing meaning. +Endpoint IDs are unique within a binding. The list records documented variants, +not an assertion that a consumer supports every variant. + +`vendor`, including on text cards, identifies the model's lab in `labs.json`. +It is independent of a model ID's namespace and its serving provider. diff --git a/catalogue/schemas/models.schema.json b/catalogue/schemas/models.schema.json index 305e17c..687c1ed 100644 --- a/catalogue/schemas/models.schema.json +++ b/catalogue/schemas/models.schema.json @@ -84,6 +84,47 @@ "items": { "type": "string" } + }, + "label": { + "type": "string", + "minLength": 1 + }, + "vendor": { + "type": "string", + "minLength": 1, + "description": "Lab ID resolving in labs.json; independent of the serving provider." + }, + "release": { + "type": "object", + "required": ["date", "basis", "source_url"], + "additionalProperties": false, + "properties": { + "date": { + "type": ["string", "null"], + "format": "date" + }, + "basis": { + "enum": ["model", "provider_endpoint"] + }, + "source_url": { + "type": "string", + "pattern": "^https://" + } + }, + "if": { + "properties": { + "date": { + "type": "null" + } + } + }, + "then": { + "properties": { + "basis": { + "const": "provider_endpoint" + } + } + } } } } @@ -153,9 +194,21 @@ "id": { "type": "string", "minLength": 1 + }, + "endpoints": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/provider_endpoint" + } } } } + }, + "vendor": { + "type": "string", + "minLength": 1, + "description": "Lab ID resolving in labs.json; independent of the serving provider." } } }, @@ -225,6 +278,24 @@ } } } + }, + "provider_endpoint": { + "type": "object", + "additionalProperties": false, + "required": ["id", "input", "source_url"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "input": { + "enum": ["text", "image", "video", "audio", "task"] + }, + "source_url": { + "type": "string", + "pattern": "^https://" + } + } } } } diff --git a/scripts/check.py b/scripts/check.py index cac07e3..6fd604c 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -136,6 +136,14 @@ def invariant_errors(facts: dict[str, Any]) -> Iterator[str]: if not binding.get("id"): yield f"{where}/providers/{name}: binding has no id" + for name, binding in card.get("providers", {}).items(): + endpoint_ids = set() + for endpoint in binding.get("endpoints", []): + endpoint_id = endpoint.get("id") + if endpoint_id in endpoint_ids: + yield f"{where}/providers/{name}: duplicate endpoint {endpoint_id!r}" + endpoint_ids.add(endpoint_id) + for exception in sorted(KNOWN_EXCEPTIONS - used_exceptions): yield f"stale known exception {exception}: remove it from KNOWN_EXCEPTIONS" diff --git a/tests/test_check.py b/tests/test_check.py index 548a6f3..1cd8161 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -322,3 +322,20 @@ def mutate(facts): facts["$schema"] = "https://example.com/other.json" assert any("$schema" in e for e in mutated(mutate)) + + +def test_provider_endpoint_needs_source_and_input(): + def mutate(facts): + first("image.models", facts)["providers"]["fal"]["endpoints"] = [{"id": "test/route"}] + assert any("required property" in e for e in mutated(mutate)) + + +def test_provider_endpoints_cannot_repeat(): + def mutate(facts): + endpoint = {"id": "test/route", "input": "text", "source_url": "https://example.com/api"} + first("image.models", facts)["providers"]["fal"]["endpoints"] = [endpoint, endpoint.copy()] + assert any("duplicate endpoint" in e for e in mutated(mutate)) + + +def test_existing_binding_without_endpoints_remains_valid(): + assert check.check(MODELS_JSON, SCHEMA) == [] From 52a4c6f6f628c35f04132c0bb93ee78fc289e7f9 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 19:29:12 +0900 Subject: [PATCH 2/4] Describe sourced calculators and provider parameter facts --- README.md | 17 +- catalogue/README.md | 57 ++++ catalogue/schemas/models.schema.json | 468 ++++++++++++++++++++++++++- scripts/check.py | 195 ++++++++++- tests/test_check.py | 201 ++++++++++++ 5 files changed, 921 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 077908d..370659f 100644 --- a/README.md +++ b/README.md @@ -80,14 +80,15 @@ consumes a projection of it. ## What is in a catalogue entry -| Kind | Examples | -| ---------------------- | --------------------------------------------------------------------------------------------- | -| **Identity** | Canonical ID (`vendor/model`), label, vendor, family (text, image, video, audio, 3D, …) | -| **Release provenance** | First broad public release date, its basis, and the first-party source URL | -| **Capabilities** | Modalities, accepted image formats, tool calling, audio, transparent background | -| **Limits** | Context window, output limit, sizes, aspect ratios, durations, size constraints | -| **Provider bindings** | The exact ID a provider uses for the model (Vercel AI Gateway, fal, OpenRouter, Replicate, …) | -| **Published prices** | Public list prices per provider, in the provider's own billing unit | +| Kind | Examples | +| ------------------------- | --------------------------------------------------------------------------------------------- | +| **Identity** | Canonical ID (`vendor/model`), label, vendor, family (text, image, video, audio, 3D, …) | +| **Release provenance** | First broad public release date, its basis, and the first-party source URL | +| **Capabilities** | Modalities, accepted image formats, tool calling, audio, transparent background | +| **Limits** | Context window, output limit, sizes, aspect ratios, durations, size constraints | +| **Provider bindings** | The exact ID a provider uses for the model (Vercel AI Gateway, fal, OpenRouter, Replicate, …) | +| **Published prices** | Public list prices per provider, in the provider's own billing unit | +| **Published calculators** | Sourced output-token formulae, examples, and image-input tokenization rules | ## What it is not diff --git a/catalogue/README.md b/catalogue/README.md index 3f4686a..657482c 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -112,3 +112,60 @@ not an assertion that a consumer supports every variant. `vendor`, including on text cards, identifies the model's lab in `labs.json`. It is independent of a model ID's namespace and its serving provider. + +## Published calculators and serving facts + +Optional additions record the facts at the scope their source describes. A card's +`output_token_calculator` and `vision_tokenization` describe the model. A binding's +`capabilities`, `limits`, and `token_pricing` describe what its provider serves. +Endpoint `parameters` and `image_constraints` describe one exact input variant. +Do not flatten a provider's limits or price bands into intrinsic model facts. +Missing fields mean unrecorded facts, never unsupported features or zero prices. + +- **Output calculators.** `output_token_calculator` carries a first-party + `source_url`, an optional `source_js_url` for the published calculator asset, + `area_offset`, `area_divisor`, `short_axis_rounding`, `token_rounding`, and + `quality_axes` (quality name to its positive integer axis). This shape describes + the published formula `ceil(K × round(K × min(width, height) / max(width, height)) +× (area_offset + width × height) / area_divisor)`. `nearest_ties_even` rounds the + short axis; `ceil` rounds output tokens. `examples` records sourced width, + height, quality, and `output_tokens` and/or `usd`. An optional `usd_precision` + records the number of decimal places in a published rounded price. A rounded + price is evidence of that price, not a basis for inventing an exact token count. + The checker reproduces any recorded token count from the formula and rejects + duplicate size/quality examples. +- **Vision tokenization.** `vision_tokenization` is a sourced set of the rules the + vendor actually publishes: `patch_size_px`, `token_multiplier`, + `detail_patch_budgets`, `detail_token_budgets`, `detail_maximum_edges`, + `maximum_patch_count`, `maximum_tokens_per_image`, `tokens_per_image`, and + `token_rounding`, when applicable. Patch counts, token budgets, and edge limits + retain distinct meanings. `token_counts_approximate: true` preserves a source's + qualification that its token examples or budgets are approximate. Omit rules + that have not been published. +- **Capabilities and limits.** A sourced `capabilities` object may record boolean + `image_input`, `tools`, `reasoning`, `structured_output`, and `streaming`. A + provider's sourced `limits` may record positive `context_tokens` and + `max_output_tokens`. Neither object chooses a consumer's supported features. +- **Token pricing.** `token_pricing` records `unit: "usd_per_token"`, a + `source_url`, and exact decimal strings for `input`, `output`, + `input_cache_read`, and `input_cache_write` where published. Optional `tiers` + carry `meter`, an inclusive `minimum_input_tokens`, an exclusive + `maximum_input_tokens`, and `usd_per_token`. An omitted or null maximum means + no published upper bound. Bands may differ by meter and cannot overlap for the + same meter. These are provider list prices, not converted application prices. +- **Endpoint parameters.** `parameters` maps exact provider field names, including + dotted paths such as `image_size.width`, to their published `type`, `enum`, + numeric `minimum`/`maximum`/`multiple_of`, array `min_items`/`max_items`, or text + `min_length`/`max_length`. Endpoint `source_url` covers these facts; a parameter + may override `source_url` when one bound is published on a different page. + This is partial descriptive metadata, not a complete request JSON Schema. For + example, string presets and custom-size object fields may be recorded + separately. No defaults or consumer choices are included. +- **Image envelopes.** `image_constraints` keeps `step`, `min_edge`, `max_edge`, + `min_pixels`, `max_pixels`, and `aspect_ratio: { min, max }` at endpoint scope, + under that endpoint's source. Each field is optional; do not infer the rest. + +Single-provider cards and `three_d.rig_check.operation` may also carry +`endpoints`, with the same shape as a binding's endpoint variants. New sourced +objects have strict field vocabularies; pre-existing card and price shapes remain +unchanged in schema version 1. diff --git a/catalogue/schemas/models.schema.json b/catalogue/schemas/models.schema.json index 687c1ed..abc1a68 100644 --- a/catalogue/schemas/models.schema.json +++ b/catalogue/schemas/models.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://grida.co/schemas/ai-model-facts-v1.json", "title": "Canonical model facts, authoring envelope version 1", - "description": "Structural interchange schema. Generated TypeScript satisfies declarations additionally check each modality's detailed capability and pricing vocabulary.", + "description": "Structural interchange schema with sourced, additive model and serving-provider facts.", "type": "object", "additionalProperties": false, "required": [ @@ -125,6 +125,19 @@ } } } + }, + "capabilities": { + "$ref": "#/$defs/capabilities" + }, + "token_pricing": { + "$ref": "#/$defs/token_pricing" + }, + "endpoints": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/provider_endpoint" + } } } } @@ -201,6 +214,15 @@ "items": { "$ref": "#/$defs/provider_endpoint" } + }, + "capabilities": { + "$ref": "#/$defs/capabilities" + }, + "token_pricing": { + "$ref": "#/$defs/token_pricing" + }, + "limits": { + "$ref": "#/$defs/limits" } } } @@ -209,6 +231,25 @@ "type": "string", "minLength": 1, "description": "Lab ID resolving in labs.json; independent of the serving provider." + }, + "output_token_calculator": { + "$ref": "#/$defs/output_token_calculator" + }, + "vision_tokenization": { + "$ref": "#/$defs/vision_tokenization" + }, + "capabilities": { + "$ref": "#/$defs/capabilities" + }, + "token_pricing": { + "$ref": "#/$defs/token_pricing" + }, + "endpoints": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/provider_endpoint" + } } } }, @@ -293,7 +334,430 @@ }, "source_url": { "type": "string", - "pattern": "^https://" + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "capabilities": { + "$ref": "#/$defs/capabilities" + }, + "token_pricing": { + "$ref": "#/$defs/token_pricing" + }, + "parameters": { + "$ref": "#/$defs/parameters" + }, + "image_constraints": { + "$ref": "#/$defs/image_constraints" + }, + "limits": { + "$ref": "#/$defs/limits" + } + } + }, + "capabilities": { + "type": "object", + "additionalProperties": false, + "required": ["source_url"], + "minProperties": 2, + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "image_input": { + "type": "boolean" + }, + "tools": { + "type": "boolean" + }, + "reasoning": { + "type": "boolean" + }, + "structured_output": { + "type": "boolean" + }, + "streaming": { + "type": "boolean" + } + } + }, + "output_token_calculator": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_url", + "area_offset", + "area_divisor", + "short_axis_rounding", + "token_rounding", + "quality_axes" + ], + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "area_offset": { + "type": "integer", + "minimum": 0 + }, + "area_divisor": { + "type": "integer", + "minimum": 1 + }, + "short_axis_rounding": { + "enum": ["nearest_ties_even"] + }, + "token_rounding": { + "enum": ["ceil"] + }, + "quality_axes": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$" + }, + "additionalProperties": { + "type": "integer", + "minimum": 1 + } + }, + "examples": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/output_token_example" + } + }, + "source_js_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + } + } + }, + "output_token_example": { + "type": "object", + "additionalProperties": false, + "required": ["width", "height", "quality", "source_url"], + "anyOf": [ + { + "required": ["output_tokens"] + }, + { + "required": ["usd"] + } + ], + "dependentRequired": { + "usd_precision": ["usd"] + }, + "properties": { + "width": { + "type": "integer", + "minimum": 1 + }, + "height": { + "type": "integer", + "minimum": 1 + }, + "quality": { + "type": "string", + "minLength": 1 + }, + "output_tokens": { + "type": "integer", + "minimum": 1 + }, + "usd": { + "type": "number", + "minimum": 0 + }, + "usd_precision": { + "type": "integer", + "minimum": 0, + "maximum": 12 + }, + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + } + } + }, + "vision_tokenization": { + "type": "object", + "additionalProperties": false, + "required": ["source_url"], + "minProperties": 2, + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "patch_size_px": { + "type": "integer", + "minimum": 1 + }, + "token_multiplier": { + "type": "number", + "exclusiveMinimum": 0 + }, + "detail_patch_budgets": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$" + }, + "additionalProperties": { + "type": "integer", + "minimum": 1 + } + }, + "detail_token_budgets": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$" + }, + "additionalProperties": { + "type": "integer", + "minimum": 1 + } + }, + "maximum_patch_count": { + "type": "integer", + "minimum": 1 + }, + "maximum_tokens_per_image": { + "type": "integer", + "minimum": 1 + }, + "tokens_per_image": { + "type": "integer", + "minimum": 1 + }, + "detail_maximum_edges": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$" + }, + "additionalProperties": { + "type": "integer", + "minimum": 1 + } + }, + "token_rounding": { + "enum": ["ceil"] + }, + "token_counts_approximate": { + "type": "boolean" + } + } + }, + "token_pricing": { + "type": "object", + "additionalProperties": false, + "required": ["source_url", "unit"], + "anyOf": [ + { + "required": ["input"] + }, + { + "required": ["output"] + }, + { + "required": ["input_cache_read"] + }, + { + "required": ["input_cache_write"] + }, + { + "required": ["tiers"] + } + ], + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "unit": { + "const": "usd_per_token" + }, + "input": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." + }, + "output": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." + }, + "input_cache_read": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." + }, + "input_cache_write": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." + }, + "tiers": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/token_price_tier" + } + } + } + }, + "token_price_tier": { + "type": "object", + "additionalProperties": false, + "required": ["meter", "minimum_input_tokens", "usd_per_token"], + "properties": { + "meter": { + "enum": ["input", "output", "input_cache_read", "input_cache_write"] + }, + "minimum_input_tokens": { + "type": "integer", + "minimum": 0, + "description": "Inclusive lower input-token bound." + }, + "maximum_input_tokens": { + "type": ["integer", "null"], + "minimum": 1, + "description": "Exclusive upper input-token bound; null or omission means no published upper bound." + }, + "usd_per_token": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." + } + } + }, + "parameter": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "type": { + "enum": ["string", "number", "integer", "boolean", "array", "object", "null"] + }, + "enum": { + "type": "array", + "minItems": 1, + "uniqueItems": true + }, + "minimum": { + "type": "number" + }, + "maximum": { + "type": "number" + }, + "multiple_of": { + "type": "number", + "exclusiveMinimum": 0 + }, + "min_items": { + "type": "integer", + "minimum": 0 + }, + "max_items": { + "type": "integer", + "minimum": 0 + }, + "min_length": { + "type": "integer", + "minimum": 0 + }, + "max_length": { + "type": "integer", + "minimum": 0 + }, + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + } + } + }, + "parameters": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*(?:\\.[a-zA-Z][a-zA-Z0-9_]*)*$" + }, + "additionalProperties": { + "$ref": "#/$defs/parameter" + } + }, + "image_constraints": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "step": { + "type": "integer", + "minimum": 1 + }, + "min_edge": { + "type": "integer", + "minimum": 1 + }, + "max_edge": { + "type": "integer", + "minimum": 1 + }, + "min_pixels": { + "type": "integer", + "minimum": 1 + }, + "max_pixels": { + "type": "integer", + "minimum": 1 + }, + "aspect_ratio": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "min": { + "type": "number", + "exclusiveMinimum": 0 + }, + "max": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + } + }, + "limits": { + "type": "object", + "additionalProperties": false, + "required": ["source_url"], + "minProperties": 2, + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "context_tokens": { + "type": "integer", + "minimum": 1 + }, + "max_output_tokens": { + "type": "integer", + "minimum": 1 } } } diff --git a/scripts/check.py b/scripts/check.py index 6fd604c..ea33bc2 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -22,9 +22,12 @@ import datetime import json +import math import sys +from fractions import Fraction from pathlib import Path from typing import Any, Iterator +from urllib.parse import urlsplit from jsonschema import Draft202012Validator @@ -87,6 +90,188 @@ def is_calendar_date(value: str) -> bool: return False +SOURCED_OBJECTS = ( + "output_token_calculator", + "vision_tokenization", + "capabilities", + "token_pricing", + "limits", +) + + +def fact_objects(facts: dict[str, Any]) -> Iterator[tuple[str, dict[str, Any]]]: + """Visit cards, provider bindings, and endpoint variants without merging scopes.""" + for section, table in facts.items(): + if not isinstance(table, dict): + continue + entries = ( + [(section, table)] if section in NON_TABLE_SECTIONS else [ + (f"{section}/{key}", card) for key, card in table.items() + ] + ) + for where, card in entries: + if not isinstance(card, dict): + continue + yield where, card + endpoints = card.get("endpoints", []) + if isinstance(endpoints, list): + for index, endpoint in enumerate(endpoints): + if isinstance(endpoint, dict): + yield f"{where}/endpoints/{index}", endpoint + providers = card.get("providers", {}) + if not isinstance(providers, dict): + continue + for name, binding in providers.items(): + if not isinstance(binding, dict): + continue + binding_where = f"{where}/providers/{name}" + yield binding_where, binding + endpoints = binding.get("endpoints", []) + if not isinstance(endpoints, list): + continue + for index, endpoint in enumerate(endpoints): + if isinstance(endpoint, dict): + yield f"{binding_where}/endpoints/{index}", endpoint + + +def source_url_errors(value: Any, where: str) -> Iterator[str]: + if not isinstance(value, str): + return + try: + url = urlsplit(value) + valid = ( + url.scheme == "https" and bool(url.hostname) + and url.username is None and url.password is None + ) + # Accessing port also catches malformed authorities. + url.port + except ValueError: + valid = False + if not valid: + yield f"{where}: source_url must be an absolute HTTPS URL without credentials" + + +def numeric_range_errors(value: dict[str, Any], where: str) -> Iterator[str]: + for lower, upper in [ + ("minimum", "maximum"), ("min_items", "max_items"), + ("min_length", "max_length"), ("min_edge", "max_edge"), + ("min_pixels", "max_pixels"), ("min", "max"), + ]: + lo, hi = value.get(lower), value.get(upper) + if isinstance(lo, (int, float)) and isinstance(hi, (int, float)) and lo > hi: + yield f"{where}: {lower} exceeds {upper}" + for key, number in value.items(): + if isinstance(number, float) and not math.isfinite(number): + yield f"{where}/{key}: number must be finite" + + +def calculator_errors(calculator: dict[str, Any], where: str) -> Iterator[str]: + axes = calculator.get("quality_axes", {}) + axes = axes if isinstance(axes, dict) else {} + seen = set() + examples = calculator.get("examples", []) + if not isinstance(examples, list): + return + for index, example in enumerate(examples): + if not isinstance(example, dict): + continue + example_where = f"{where}/examples/{index}" + yield from source_url_errors(example.get("source_url"), example_where) + key = (example.get("width"), example.get("height"), example.get("quality")) + # Invalid compound values are already schema errors; avoid hashing them. + if not all(isinstance(part, (int, str)) for part in key): + continue + if key in seen: + yield f"{example_where}: duplicate size/quality example" + seen.add(key) + if example.get("quality") not in axes: + yield f"{example_where}: quality has no calculator axis" + yield from numeric_range_errors(example, example_where) + width, height, quality = key + axis = axes.get(quality) + offset, divisor = calculator.get("area_offset"), calculator.get("area_divisor") + output_tokens = example.get("output_tokens") + numbers = (width, height, axis, offset, divisor, output_tokens) + if ( + all(isinstance(number, int) and not isinstance(number, bool) for number in numbers) + and min(width, height, axis, divisor) > 0 + ): + short_axis = round(Fraction(axis * min(width, height), max(width, height))) + numerator = axis * short_axis * (offset + width * height) + expected = -(-numerator // divisor) + if output_tokens != expected: + yield f"{example_where}: output_tokens {output_tokens} does not match calculator {expected}" + + +def token_pricing_errors(pricing: dict[str, Any], where: str) -> Iterator[str]: + tiers = pricing.get("tiers", []) + if not isinstance(tiers, list): + return + ranges: dict[str, list[tuple[int, int | None]]] = {} + for index, tier in enumerate(tiers): + if not isinstance(tier, dict): + continue + lo = tier.get("minimum_input_tokens") + hi = tier.get("maximum_input_tokens") + meter = tier.get("meter") + if not isinstance(lo, int) or isinstance(lo, bool) or not isinstance(meter, str): + continue + if hi is not None and (not isinstance(hi, int) or isinstance(hi, bool)): + continue + if hi is not None and hi <= lo: + yield f"{where}/tiers/{index}: exclusive maximum_input_tokens must exceed minimum_input_tokens" + ranges.setdefault(meter, []).append((lo, hi)) + for meter, bands in ranges.items(): + bands.sort(key=lambda band: band[0]) + for (_, previous_maximum), (minimum, _) in zip(bands, bands[1:]): + if previous_maximum is None or previous_maximum > minimum: + yield f"{where}: overlapping input-token tiers for {meter}" + + +def published_fact_errors(facts: dict[str, Any]) -> Iterator[str]: + for where, value in fact_objects(facts): + endpoints = value.get("endpoints", []) + seen_endpoint_ids = set() + if isinstance(endpoints, list): + for endpoint in endpoints: + endpoint_id = endpoint.get("id") if isinstance(endpoint, dict) else None + if not isinstance(endpoint_id, str): + continue + if endpoint_id in seen_endpoint_ids: + yield f"{where}: duplicate endpoint {endpoint_id!r}" + seen_endpoint_ids.add(endpoint_id) + for name in SOURCED_OBJECTS: + sourced = value.get(name) + if not isinstance(sourced, dict): + continue + sourced_where = f"{where}/{name}" + yield from source_url_errors(sourced.get("source_url"), sourced_where) + if "source_js_url" in sourced: + yield from source_url_errors(sourced["source_js_url"], f"{sourced_where}/source_js_url") + yield from numeric_range_errors(sourced, sourced_where) + if name == "output_token_calculator": + yield from calculator_errors(sourced, sourced_where) + elif name == "token_pricing": + yield from token_pricing_errors(sourced, sourced_where) + if "parameters" in value or "image_constraints" in value: + yield from source_url_errors(value.get("source_url"), where) + parameters = value.get("parameters", {}) + if isinstance(parameters, dict): + for name, parameter in parameters.items(): + if isinstance(parameter, dict): + parameter_where = f"{where}/parameters/{name}" + yield from numeric_range_errors(parameter, parameter_where) + if "source_url" in parameter: + yield from source_url_errors(parameter["source_url"], parameter_where) + constraints = value.get("image_constraints") + if isinstance(constraints, dict): + yield from numeric_range_errors(constraints, f"{where}/image_constraints") + if isinstance(constraints.get("aspect_ratio"), dict): + yield from numeric_range_errors( + constraints["aspect_ratio"], f"{where}/image_constraints/aspect_ratio" + ) + + def invariant_errors(facts: dict[str, Any]) -> Iterator[str]: used_exceptions: set[tuple[str, str, str]] = set() @@ -131,18 +316,14 @@ def invariant_errors(facts: dict[str, Any]) -> Iterator[str]: yield f"{where}: has no provider bindings" continue for name, binding in providers.items(): + if not isinstance(binding, dict): + continue if binding.get("provider") != name: yield f"{where}/providers/{name}: binding.provider is {binding.get('provider')!r}" if not binding.get("id"): yield f"{where}/providers/{name}: binding has no id" - for name, binding in card.get("providers", {}).items(): - endpoint_ids = set() - for endpoint in binding.get("endpoints", []): - endpoint_id = endpoint.get("id") - if endpoint_id in endpoint_ids: - yield f"{where}/providers/{name}: duplicate endpoint {endpoint_id!r}" - endpoint_ids.add(endpoint_id) + yield from published_fact_errors(facts) for exception in sorted(KNOWN_EXCEPTIONS - used_exceptions): yield f"stale known exception {exception}: remove it from KNOWN_EXCEPTIONS" diff --git a/tests/test_check.py b/tests/test_check.py index 1cd8161..1bc6971 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -339,3 +339,204 @@ def mutate(facts): def test_existing_binding_without_endpoints_remains_valid(): assert check.check(MODELS_JSON, SCHEMA) == [] + + +# Additive published calculators, serving facts, and endpoint parameters. + +SOURCE = "https://provider.example/models/example/api" +CALCULATOR = { + "source_url": SOURCE, + "area_offset": 2000000, + "area_divisor": 4000000, + "short_axis_rounding": "nearest_ties_even", + "token_rounding": "ceil", + "quality_axes": {"low": 16, "medium": 48, "high": 96}, + "examples": [{"width": 1024, "height": 1024, "quality": "low", "output_tokens": 196, "source_url": SOURCE}], +} +TOKEN_PRICING = { + "source_url": SOURCE, + "unit": "usd_per_token", + "input": "0.000005", + "output": "0.00003", + "tiers": [ + {"meter": "input", "minimum_input_tokens": 0, "maximum_input_tokens": 272000, "usd_per_token": "0.000005"}, + {"meter": "input", "minimum_input_tokens": 272000, "usd_per_token": "0.00001"}, + ], +} +ENDPOINT = { + "id": "provider/example/edit", "input": "image", "source_url": SOURCE, + "parameters": { + "image_size.width": {"type": "integer", "minimum": 16, "maximum": 3840, "multiple_of": 16}, + "quality": {"type": "string", "enum": ["low", "medium", "high"]}, + "image_urls": {"type": "array", "min_items": 1, "max_items": 16}, + }, + "image_constraints": {"step": 16, "min_pixels": 655360, "max_pixels": 8294400, "aspect_ratio": {"max": 3}}, +} + + +def enriched(mutate=lambda card: None): + facts = copy.deepcopy(MODELS_JSON) + card = first("image.models", facts) + card.update({ + "output_token_calculator": copy.deepcopy(CALCULATOR), + "vision_tokenization": {"source_url": SOURCE, "patch_size_px": 32, "token_multiplier": 1.2, + "detail_patch_budgets": {"high": 25088}, "detail_maximum_edges": {"low": 512}, + "maximum_tokens_per_image": 4784, "token_rounding": "ceil"}, + "capabilities": {"source_url": SOURCE, "reasoning": True, "structured_output": False}, + }) + binding = card["providers"]["fal"] + binding.update({"token_pricing": copy.deepcopy(TOKEN_PRICING), + "limits": {"source_url": SOURCE, "context_tokens": 272000, "max_output_tokens": 32000}, + "endpoints": [copy.deepcopy(ENDPOINT)]}) + mutate(card) + return check.check(facts, SCHEMA) + + +def test_additive_sourced_facts_validate_together(): + assert enriched() == [] + + +@pytest.mark.parametrize("field", ["output_token_calculator", "vision_tokenization", "capabilities"]) +def test_new_card_facts_require_sources(field): + assert any("source_url" in error for error in enriched(lambda card: card[field].pop("source_url"))) + + +def test_new_binding_facts_require_sources(): + assert any("source_url" in error for error in enriched(lambda card: card["providers"]["fal"]["token_pricing"].pop("source_url"))) + + +@pytest.mark.parametrize("url", ["https://", "https://operator:secret@provider.example/api", "http://provider.example/api"]) +def test_new_fact_urls_are_absolute_https_without_credentials(url): + assert any("source_url" in error for error in enriched(lambda card: card["capabilities"].update(source_url=url))) + + +def test_calculator_cannot_carry_arbitrary_consumer_fields(): + assert any("additional" in error.lower() for error in enriched(lambda card: card["output_token_calculator"].update(preference_order=["fast", "quality"]))) + + +def test_calculator_axes_must_be_positive(): + assert any("quality_axes" in error for error in enriched(lambda card: card["output_token_calculator"]["quality_axes"].update(low=0))) + + +def test_calculator_divisor_cannot_be_zero(): + assert any("area_divisor" in error for error in enriched(lambda card: card["output_token_calculator"].update(area_divisor=0))) + + +def test_calculator_example_quality_must_have_an_axis(): + assert any("quality has no" in error for error in enriched(lambda card: card["output_token_calculator"]["examples"][0].update(quality="unknown"))) + + +def test_calculator_examples_are_unique_per_size_and_quality(): + def mutate(card): + calc = card["output_token_calculator"] + calc["examples"].append(copy.deepcopy(calc["examples"][0])) + assert any("duplicate size/quality" in error for error in enriched(mutate)) + + +def test_verified_output_token_example_must_match_calculator(): + assert any("does not match calculator" in error for error in enriched(lambda card: card["output_token_calculator"]["examples"][0].update(output_tokens=197))) + + +def test_rounded_price_example_never_requires_inferred_tokens(): + def mutate(card): + example = card["output_token_calculator"]["examples"][0] + example.pop("output_tokens") + example.update(usd=0.006, usd_precision=3) + assert enriched(mutate) == [] + + +def test_example_requires_a_documented_token_or_price_value(): + assert any("examples" in error for error in enriched(lambda card: card["output_token_calculator"]["examples"][0].pop("output_tokens"))) + + +def test_example_precision_requires_price(): + assert any("usd" in error for error in enriched(lambda card: card["output_token_calculator"]["examples"][0].update(usd_precision=3))) + + +def test_vision_rules_cannot_be_empty_source_only(): + assert any("vision_tokenization" in error for error in enriched(lambda card: card.update(vision_tokenization={"source_url": SOURCE}))) + + +def test_vision_quantities_must_be_positive(): + assert any("maximum_tokens_per_image" in error for error in enriched(lambda card: card["vision_tokenization"].update(maximum_tokens_per_image=0))) + + +def test_serving_capabilities_must_be_boolean(): + assert any("capabilities" in error for error in enriched(lambda card: card["capabilities"].update(reasoning="yes"))) + + +def test_provider_limits_must_be_positive(): + assert any("context_tokens" in error for error in enriched(lambda card: card["providers"]["fal"]["limits"].update(context_tokens=0))) + + +def test_token_rates_preserve_exact_decimal_strings(): + assert any("token_pricing" in error for error in enriched(lambda card: card["providers"]["fal"]["token_pricing"].update(input=0.000005))) + + +def test_token_price_unit_is_explicit_and_fixed(): + assert any("token_pricing" in error for error in enriched(lambda card: card["providers"]["fal"]["token_pricing"].update(unit="usd_per_million_tokens"))) + + +def test_tiers_use_half_open_bounds_and_allow_unbounded_last_band(): + assert enriched() == [] + + +def test_token_tiers_cannot_overlap_for_the_same_meter(): + assert any("overlapping" in error for error in enriched(lambda card: card["providers"]["fal"]["token_pricing"]["tiers"][1].update(minimum_input_tokens=271999))) + + +def test_token_tier_cannot_be_empty_or_reversed(): + assert any("exclusive maximum" in error for error in enriched(lambda card: card["providers"]["fal"]["token_pricing"]["tiers"][0].update(maximum_input_tokens=0))) + + +def test_token_tiers_may_overlap_between_distinct_meters(): + def mutate(card): + pricing = card["providers"]["fal"]["token_pricing"] + pricing["tiers"].append({"meter": "output", "minimum_input_tokens": 0, "usd_per_token": "0.00003"}) + assert enriched(mutate) == [] + + +def test_endpoint_dotted_native_parameter_names_are_preserved(): + assert enriched() == [] + + +def test_endpoint_parameters_must_have_ordered_ranges(): + assert any("minimum exceeds maximum" in error for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["parameters"]["image_size.width"].update(minimum=4000))) + + +def test_endpoint_reference_count_range_cannot_be_reversed(): + assert any("min_items exceeds max_items" in error for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["parameters"]["image_urls"].update(min_items=17))) + + +def test_endpoint_image_constraint_range_cannot_be_reversed(): + assert any("min_pixels exceeds max_pixels" in error for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["image_constraints"].update(min_pixels=9000000))) + + +def test_parameter_enum_must_not_repeat_values(): + assert any("non-unique" in error for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["parameters"]["quality"].update(enum=["low", "low"]))) + + +def test_nonfinite_published_numbers_are_rejected(): + assert any("finite" in error for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["parameters"]["image_size.width"].update(maximum=float("inf")))) + + +def test_endpoint_fact_fields_cannot_hold_arbitrary_consumer_settings(): + assert any("additional" in error.lower() for error in enriched(lambda card: card["providers"]["fal"]["endpoints"][0]["parameters"]["quality"].update(default="high"))) + + +def test_single_provider_card_can_record_sourced_endpoint_variants(): + facts = copy.deepcopy(MODELS_JSON) + first("three_d.models", facts)["endpoints"] = [copy.deepcopy(ENDPOINT)] + assert check.check(facts, SCHEMA) == [] + + +def test_single_provider_endpoint_ids_cannot_repeat(): + facts = copy.deepcopy(MODELS_JSON) + first("three_d.models", facts)["endpoints"] = [copy.deepcopy(ENDPOINT), copy.deepcopy(ENDPOINT)] + assert any("duplicate endpoint" in error for error in check.check(facts, SCHEMA)) + + +def test_singleton_operation_can_record_sourced_endpoint_parameters(): + facts = copy.deepcopy(MODELS_JSON) + facts["three_d.rig_check.operation"]["endpoints"] = [copy.deepcopy(ENDPOINT)] + assert check.check(facts, SCHEMA) == [] From 263759621b0845338fd92487a1b58ed41b907130 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 21:05:51 +0900 Subject: [PATCH 3/4] Describe public provider coverage, prices and model distribution --- .agents/skills/catalogue/SKILL.md | 74 +++-- .agents/skills/catalogue/pricing.md | 38 ++- .agents/skills/catalogue/providers.md | 17 +- .agents/skills/provenance/SKILL.md | 4 +- AGENTS.md | 24 +- README.md | 35 ++- catalogue/README.md | 104 ++++++- catalogue/schemas/models.schema.json | 383 ++++++++++++++++++++++++-- scripts/check.py | 135 ++++++++- tests/test_distribution.py | 264 ++++++++++++++++++ tests/test_provider_coverage.py | 234 ++++++++++++++++ 11 files changed, 1207 insertions(+), 105 deletions(-) create mode 100644 tests/test_distribution.py create mode 100644 tests/test_provider_coverage.py diff --git a/.agents/skills/catalogue/SKILL.md b/.agents/skills/catalogue/SKILL.md index bb65003..c9e9729 100644 --- a/.agents/skills/catalogue/SKILL.md +++ b/.agents/skills/catalogue/SKILL.md @@ -4,8 +4,8 @@ description: > Research, add, update, deprecate, or remove AI model catalogue entries — text, image, video, audio, 3D, image tools, and embedding models — with their capabilities, limits, provider bindings, and published prices. - Covers what an entry is, per-provider ID rules, pricing shapes, lifecycle - rules (steady-state pricing, deprecate vs. delete), and the verification + Covers what an entry is, per-provider ID rules, regular pricing, native + reference and provider coverage, public distribution, and the verification checklist. Use when bumping a model version, adding a model or provider binding, updating a price, or auditing entries against provider docs. Release dates are covered by $provenance. @@ -37,13 +37,19 @@ An entry is a set of facts about one model, each traceable to a source: - **Identity:** `id`, `label`, `vendor`, family. - **Release:** see `$provenance`. +- **Distribution:** official code and weight links, exact weight variants, access + conditions, and separately sourced code/weight licenses. Use the model-level + `distribution` shape in [`catalogue/README.md`](../../../catalogue/README.md#model-distribution). + Leave unknown facts omitted. Available weights do not establish a native hosted API; + public downloads do not establish an open-source classification or unrestricted use. - **Capabilities:** modalities, accepted image MIME types, tool calling, audio, transparent background, and so on. Never derive one capability from another (for example, image MIME types from `multimodal`); record each from its own source. - **Limits:** context window and output limit (text); sizes, size constraints, aspect ratios (image); durations and aspect ratios (video). -- **Bindings:** for each provider that serves it, the provider's exact ID, its published - price, and optionally a URL and a deprecation flag. +- **Bindings:** exact call IDs, sourced regular prices when known, and provider-specific + contracts. A native `reference_provider` is a presentation choice. Explicit + `provider_coverage` distinguishes listed providers from source-backed absence/exclusion. Facts are independent of products. Adding a verified fact does not list the model in any Grida product, and removing a product listing does not delete the fact. @@ -52,12 +58,12 @@ Grida product, and removing a product listing does not delete the fact. The same model has different IDs, availability, and prices on different providers. -- **Text, audio, image tools, 3D:** one entry per provider endpoint. The `id` is in that - provider's format, and the provider or namespace fixes the route. +- **Text, audio, image tools, 3D:** retain existing endpoint-shaped IDs and direct-provider + facts. Add `providers` bindings without rewriting those existing identities. - **Image and video:** one canonical entry (`vendor/model`, e.g. `google/veo-3.1`) holding the model's intrinsic facts, plus a `providers` record keyed by provider. Each binding has its - own call ID and price. Bindings carry no preference order; choosing a provider is a - consumer's decision. + own call ID and price. `reference_provider` identifies the verified native API to show + first; choosing which provider to call remains a consumer decision. Preserve IDs exactly as the provider spells them. Never case-convert or normalize them. Per-provider ID formats, pricing pages, and known quirks are in @@ -68,20 +74,22 @@ Per-provider ID formats, pricing pages, and known quirks are in The catalogue states what is true and useful **now**. Its shape must never record how recently someone got round to updating it. -- **Price the steady state, not the promotion.** If a vendor runs an introductory or - time-limited rate, record the price that applies after it ends, and note the promotion and - its end date in `PROVENANCE.md`. Otherwise the promotion ending is a silent cost increase. - Recheck when the date passes: a promotion can become permanent, which changes the fact. -- **Deprecate an entry that is still a real choice.** Mark it deprecated (Grida's - `legacy: true`) when someone might reasonably still pick it: same price as its successor, - or better at something. -- **Delete an entry whose successor is strictly dominant:** never worse on any axis, better - on at least one. An entry nobody should choose is noise in every picker. -- **A binding needs a usable, verified price.** If a provider exposes no billable unit for a - model (for example, a $0/MTok placeholder for video), leave that binding out rather than - inventing a price. -- **Catalogue only what is used.** Do not add a capability, endpoint variant, or field for a - case nobody serves yet. Add it when the second case arrives. +- **Regular, non-promotional prices only.** Record the provider's sourced regular rate, + even when an introductory or time-limited discount is active when adding or updating a + model. Do not track promotional rates, end dates, or expiry rechecks. A discounted live + feed does not establish the regular price; do not infer it from a discount percentage or + another provider's rate. If the regular rate cannot be verified, report the missing + evidence and leave the price unrecorded. +- **Preserve existing entries during coverage work.** Do not delete a model because a + successor exists or because a consumer does not currently serve it. Lifecycle and + offering changes need separate review. Do not introduce product policy fields. +- **Unknown prices remain unknown.** Verified endpoint existence can be recorded without + a price. A $0/MTok video placeholder is not a usable unit; omit that price and record + the qualification. If the contract itself cannot be represented reliably, use a + source-backed exclusion rather than inventing capabilities or amounts. +- **Coverage is source-backed.** Add the native API and verified OpenRouter, Vercel and + fal media bindings. Add Grida only where its public service actually offers the model. + An unserved catalogue entry remains useful; do not invent a Grida binding. Removing an entry is a contract change for consumers: some embed the catalogue at build time and only see the change in their next release. Say so in the PR. @@ -98,12 +106,16 @@ time and only see the change in their next release. Say so in the PR. - When two first-party sources conflict, first check they describe the same thing (variant, provider, region, tier, resolution, date). If they truly conflict, decide which wins and why (more specific page, dated changelog), record it in `catalogue/PROVENANCE.md`, and - never average. If it can't be resolved from public sources, leave the value and ask. + never average. If it cannot be resolved from public sources, leave the disputed fact + unrecorded and document the gap. Aggregate behavioral questions for a separately + approved probe; do not make paid inference calls as part of catalogue maintenance. ## Pricing -Record the provider's published list price in the provider's own billing unit. Never -convert to credits, add margin, or substitute a negotiated or account-specific rate. The +Record the provider's advertised regular price in its published billing unit. Public +Grida prices are provider facts too; they do not authorize disclosing internal economics +or deriving a different price from another provider. Never convert to credits, add an +internal margin, or substitute a negotiated/account-specific rate. The pricing shapes in use and their rules are in [`pricing.md`](./pricing.md). ## Workflow @@ -128,17 +140,21 @@ pricing shapes in use and their rules are in [`pricing.md`](./pricing.md). homepage, and its official logo (see [`models/README.md`](../../../models/README.md#a-lab)). Fields are `lower_snake_case`; IDs keep their exact spelling. Run `uv run scripts/build.py`, then `uv run scripts/check.py`. -5. **Record qualifications** that do not fit the JSON (promotions, long-context bands, - ambiguous units) in `catalogue/PROVENANCE.md`, with source URLs. +5. **Record qualifications** that do not fit the JSON (regular-price evidence, + long-context bands, ambiguous units) in `catalogue/PROVENANCE.md`, with source URLs. ## After any update - [ ] Every new or changed fact has a first-party source, cited in the PR description - [ ] No value was estimated, rounded, or copied from an aggregator without verification -- [ ] Prices are published list prices in the provider's own unit, at the steady state +- [ ] Prices are sourced regular, non-promotional rates in the provider's own unit; + promotional rates and expiry tracking are excluded - [ ] IDs are exact; field names are `lower_snake_case` +- [ ] Distribution links are official; code and weight licenses/access are independently + sourced, with no inferred hosted API or license permissions - [ ] Every entry has a complete `release` (`$provenance`) -- [ ] Deprecated vs. deleted was decided by the lifecycle rules above +- [ ] Native reference, canonical provider coverage and any exclusions are sourced; existing + model identities and unserved entries are preserved - [ ] `catalogue/models.json` was regenerated with `uv run scripts/build.py`, not edited by hand - [ ] `uv run scripts/check.py` passes - [ ] Nothing from a private source made it into the diff, the commit, or the PR (see `AGENTS.md`) diff --git a/.agents/skills/catalogue/pricing.md b/.agents/skills/catalogue/pricing.md index b9b1f92..57e816b 100644 --- a/.agents/skills/catalogue/pricing.md +++ b/.agents/skills/catalogue/pricing.md @@ -7,12 +7,20 @@ Supplements [`SKILL.md`](./SKILL.md). Load this when you add or change a price. - **Published list price, in the provider's unit.** Record what the provider's public pricing page says, in the unit it bills: per million tokens, per image, per second, per run, per 1,000 characters, or provider credits with their published USD rate. Do not - convert units or currencies. + convert currencies. Exact decimal unit normalization is allowed when both the source + unit and contract unit are explicit; it must not round or change billing semantics. - **A price belongs to a binding.** The same model can cost different amounts on different - providers. On canonical entries (image, video), each binding carries its own `pricing`. -- **Steady state, not promotion.** See the lifecycle rules in [`SKILL.md`](./SKILL.md). -- **No usable unit, no binding.** If a provider shows a placeholder price (for example, - $0/MTok for video), leave the binding out. + providers. Each binding or endpoint carries its own price without changing intrinsic + model facts. Public Grida service prices are eligible; its supplier selection, funding + fees, credit conversions and margin are not catalogue data. +- **Regular, non-promotional rates only, even during an active promotion.** Do not + maintain promotional prices, expiry dates, or a promotion calendar. Verify the regular + rate from that provider's own source; a discounted feed or another provider's rate is + insufficient. See the lifecycle rules in [`SKILL.md`](./SKILL.md). +- **No usable unit, no price.** If a provider shows a placeholder price (for example, + $0/MTok for video), leave that price out. A verified endpoint can still have a binding; + document unknown regular rates. Use a sourced exclusion only for a contract that cannot + be reliably listed, not to hide unfinished research. - **Estimates are not prices.** `avg_cost_usd` and `cost_usd` are coarse per-request estimates, not published facts. Keep them consistent with the published rates and never cite them as a source. @@ -22,6 +30,19 @@ Supplements [`SKILL.md`](./SKILL.md). Load this when you add or change a price. These are the `pricing.type` values in the current authored data. Before adding a new shape, check whether an existing one can express the price without loss. +`published_pricing` is an additive provider-neutral shape for publicly advertised +USD meter components, conditions, request-wide quantity bands, and quantity +rounding. Its contract is in +[`catalogue/README.md`](../../../catalogue/README.md#published-service-prices-and-contracts). +Use it only from the service's own public evidence. Its source is the advertised +price, not assumptions about that service's suppliers or margins. + +`token_pricing` records exact decimal USD-per-token rates. Keep cache-write TTLs +distinct: `input_cache_write_1h` records the one-hour rate when published; +`scope` qualifies the standard tier and the base cache-write duration. A +cache-storage-per-hour amount is a different unit and must not be inserted as +a per-token write price. + ### Text Text entries use a `cost` object instead of a typed `pricing` union. Values are USD per @@ -90,7 +111,10 @@ silent output, omit those keys rather than copying another provider's rates. 1. Open the provider's own pricing page or model page (see [`providers.md`](./providers.md)). 2. Record the date you checked it in the PR description. -3. If the page qualifies the price (region, tier, batch discount, promotion end date), note - the qualification and URL in `PROVENANCE.md`. +3. Verify that the quoted rate is regular and non-promotional. Record its source and + applicable conditions (region, service tier, batch mode, billing unit) in + `PROVENANCE.md` when needed. Do not record promotional rates or end dates. If public + sources establish only a promotional rate, leave the regular price unrecorded and + document the verification gap; do not guess it. 4. If two first-party pages disagree, follow "When sources disagree" in [`SKILL.md`](./SKILL.md). diff --git a/.agents/skills/catalogue/providers.md b/.agents/skills/catalogue/providers.md index edb5af8..817c644 100644 --- a/.agents/skills/catalogue/providers.md +++ b/.agents/skills/catalogue/providers.md @@ -22,6 +22,7 @@ change a `basis: "model"` release date. | fal | Endpoint path: `fal-ai/veo3.1/image-to-video`, `fal-ai/flux-2-pro/edit` | | OpenRouter | `vendor/model`: `google/veo-3.1`, `google/veo-3.1-fast` | | Replicate | `owner/model`: `google/lyria-3`, `nightmareai/real-esrgan` | +| Grida | Public model ID or model-specific operation ID from its API catalogue | On fal, the capability is often part of the ID (`…/image-to-video`, `…/edit`). A different capability on fal is a different endpoint, so it is a different binding. @@ -37,17 +38,27 @@ capability on fal is a different endpoint, so it is a different binding. | fal | `https://fal.ai/models/`; pricing API: `https://fal.ai/docs/documentation/model-apis/pricing` | | OpenRouter | `https://openrouter.ai//` | | Vercel AI Gateway | `https://vercel.com/ai-gateway/models/` | +| Grida | `https://gateway.grida.ai/llms.txt`; linked discovery and operation schemas | | models.dev | `https://models.dev/api.json`. **Discovery only**, never a source | ## Known quirks -- **Availability differs by provider.** A model can be on OpenRouter and fal but not on the - Vercel AI Gateway. A canonical entry simply omits the binding it does not have. +- **Availability differs by provider.** A model can be on OpenRouter and fal but not on + Vercel. Verify native API, OpenRouter, Vercel and fal (media only) against first-party + sources; retain `provider_coverage` dispositions. Grida is required only when its + public service actually offers the model. Absence needs evidence, not an incomplete + search. A consistent spec/identity/pricing incompatibility may justify a sourced + per-model exclusion. Never use private routing preference as an exclusion reason. - **Metering differs by provider.** For one video model, one provider may meter audio and silent output and 4K, while another sells only audio-on output at 720p/1080p. Record each provider's own matrix. - **Placeholder prices.** OpenRouter can show `$0/MTok` for video models. That is not a - usable price: leave the binding out. + usable price: omit the price and document the qualification. Verify image/video + endpoints directly; a text feed does not describe those modalities. +- **Native reference.** Set `reference_provider` to the actual native API binding, or an + existing direct native provider, after verification. Prefer one qualified Google API + unless another has a meaningful published contract difference. Open-weight code or + artifacts establish `distribution`, not a billable API run by the model lab. - **Namespaces differ from vendor IDs.** For example, image IDs may use a short namespace (`bfl/…`) while the vendor field is `black-forest-labs`. Keep whatever the authored data uses; do not "fix" an ID to match a vendor name. diff --git a/.agents/skills/provenance/SKILL.md b/.agents/skills/provenance/SKILL.md index 08965cb..675709d 100644 --- a/.agents/skills/provenance/SKILL.md +++ b/.agents/skills/provenance/SKILL.md @@ -77,7 +77,9 @@ history. Never fill in an approximate day. A JSON value cannot hold its own caveats. Write a note in [`catalogue/PROVENANCE.md`](../../../catalogue/PROVENANCE.md) when a fact needs one. For example: -- A promotion: the steady-state price that was recorded, and the promotion's end date. +- Regular-price evidence: the provider's source establishing its non-promotional rate, + or a verification gap when only a promotional rate is published. Do not track discounted + prices, promotion end dates, or expiry rechecks. - A pricing band, such as long-context multipliers, with its threshold and source. - A capability shared by a model family, with the page that states it. - A conflict between sources, and why one was chosen. diff --git a/AGENTS.md b/AGENTS.md index cabc097..023d3a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,9 +63,13 @@ file, commit, branch name, or PR: - the name, URL, path, file, PR, or tooling of a private repository or service; - how a consumer routes requests (which provider for which model, fallback order); -- negotiated, wholesale, or account-specific prices; margins, markups, credit conversions; +- negotiated, wholesale, or account-specific prices; internal margins, funding fees, credit conversions; - provider account details, contract or rights reviews; -- hostnames, account IDs, credential or environment variable names, budgets, roadmap. +- private hostnames, account IDs, credential or environment variable names, budgets, roadmap. + +Published service contracts, public API URLs and advertised regular prices are public +provider facts, including Grida's. Record them without disclosing supplier selection, +internal tariff calculations or funding policy. Restate a need in neutral, public terms ("per-second video pricing keyed by resolution"). When in doubt, leave it out and ask. @@ -81,14 +85,24 @@ doesn't belong in the contract. See `$catalogue` and `$provenance`. - **Never fabricate.** If a value cannot be verified, leave it out and say so. Do not estimate a price, round a limit, or infer a release day. -- **Published prices only.** No negotiated, wholesale, account-specific, or internal prices. - No markup, credit conversion, or margin. +- **Published regular prices only.** Use sourced, non-promotional provider rates even + during an active promotion. Do not maintain promotional prices or expiry tracking. + No negotiated, wholesale, account-specific, or internal prices; no internal markup, credit + conversion, or internal margin. A provider's publicly advertised rate is eligible even + when its internal economics are unknown. Never infer an unknown regular rate. - **Preserve IDs exactly.** Model and provider IDs keep their exact spelling and casing. Authored field names are `lower_snake_case`. - **Additive by default.** Add fields; do not rename or repurpose them. A breaking change needs a new schema version, decided with the user. - **Facts are independent of products.** Being catalogued is not the same as being offered - by a Grida product. + by a Grida product. Keep existing entries when reviewing provider coverage; deleting a + model or expanding an offering requires a separate review. +- **Provider coverage is explicit.** Qualify the native lab API as `reference_provider` + where it exists. It is a presentation reference, never a routing default. Record + OpenRouter, Vercel and fal (media only) where verified, and Grida where it serves the + model. Record source-backed absence or exclusion in `provider_coverage`; unfinished + research is not evidence of absence. Open weights do not imply a native hosted API. + See the catalogue skill for coverage and public price rules. ## Skills diff --git a/README.md b/README.md index 370659f..18a740f 100644 --- a/README.md +++ b/README.md @@ -80,22 +80,24 @@ consumes a projection of it. ## What is in a catalogue entry -| Kind | Examples | -| ------------------------- | --------------------------------------------------------------------------------------------- | -| **Identity** | Canonical ID (`vendor/model`), label, vendor, family (text, image, video, audio, 3D, …) | -| **Release provenance** | First broad public release date, its basis, and the first-party source URL | -| **Capabilities** | Modalities, accepted image formats, tool calling, audio, transparent background | -| **Limits** | Context window, output limit, sizes, aspect ratios, durations, size constraints | -| **Provider bindings** | The exact ID a provider uses for the model (Vercel AI Gateway, fal, OpenRouter, Replicate, …) | -| **Published prices** | Public list prices per provider, in the provider's own billing unit | -| **Published calculators** | Sourced output-token formulae, examples, and image-input tokenization rules | +| Kind | Examples | +| ------------------------- | --------------------------------------------------------------------------------------- | +| **Identity** | Canonical ID (`vendor/model`), label, vendor, family (text, image, video, audio, 3D, …) | +| **Release provenance** | First broad public release date, its basis, and the first-party source URL | +| **Capabilities** | Modalities, accepted image formats, tool calling, audio, transparent background | +| **Limits** | Context window, output limit, sizes, aspect ratios, durations, size constraints | +| **Provider bindings** | Exact provider IDs, a native API reference, and sourced coverage/exclusion reasons | +| **Model distribution** | Official code and weight links, exact variants, access conditions and separate licenses | +| **Published prices** | Public list prices per provider, in the provider's own billing unit | +| **Published calculators** | Sourced output-token formulae, examples, and image-input tokenization rules | ## What it is not - **Not a router or SDK.** A provider binding records that an endpoint exists and what it costs. It does not implement a client or grant permission to call it. -- **Not billing.** The catalogue holds published list prices. Credits, markups, tariffs, and - negotiated or account-specific rates belong to their owners, not here. +- **Not billing execution.** The catalogue holds publicly advertised regular prices, + including Grida's public service price. Funding fees, credit conversion, margins, + supplier selection and negotiated/account-specific rates stay with their owners. - **Not a live mirror.** Nothing is scraped into this repository automatically. Discovery tools such as [models.dev](https://models.dev) give leads; a person verifies each fact against a first-party source. @@ -111,10 +113,13 @@ with a JSON Schema, readable from any language. Its shape and rules are document ## Principles - **Every fact has a source.** Prices, limits, and release dates cite a first-party page. -- **The catalogue states what is true now.** Price the steady state, not the promotion. - Deprecate a model someone might still pick; remove one whose successor is strictly better. -- **An ID is never portable.** The same model has different IDs, availability, and prices on - different providers. Bindings are recorded per provider. +- **Regular prices, without promotion tracking.** Record sourced, non-promotional + provider rates even during an active discount. Do not maintain discounted prices or + promotion expiry dates. Existing models remain available for consumer review; provider + coverage changes do not silently remove them. +- **An ID is never portable.** The same model has different IDs, availability, parameters + and prices on different providers. Bindings retain those differences. A sourced native + `reference_provider` is presented first; it does not choose a consumer's route. - **Additive by default.** Consumers depend on the shape. Fields are added; breaking changes get a new schema version. diff --git a/catalogue/README.md b/catalogue/README.md index 657482c..867938f 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -50,8 +50,9 @@ section is a table of cards keyed by card ID: Cards are listed in ID order within each section; the order carries no meaning. Every card has `id` (equal to its table key), `label`, and `release` (`{ date, basis, source_url }`). The rest depends on the family. Image and video cards are -canonical (`vendor/model`) and hold a `providers` record of bindings keyed by provider; -other families have one card per provider endpoint. +canonical (`vendor/model`) and hold a `providers` record of bindings keyed by provider. +Other families retain their existing IDs and direct-provider facts, and may also add +`providers`; the same additions are supported on embeddings and the rig-check operation. ## Labs @@ -71,8 +72,9 @@ license, and are used under each owner's guidelines. breaking change needs a new schema version and a new `$id`. - **IDs are exact.** Model and provider IDs keep the spelling their owner uses. - **Fields are `lower_snake_case`.** -- **Facts only.** No listing, lifecycle, ordering, or default choices (`listed`, `status`, +- **Facts only.** No product listing, lifecycle or routing defaults (`listed`, `status`, `legacy`, `deprecated`, `default`); those belong to each consumer. + `reference_provider` identifies the native API for presentation, not a routing default. - **Sourced.** Every value traces to a first-party page; qualifications go in `PROVENANCE.md`. The schema's `$id` is `https://grida.co/schemas/ai-model-facts-v1.json`. It is kept @@ -104,15 +106,101 @@ uv run pytest # the checker rejects what it should ## Provider endpoints A provider binding may list `endpoints`: exact call IDs with `input` (`text`, -`image`, `video`, `audio`, or `task`) and a first-party `source_url`. These identify -input variants, not a routing preference. The existing binding `id` remains its -primary call ID; `references` and pricing retain their existing meaning. +`image`, `multi_image`, `video`, `audio`, `task`, or `asset`) and a first-party `source_url`. These identify +input variants, not a routing preference. The existing binding `id` retains its +provider-defined model or primary call identity; `references` and pricing retain +their existing meaning. Endpoint IDs are unique within a binding. The list records documented variants, not an assertion that a consumer supports every variant. `vendor`, including on text cards, identifies the model's lab in `labs.json`. It is independent of a model ID's namespace and its serving provider. +### Native reference and coverage + +Every family, including embedding and the rig-check operation, may add a +`providers` record without changing its existing ID or direct-provider fields. +Each binding has exact `provider` and `id` values, an optional first-party +`source_url`, and independently sourced serving facts. A verified binding can +omit an unknown regular price; omission never means free inference. + +Optional `reference_provider` names a real native API binding, or the existing +direct `provider` when that card describes the native API already. Consumers may +present it first as the lab's reference. It does not prescribe which provider to +call, and JSON object order carries no meaning. Official weights do not establish +a native hosted API. This field records a sourced reference, not a provider registry. + +`provider_coverage` maps provider names to `{ disposition, source_url, reason? }`: + +- `listed` requires a binding or matching direct provider. +- `not_listed` records verified absence from that provider's published catalogue. +- `excluded` records an incompatible or unverifiable specification, identity or + pricing contract that makes a reliable listing inappropriate. + +The last two require a public, source-backed `reason` and cannot coexist with a +binding for that provider. Do not use an exclusion for routing preference or +unfinished research. Missing coverage remains unrecorded, not proof of absence. +Maintain coverage for native APIs, OpenRouter, Vercel and fal media where verified. +Grida bindings describe only models actually offered by its public service; +other catalogue models remain useful independently. + +### Published service prices and contracts + +A binding or endpoint may record `published_pricing` with first-party +`source_url`, `currency: "USD"`, and nonempty `components`. This provider-neutral +shape expresses advertised regular prices, including Grida's public price, using: + +- `meter`, `unit`, and exact decimal-string `usd_per_unit`; optional + `per_quantity` is a positive decimal denominator (one when omitted). +- Optional `id` identifies a component uniquely within that price. +- Optional `when` is a JSON Schema predicate over public request parameters. +- Optional `quantity_range: { meter, min, max }` is a half-open range with decimal + bounds; an omitted or null `max` leaves it unbounded. It selects a component for the range, + not marginal volume pricing. +- Optional `quantity_rounding: { mode, increment }` preserves quantity rounding: + `none` permits an omitted or null increment; `ceil` requires a positive decimal increment. + +For TOML arrays, omit the optional null upper bound or increment; `null_keys` +addresses object paths, not array indexes. Omission has the same defined semantics +as JSON null here and never means a zero price. + +Rates, ranges and rounding are sourced service facts. Internal margins, funding +fees, credit conversions, supplier routing and account-specific prices are excluded. +Do not derive a public service price from another provider's price without the +service's own published evidence. Unknown or promotional rates stay unrecorded. + +Endpoints may name a public `operation` and `interface`, and link authoritative +`request_schema_url` and `result_schema_url`. Links are absolute HTTPS without +credentials; schemas are not copied or fetched by the catalogue checker. +Existing `parameters` remains partial descriptive metadata, not a substitute for +the linked contract. A provider-specific capability difference stays on its binding +or endpoint rather than overwriting the model's intrinsic facts. + +## Model distribution + +A card may record `distribution`, independently of its hosted provider bindings. +It requires a first-party `source_url` and at least `code` or `weights`: + +- `code.repository_url` identifies the official source-code repository. +- `weights` is a nonempty array of official weight repositories or artifact pages. + Each item has `url`; an optional `id` preserves the exact published weight variant. + Distinct variants may share a repository URL, but their IDs must be distinct. +- Code and each weight item may independently record `license: { name, url }`. + Use the publisher's license name and its license text or model-card URL, including + custom licenses. An unverified license stays omitted; code and weights need not + use the same license. +- A weight item's optional `access` is `public` or `gated`, as documented by its + publisher. Omission means unrecorded access conditions. Public access does not + establish unrestricted use or an open-source classification. + +All URLs must be absolute HTTPS URLs without embedded credentials. Unknown +distribution facts stay omitted. These records describe published artifacts; +they do not establish a hosted API, an inference price, or permissions beyond the +linked licenses. `distribution` belongs to model cards, not provider bindings, +endpoint variants, or the rig-check operation. Existing cards remain valid +without it. See [TRELLIS.2](../models/microsoft/trellis-2/card.toml) for sourced +code and weight records. + ## Published calculators and serving facts Optional additions record the facts at the scope their source describes. A card's @@ -148,7 +236,9 @@ Missing fields mean unrecorded facts, never unsupported features or zero prices. `max_output_tokens`. Neither object chooses a consumer's supported features. - **Token pricing.** `token_pricing` records `unit: "usd_per_token"`, a `source_url`, and exact decimal strings for `input`, `output`, - `input_cache_read`, and `input_cache_write` where published. Optional `tiers` + `input_cache_read`, `input_cache_write`, and `input_cache_write_1h` where published. + Optional `scope` records a source-backed service tier or cache-duration qualification; + a storage-per-hour rate does not belong in a per-token meter. Optional `tiers` carry `meter`, an inclusive `minimum_input_tokens`, an exclusive `maximum_input_tokens`, and `usd_per_token`. An omitted or null maximum means no published upper bound. Bands may differ by meter and cannot overlap for the diff --git a/catalogue/schemas/models.schema.json b/catalogue/schemas/models.schema.json index abc1a68..103f7ff 100644 --- a/catalogue/schemas/models.schema.json +++ b/catalogue/schemas/models.schema.json @@ -62,7 +62,26 @@ "type": "object", "additionalProperties": { "type": "object", - "required": ["id", "label"] + "required": ["id", "label"], + "properties": { + "distribution": { + "$ref": "#/$defs/distribution" + }, + "providers": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/provider_binding" + } + }, + "reference_provider": { + "type": "string", + "minLength": 1, + "description": "Native API binding to present first as a model reference; not a routing choice." + }, + "provider_coverage": { + "$ref": "#/$defs/provider_coverage" + } + } } }, "three_d.rig_check.operation": { @@ -75,6 +94,7 @@ "provider": { "const": "tripo" }, + "distribution": false, "binding_id": { "const": "rig-check" }, @@ -138,6 +158,20 @@ "items": { "$ref": "#/$defs/provider_endpoint" } + }, + "providers": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/provider_binding" + } + }, + "reference_provider": { + "type": "string", + "minLength": 1, + "description": "Native API binding to present first as a model reference; not a routing choice." + }, + "provider_coverage": { + "$ref": "#/$defs/provider_coverage" } } } @@ -198,33 +232,7 @@ "providers": { "type": "object", "additionalProperties": { - "type": "object", - "required": ["provider", "id"], - "properties": { - "provider": { - "type": "string" - }, - "id": { - "type": "string", - "minLength": 1 - }, - "endpoints": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/provider_endpoint" - } - }, - "capabilities": { - "$ref": "#/$defs/capabilities" - }, - "token_pricing": { - "$ref": "#/$defs/token_pricing" - }, - "limits": { - "$ref": "#/$defs/limits" - } - } + "$ref": "#/$defs/provider_binding" } }, "vendor": { @@ -232,6 +240,9 @@ "minLength": 1, "description": "Lab ID resolving in labs.json; independent of the serving provider." }, + "distribution": { + "$ref": "#/$defs/distribution" + }, "output_token_calculator": { "$ref": "#/$defs/output_token_calculator" }, @@ -250,6 +261,14 @@ "items": { "$ref": "#/$defs/provider_endpoint" } + }, + "reference_provider": { + "type": "string", + "minLength": 1, + "description": "Native API binding to present first as a model reference; not a routing choice." + }, + "provider_coverage": { + "$ref": "#/$defs/provider_coverage" } } }, @@ -330,7 +349,7 @@ "minLength": 1 }, "input": { - "enum": ["text", "image", "video", "audio", "task"] + "enum": ["text", "image", "video", "audio", "task", "multi_image", "asset"] }, "source_url": { "type": "string", @@ -351,6 +370,27 @@ }, "limits": { "$ref": "#/$defs/limits" + }, + "operation": { + "type": "string", + "minLength": 1 + }, + "interface": { + "type": "string", + "minLength": 1 + }, + "published_pricing": { + "$ref": "#/$defs/published_pricing" + }, + "request_schema_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "result_schema_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" } } }, @@ -614,6 +654,15 @@ "items": { "$ref": "#/$defs/token_price_tier" } + }, + "scope": { + "type": "string", + "minLength": 1 + }, + "input_cache_write_1h": { + "type": "string", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + "description": "Exact non-negative decimal amount; no exponent notation." } } }, @@ -623,7 +672,13 @@ "required": ["meter", "minimum_input_tokens", "usd_per_token"], "properties": { "meter": { - "enum": ["input", "output", "input_cache_read", "input_cache_write"] + "enum": [ + "input", + "output", + "input_cache_read", + "input_cache_write", + "input_cache_write_1h" + ] }, "minimum_input_tokens": { "type": "integer", @@ -740,6 +795,87 @@ } } }, + "distribution": { + "description": "Official code and weight distribution facts. Availability does not establish a hosted API or permissions beyond the linked licenses.", + "type": "object", + "additionalProperties": false, + "required": ["source_url"], + "anyOf": [ + { + "required": ["code"] + }, + { + "required": ["weights"] + } + ], + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "code": { + "type": "object", + "additionalProperties": false, + "required": ["repository_url"], + "properties": { + "repository_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "license": { + "$ref": "#/$defs/artifact_license" + } + } + }, + "weights": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["url"], + "properties": { + "id": { + "description": "The exact published weight variant identifier, when available.", + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "license": { + "$ref": "#/$defs/artifact_license" + }, + "access": { + "description": "Public download or gated access as documented by the publisher; independent of license permissions.", + "enum": ["public", "gated"] + } + } + } + } + } + }, + "artifact_license": { + "type": "object", + "additionalProperties": false, + "required": ["name", "url"], + "properties": { + "name": { + "description": "The publisher's license name, including custom licenses; do not infer an SPDX identifier.", + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + } + } + }, "limits": { "type": "object", "additionalProperties": false, @@ -760,6 +896,193 @@ "minimum": 1 } } + }, + "provider_coverage": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["disposition", "source_url"], + "properties": { + "disposition": { + "enum": ["listed", "not_listed", "excluded"] + }, + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "reason": { + "type": "string", + "minLength": 1 + } + }, + "if": { + "properties": { + "disposition": { + "enum": ["not_listed", "excluded"] + } + } + }, + "then": { + "required": ["reason"] + } + } + }, + "published_price_component": { + "type": "object", + "additionalProperties": false, + "required": ["meter", "unit", "usd_per_unit"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "meter": { + "type": "string", + "minLength": 1 + }, + "unit": { + "type": "string", + "minLength": 1 + }, + "usd_per_unit": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]+)?$" + }, + "per_quantity": { + "type": "string", + "pattern": "^(?=.*[1-9])(0|[1-9][0-9]*)(\\.[0-9]+)?$" + }, + "when": { + "type": "object", + "description": "Public JSON Schema predicate restricting the request parameters to which this component applies." + }, + "quantity_rounding": { + "type": "object", + "additionalProperties": false, + "required": ["mode"], + "properties": { + "mode": { + "enum": ["none", "ceil"] + }, + "increment": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^(?=.*[1-9])(0|[1-9][0-9]*)(\\.[0-9]+)?$" + } + ] + } + }, + "if": { + "properties": { + "mode": { + "const": "ceil" + } + } + }, + "then": { + "required": ["increment"], + "properties": { + "increment": { + "type": "string", + "pattern": "^(?=.*[1-9])(0|[1-9][0-9]*)(\\.[0-9]+)?$" + } + } + } + }, + "quantity_range": { + "type": "object", + "additionalProperties": false, + "required": ["meter", "min"], + "properties": { + "meter": { + "type": "string", + "minLength": 1 + }, + "min": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]+)?$" + }, + "max": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^(?=.*[1-9])(0|[1-9][0-9]*)(\\.[0-9]+)?$" + } + ] + } + } + } + } + }, + "published_pricing": { + "type": "object", + "additionalProperties": false, + "required": ["source_url", "currency", "components"], + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "currency": { + "const": "USD" + }, + "components": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/published_price_component" + } + } + } + }, + "provider_binding": { + "type": "object", + "required": ["provider", "id"], + "properties": { + "provider": { + "type": "string" + }, + "id": { + "type": "string", + "minLength": 1 + }, + "distribution": false, + "endpoints": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/provider_endpoint" + } + }, + "capabilities": { + "$ref": "#/$defs/capabilities" + }, + "token_pricing": { + "$ref": "#/$defs/token_pricing" + }, + "limits": { + "$ref": "#/$defs/limits" + }, + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "published_pricing": { + "$ref": "#/$defs/published_pricing" + } + } } } } diff --git a/scripts/check.py b/scripts/check.py index ea33bc2..21e7761 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -24,12 +24,14 @@ import json import math import sys +from decimal import Decimal, InvalidOperation from fractions import Fraction from pathlib import Path from typing import Any, Iterator from urllib.parse import urlsplit from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError import build import cards @@ -91,10 +93,12 @@ def is_calendar_date(value: str) -> bool: SOURCED_OBJECTS = ( + "distribution", "output_token_calculator", "vision_tokenization", "capabilities", "token_pricing", + "published_pricing", "limits", ) @@ -228,8 +232,125 @@ def token_pricing_errors(pricing: dict[str, Any], where: str) -> Iterator[str]: yield f"{where}: overlapping input-token tiers for {meter}" +def distribution_errors(distribution: dict[str, Any], where: str) -> Iterator[str]: + code = distribution.get("code") + artifacts = [] + if isinstance(code, dict): + code_where = f"{where}/code" + yield from source_url_errors(code.get("repository_url"), f"{code_where}/repository_url") + artifacts.append((code_where, code)) + weights = distribution.get("weights", []) + seen_ids: set[str] = set() + seen_urls: dict[str, list[str | None]] = {} + if isinstance(weights, list): + for index, weight in enumerate(weights): + if not isinstance(weight, dict): + continue + weight_where = f"{where}/weights/{index}" + artifacts.append((weight_where, weight)) + url, variant = weight.get("url"), weight.get("id") + yield from source_url_errors(url, f"{weight_where}/url") + if isinstance(variant, str): + if variant in seen_ids: + yield f"{weight_where}: duplicate weight variant {variant!r}" + seen_ids.add(variant) + else: + variant = None + if isinstance(url, str): + previous = seen_urls.setdefault(url, []) + if previous and (variant is None or None in previous or variant in previous): + yield f"{weight_where}: duplicate weight URL without distinct variant IDs" + previous.append(variant) + for artifact_where, artifact in artifacts: + license = artifact.get("license") + if isinstance(license, dict): + yield from source_url_errors(license.get("url"), f"{artifact_where}/license/url") + + +def published_pricing_errors(pricing: dict[str, Any], where: str) -> Iterator[str]: + """Preserve exact public meter amounts without accepting malformed bands.""" + seen_ids = set() + components = pricing.get("components", []) + if not isinstance(components, list): + return + for index, component in enumerate(components): + if not isinstance(component, dict): + continue + component_where = f"{where}/components/{index}" + identifier = component.get("id") + if isinstance(identifier, str): + if identifier in seen_ids: + yield f"{component_where}: duplicate price component id {identifier!r}" + seen_ids.add(identifier) + condition = component.get("when") + if isinstance(condition, dict): + try: + json.dumps(condition, allow_nan=False) + Draft202012Validator.check_schema(condition) + except (SchemaError, TypeError, ValueError): + yield f"{component_where}: when must be a valid JSON Schema predicate" + rounding = component.get("quantity_rounding") + if isinstance(rounding, dict): + if rounding.get("mode") == "none" and rounding.get("increment") is not None: + yield f"{component_where}: no rounding requires a null increment" + if rounding.get("mode") == "ceil" and rounding.get("increment") is None: + yield f"{component_where}: ceil rounding requires an increment" + band = component.get("quantity_range") + if isinstance(band, dict) and band.get("max") is not None: + try: + minimum, maximum = Decimal(band.get("min")), Decimal(band.get("max")) + except (InvalidOperation, TypeError, ValueError): + continue # the schema reports malformed numbers + if minimum.is_finite() and maximum.is_finite() and minimum >= maximum: + yield f"{component_where}: exclusive range max must exceed min" + + +def provider_fact_errors(facts: dict[str, Any]) -> Iterator[str]: + for section, table in facts.items(): + if not isinstance(table, dict): + continue + entries = [(section, table)] if section in NON_TABLE_SECTIONS else [ + (f"{section}/{key}", card) for key, card in table.items() + ] + for where, card in entries: + if not isinstance(card, dict): + continue + providers = card.get("providers", {}) + providers = providers if isinstance(providers, dict) else {} + for name, binding in providers.items(): + if not isinstance(binding, dict): + continue + if binding.get("provider") != name: + yield f"{where}/providers/{name}: binding.provider is {binding.get('provider')!r}" + if not binding.get("id"): + yield f"{where}/providers/{name}: binding has no id" + if "source_url" in binding: + yield from source_url_errors(binding["source_url"], f"{where}/providers/{name}") + direct = card.get("provider") + reference = card.get("reference_provider") + if isinstance(reference, str) and reference not in providers and reference != direct: + yield f"{where}: reference_provider has no provider binding or direct provider" + coverage = card.get("provider_coverage", {}) + if not isinstance(coverage, dict): + continue + for provider, record in coverage.items(): + if not isinstance(record, dict): + continue + record_where = f"{where}/provider_coverage/{provider}" + yield from source_url_errors(record.get("source_url"), record_where) + listed = provider in providers or provider == direct + disposition = record.get("disposition") + if disposition == "listed" and not listed: + yield f"{record_where}: listed provider has no binding or direct provider" + if isinstance(disposition, str) and disposition in {"not_listed", "excluded"} and listed: + yield f"{record_where}: absent or excluded provider still has a binding" + + def published_fact_errors(facts: dict[str, Any]) -> Iterator[str]: for where, value in fact_objects(facts): + for key in ("source_url", "request_schema_url", "result_schema_url"): + if key in value: + yield from source_url_errors(value[key], f"{where}/{key}") endpoints = value.get("endpoints", []) seen_endpoint_ids = set() if isinstance(endpoints, list): @@ -249,10 +370,14 @@ def published_fact_errors(facts: dict[str, Any]) -> Iterator[str]: if "source_js_url" in sourced: yield from source_url_errors(sourced["source_js_url"], f"{sourced_where}/source_js_url") yield from numeric_range_errors(sourced, sourced_where) - if name == "output_token_calculator": + if name == "distribution": + yield from distribution_errors(sourced, sourced_where) + elif name == "output_token_calculator": yield from calculator_errors(sourced, sourced_where) elif name == "token_pricing": yield from token_pricing_errors(sourced, sourced_where) + elif name == "published_pricing": + yield from published_pricing_errors(sourced, sourced_where) if "parameters" in value or "image_constraints" in value: yield from source_url_errors(value.get("source_url"), where) parameters = value.get("parameters", {}) @@ -315,14 +440,8 @@ def invariant_errors(facts: dict[str, Any]) -> Iterator[str]: if not isinstance(providers, dict) or not providers: yield f"{where}: has no provider bindings" continue - for name, binding in providers.items(): - if not isinstance(binding, dict): - continue - if binding.get("provider") != name: - yield f"{where}/providers/{name}: binding.provider is {binding.get('provider')!r}" - if not binding.get("id"): - yield f"{where}/providers/{name}: binding has no id" + yield from provider_fact_errors(facts) yield from published_fact_errors(facts) for exception in sorted(KNOWN_EXCEPTIONS - used_exceptions): diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..4b94cf5 --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,264 @@ +"""Source and weight distribution facts belong to models, not hosted endpoints.""" + +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location("distribution_check", ROOT / "scripts" / "check.py") +check = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(check) + +CATALOGUE = check.load(check.MODELS_JSON) +SCHEMA = check.load(check.MODELS_SCHEMA) +MODEL_SECTIONS = tuple( + section for section in CATALOGUE if section not in {"$schema", "three_d.rig_check.operation"} +) +SOURCE = "https://model.example/releases/example" +CODE_URL = "https://github.com/example/model" +WEIGHTS_URL = "https://huggingface.co/example/model" +DISTRIBUTION = { + "source_url": SOURCE, + "code": { + "repository_url": CODE_URL, + "license": {"name": "MIT", "url": f"{CODE_URL}/blob/main/LICENSE"}, + }, + "weights": [ + { + "id": "example-4b", + "url": WEIGHTS_URL, + "license": {"name": "Example Weights License", "url": f"{WEIGHTS_URL}/blob/main/LICENSE"}, + "access": "gated", + } + ], +} +ENDPOINT = { + "id": "provider/example", + "input": "text", + "source_url": "https://provider.example/models/example/api", +} + + +def first(facts, section="image.models"): + return next(iter(facts[section].values())) + + +def validate(distribution=DISTRIBUTION, section="image.models"): + facts = copy.deepcopy(CATALOGUE) + first(facts, section)["distribution"] = copy.deepcopy(distribution) + return check.check(facts, SCHEMA) + + +def test_omitted_distribution_preserves_existing_cards(): + facts = copy.deepcopy(CATALOGUE) + for section in MODEL_SECTIONS: + for card in facts[section].values(): + card.pop("distribution", None) + assert check.check(facts, SCHEMA) == [] + + +@pytest.mark.parametrize("section", MODEL_SECTIONS) +def test_distribution_is_intrinsic_to_every_model_family(section): + assert validate(section=section) == [] + + +@pytest.mark.parametrize( + "distribution", + [ + {"source_url": SOURCE, "code": {"repository_url": CODE_URL}}, + {"source_url": SOURCE, "weights": [{"url": WEIGHTS_URL}]}, + { + "source_url": SOURCE, + "code": {"repository_url": CODE_URL}, + "weights": [{"url": WEIGHTS_URL}], + }, + ], +) +def test_code_weights_licenses_and_access_are_independent_optional_facts(distribution): + assert validate(distribution) == [] + + +@pytest.mark.parametrize("access", ["public", "gated"]) +def test_weight_access_records_published_access_conditions(access): + distribution = copy.deepcopy(DISTRIBUTION) + distribution["weights"][0]["access"] = access + assert validate(distribution) == [] + + +@pytest.mark.parametrize( + "distribution", + [ + {}, + {"source_url": SOURCE}, + {"code": {"repository_url": CODE_URL}}, + {"source_url": SOURCE, "code": {}}, + {"source_url": SOURCE, "weights": []}, + {"source_url": SOURCE, "weights": [{}]}, + {"source_url": SOURCE, "code": None}, + {"source_url": SOURCE, "weights": None}, + ], +) +def test_distribution_requires_provenance_and_at_least_one_usable_distribution(distribution): + assert validate(distribution) + + +@pytest.mark.parametrize("kind", ["code", "weights"]) +@pytest.mark.parametrize("missing", ["name", "url"]) +def test_present_license_requires_its_name_and_source_link(kind, missing): + distribution = copy.deepcopy(DISTRIBUTION) + target = distribution["code"] if kind == "code" else distribution["weights"][0] + target["license"].pop(missing) + assert validate(distribution) + + +@pytest.mark.parametrize( + "path", + [ + ("source_url",), + ("code", "repository_url"), + ("code", "license", "name"), + ("code", "license", "url"), + ("weights", 0, "id"), + ("weights", 0, "url"), + ("weights", 0, "license", "name"), + ("weights", 0, "license", "url"), + ], +) +def test_present_distribution_identifiers_and_links_cannot_be_empty(path): + distribution = copy.deepcopy(DISTRIBUTION) + target = distribution + for part in path[:-1]: + target = target[part] + target[path[-1]] = "" + assert validate(distribution) + + +@pytest.mark.parametrize( + "path", + [ + ("source_url",), + ("code", "repository_url"), + ("code", "license", "url"), + ("weights", 0, "url"), + ("weights", 0, "license", "url"), + ], +) +@pytest.mark.parametrize( + "url", + [ + "http://model.example/artifact", + "/artifacts/model", + "https://", + "https://user:password@model.example/artifact", + "https://user@model.example/artifact", + ], +) +def test_every_distribution_link_is_absolute_https_without_credentials(path, url): + distribution = copy.deepcopy(DISTRIBUTION) + target = distribution + for part in path[:-1]: + target = target[part] + target[path[-1]] = url + assert validate(distribution) + + +@pytest.mark.parametrize("access", ["open", "private", "unknown", "", True]) +def test_weight_access_does_not_infer_other_permissions(access): + distribution = copy.deepcopy(DISTRIBUTION) + distribution["weights"][0]["access"] = access + assert validate(distribution) + + +@pytest.mark.parametrize( + "path, key", + [ + ((), "open_source"), + ((), "commercial_use"), + (("code",), "commercial_use"), + (("code", "license"), "approved"), + (("weights", 0), "commercial_use"), + (("weights", 0, "license"), "approved"), + ], +) +def test_distribution_objects_do_not_accept_inferred_rights_or_arbitrary_fields(path, key): + distribution = copy.deepcopy(DISTRIBUTION) + target = distribution + for part in path: + target = target[part] + target[key] = True + assert validate(distribution) + + +def test_distinct_exact_variants_can_share_one_weights_repository(): + distribution = { + "source_url": SOURCE, + "weights": [ + {"id": "example-4b", "url": WEIGHTS_URL}, + {"id": "example-8b", "url": WEIGHTS_URL}, + ], + } + assert validate(distribution) == [] + + +def test_distinct_weights_repositories_do_not_require_invented_variant_ids(): + distribution = { + "source_url": SOURCE, + "weights": [ + {"url": f"{WEIGHTS_URL}-small"}, + {"url": f"{WEIGHTS_URL}-large"}, + ], + } + assert validate(distribution) == [] + + +def test_exact_variant_ids_cannot_repeat_even_with_different_repository_urls(): + distribution = { + "source_url": SOURCE, + "weights": [ + {"id": "example-4b", "url": WEIGHTS_URL}, + {"id": "example-4b", "url": f"{WEIGHTS_URL}-mirror"}, + ], + } + assert validate(distribution) + + +@pytest.mark.parametrize("ids", [(None, None), (None, "example-4b"), ("example-4b", None)]) +def test_repeated_weights_repository_requires_distinct_explicit_variant_ids(ids): + weights = [{"url": WEIGHTS_URL}, {"url": WEIGHTS_URL}] + for weight, variant_id in zip(weights, ids): + if variant_id is not None: + weight["id"] = variant_id + assert validate({"source_url": SOURCE, "weights": weights}) + + +def test_distribution_is_not_a_hosted_provider_binding_fact(): + facts = copy.deepcopy(CATALOGUE) + binding = next(iter(first(facts)["providers"].values())) + binding["distribution"] = copy.deepcopy(DISTRIBUTION) + assert check.check(facts, SCHEMA) + + +@pytest.mark.parametrize("placement", ["binding", "model", "operation"]) +def test_distribution_cannot_be_attached_to_endpoint_variants(placement): + facts = copy.deepcopy(CATALOGUE) + if placement == "binding": + target = next(iter(first(facts)["providers"].values())) + elif placement == "model": + target = first(facts, "three_d.models") + else: + target = facts["three_d.rig_check.operation"] + endpoint = copy.deepcopy(ENDPOINT) + endpoint["distribution"] = copy.deepcopy(DISTRIBUTION) + target["endpoints"] = [endpoint] + assert check.check(facts, SCHEMA) + + +def test_distribution_is_not_a_rig_check_operation_fact(): + facts = copy.deepcopy(CATALOGUE) + facts["three_d.rig_check.operation"]["distribution"] = copy.deepcopy(DISTRIBUTION) + assert check.check(facts, SCHEMA) diff --git a/tests/test_provider_coverage.py b/tests/test_provider_coverage.py new file mode 100644 index 0000000..b10eb91 --- /dev/null +++ b/tests/test_provider_coverage.py @@ -0,0 +1,234 @@ +"""Provider facts apply uniformly to model families and preserve public price meaning.""" + +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location("coverage_check", ROOT / "scripts" / "check.py") +check = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(check) +CATALOGUE = check.load(check.MODELS_JSON) +SCHEMA = check.load(check.MODELS_SCHEMA) +SECTIONS = tuple(section for section in CATALOGUE if section != "$schema") +SOURCE = "https://provider.example/models" +BINDING = {"id": "test-model", "provider": "example", "source_url": SOURCE} +PRICE = { + "source_url": SOURCE, + "currency": "USD", + "components": [{ + "id": "input-band", "meter": "input_tokens", "unit": "token", + "usd_per_unit": "0.0000015", "per_quantity": "1", + "when": {"properties": {"quality": {"enum": ["standard", "high"]}}}, + "quantity_rounding": {"mode": "none", "increment": None}, + "quantity_range": {"meter": "input_tokens", "min": "0", "max": "1000"}, + }], +} + + +def mutate(callback, section="image.models"): + facts = copy.deepcopy(CATALOGUE) + card = facts[section] if section in check.NON_TABLE_SECTIONS else next(iter(facts[section].values())) + callback(card) + return check.check(facts, SCHEMA) + + +def add_binding(card): + card.setdefault("providers", {})["example"] = copy.deepcopy(BINDING) + card["reference_provider"] = "example" + card.setdefault("provider_coverage", {})["example"] = { + "disposition": "listed", "source_url": SOURCE, + } + + +@pytest.mark.parametrize("section", SECTIONS) +def test_reference_and_coverage_are_valid_in_every_family_and_operation(section): + assert mutate(add_binding, section) == [] + + +@pytest.mark.parametrize("section", SECTIONS) +def test_provider_key_must_match_even_outside_image_and_video(section): + def edit(card): + add_binding(card) + card["providers"]["example"]["provider"] = "different" + assert any("binding.provider" in error for error in mutate(edit, section)) + + +def test_reference_resolves_existing_native_direct_provider_without_duplicate_binding(): + def edit(card): + card["reference_provider"] = card["provider"] + assert mutate(edit, "three_d.rig_check.operation") == [] + + +def test_reference_does_not_create_an_invented_provider(): + assert any("reference_provider" in error for error in mutate(lambda card: card.update(reference_provider="invented"))) + + +@pytest.mark.parametrize("disposition", ["not_listed", "excluded"]) +def test_absence_requires_a_public_reason_and_cannot_coexist_with_binding(disposition): + def no_reason(card): + card["provider_coverage"] = {"unlisted": {"disposition": disposition, "source_url": SOURCE}} + assert mutate(no_reason) + def contradictory(card): + add_binding(card) + card["provider_coverage"]["example"].update(disposition=disposition, reason="Documented incompatible contract") + assert any("still has a binding" in error for error in mutate(contradictory)) + def valid(card): + card["provider_coverage"] = {"unlisted": { + "disposition": disposition, "source_url": SOURCE, "reason": "Published endpoint uses a different model identity", + }} + assert mutate(valid) == [] + + +def test_listed_coverage_cannot_replace_a_missing_binding(): + def edit(card): + card["provider_coverage"] = {"invented": {"disposition": "listed", "source_url": SOURCE}} + assert any("no binding" in error for error in mutate(edit)) + + +@pytest.mark.parametrize("url", ["http://example.com", "https://user:password@example.com", "/models"]) +def test_coverage_and_binding_sources_reject_unsafe_or_relative_links(url): + def edit(card): + add_binding(card) + card["provider_coverage"]["example"]["source_url"] = url + assert mutate(edit) + def binding(card): + add_binding(card) + card["providers"]["example"]["source_url"] = url + assert mutate(binding) + + +def price_mutation(callback=lambda price: None, *, endpoint=False): + def edit(card): + add_binding(card) + target = card["providers"]["example"] + if endpoint: + target["endpoints"] = [{"id": "generate", "operation": "image.generate", "interface": "openai.images", "input": "multi_image", "source_url": SOURCE}] + target = target["endpoints"][0] + target["published_pricing"] = copy.deepcopy(PRICE) + callback(target["published_pricing"]) + return mutate(edit) + + +@pytest.mark.parametrize("endpoint", [False, True]) +def test_public_usd_components_are_exact_and_provider_neutral(endpoint): + assert price_mutation(endpoint=endpoint) == [] + + +@pytest.mark.parametrize("value", [1.5, "-1", "NaN", "1e-5", "0.0.1"]) +def test_published_rates_use_nonnegative_decimal_strings(value): + assert price_mutation(lambda price: price["components"][0].update(usd_per_unit=value)) + + +@pytest.mark.parametrize("value", ["0", "0.0", "-1", 1]) +def test_price_denominator_must_be_a_positive_decimal(value): + assert price_mutation(lambda price: price["components"][0].update(per_quantity=value)) + + +@pytest.mark.parametrize("band", [{"min": "1000", "max": "1000"}, {"min": "1001", "max": "1000"}]) +def test_half_open_price_bands_must_have_positive_width(band): + assert price_mutation(lambda price: price["components"][0]["quantity_range"].update(band)) + + +def test_unbounded_price_band_remains_valid(): + assert price_mutation(lambda price: price["components"][0]["quantity_range"].update(max=None)) == [] + + +@pytest.mark.parametrize("rounding", [{"mode": "none", "increment": "1"}, {"mode": "ceil", "increment": None}]) +def test_rounding_mode_and_increment_must_agree(rounding): + assert price_mutation(lambda price: price["components"][0].update(quantity_rounding=rounding)) + + +def test_ceil_quantity_rounding_preserves_its_positive_increment(): + assert price_mutation(lambda price: price["components"][0].update(quantity_rounding={"mode": "ceil", "increment": "1"})) == [] + + +def test_duplicate_component_identity_rejected(): + assert price_mutation(lambda price: price["components"].append(copy.deepcopy(price["components"][0]))) + + +@pytest.mark.parametrize("extra", ["margin", "credits_per_dollar", "promotion_end", "account_price"]) +def test_published_pricing_does_not_accept_business_or_promotion_fields(extra): + assert price_mutation(lambda price: price.update({extra: "example"})) + + +def test_public_price_requires_currency_source_and_components(): + for field in ["source_url", "currency", "components"]: + assert price_mutation(lambda price: price.pop(field)) + + +def test_price_conditions_are_valid_json_schema_predicates(): + assert price_mutation(lambda price: price["components"][0].update(when={"properties": {"quality": {"const": "high"}}})) == [] + assert price_mutation(lambda price: price["components"][0].update(when={"type": "not-a-json-type"})) + + +@pytest.mark.parametrize("field", ["request_schema_url", "result_schema_url"]) +def test_public_contract_links_are_https_without_credentials(field): + def edit(card): + add_binding(card) + card["providers"]["example"]["endpoints"] = [{ + "id": "generate", "input": "image", "source_url": SOURCE, + field: "https://user:password@example.com/schema", + }] + assert mutate(edit) + + +def test_cache_write_ttls_remain_separate_prices(): + def edit(card): + add_binding(card) + card["providers"]["example"]["token_pricing"] = { + "source_url": SOURCE, "unit": "usd_per_token", "scope": "Standard service; cache writes 5 minutes and 1 hour", + "input_cache_write": "0.00000125", "input_cache_write_1h": "0.000002", + } + assert mutate(edit) == [] + + +def test_toml_omission_preserves_unbounded_range_and_no_rounding(): + """Arrays need no synthetic nulls: TOML omits only semantically optional fields.""" + import tomllib + pricing = tomllib.loads(''' +source_url = "https://provider.example/models" +currency = "USD" +[[components]] +meter = "input_tokens" +unit = "token" +usd_per_unit = "0.000005" +[components.quantity_rounding] +mode = "none" +[components.quantity_range] +meter = "input_tokens" +min = "1000" +''') + def replace(price): + price.clear() + price.update(pricing) + assert price_mutation(replace, endpoint=True) == [] + assert "max" not in pricing["components"][0]["quantity_range"] + assert "increment" not in pricing["components"][0]["quantity_rounding"] + + +def test_missing_component_rate_is_unknown_not_zero(): + assert price_mutation(lambda price: price["components"][0].pop("usd_per_unit")) + assert mutate(add_binding) == [] # known provider can omit the entire unknown price + + +def test_ceil_cannot_omit_increment(): + assert price_mutation(lambda price: price["components"][0].update(quantity_rounding={"mode": "ceil"})) + + +@pytest.mark.parametrize("disposition", [[], {}, 1, None]) +def test_malformed_coverage_disposition_reports_errors_without_crashing(disposition): + def edit(card): + add_binding(card) + card["provider_coverage"]["example"]["disposition"] = disposition + assert mutate(edit) + + +@pytest.mark.parametrize("number", [float("nan"), float("inf"), -float("inf")]) +def test_price_conditions_are_finite_json(number): + assert price_mutation(lambda price: price["components"][0].update(when={"properties": {"quantity": {"maximum": number}}})) From f87a5402da142de8e1a46808840619cc0d3d58f9 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 21:07:00 +0900 Subject: [PATCH 4/4] Validate sourced provider qualifications --- catalogue/README.md | 8 ++++++-- catalogue/schemas/models.schema.json | 20 ++++++++++++++++++++ scripts/check.py | 8 ++++++++ tests/test_provider_coverage.py | 20 ++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/catalogue/README.md b/catalogue/README.md index 867938f..86c8add 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -122,7 +122,9 @@ Every family, including embedding and the rig-check operation, may add a `providers` record without changing its existing ID or direct-provider fields. Each binding has exact `provider` and `id` values, an optional first-party `source_url`, and independently sourced serving facts. A verified binding can -omit an unknown regular price; omission never means free inference. +omit an unknown regular price; omission never means free inference. Optional +`qualifications` records nonempty `{ source_url, notes }` entries for unresolved +pricing or specification facts, without inventing a replacement value. Optional `reference_provider` names a real native API binding, or the existing direct `provider` when that card describes the native API already. Consumers may @@ -135,7 +137,9 @@ a native hosted API. This field records a sourced reference, not a provider regi - `listed` requires a binding or matching direct provider. - `not_listed` records verified absence from that provider's published catalogue. - `excluded` records an incompatible or unverifiable specification, identity or - pricing contract that makes a reliable listing inappropriate. + pricing contract that makes a reliable listing inappropriate, or an explicit + modality scope such as fal media-only coverage. A scope exclusion is not a claim + that the provider has no such model. The last two require a public, source-backed `reason` and cannot coexist with a binding for that provider. Do not use an exclusion for routing preference or diff --git a/catalogue/schemas/models.schema.json b/catalogue/schemas/models.schema.json index 103f7ff..53ec55d 100644 --- a/catalogue/schemas/models.schema.json +++ b/catalogue/schemas/models.schema.json @@ -1081,6 +1081,26 @@ }, "published_pricing": { "$ref": "#/$defs/published_pricing" + }, + "qualifications": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["source_url", "notes"], + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "notes": { + "type": "string", + "minLength": 1 + } + } + } } } } diff --git a/scripts/check.py b/scripts/check.py index 21e7761..532f02d 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -326,6 +326,14 @@ def provider_fact_errors(facts: dict[str, Any]) -> Iterator[str]: yield f"{where}/providers/{name}: binding has no id" if "source_url" in binding: yield from source_url_errors(binding["source_url"], f"{where}/providers/{name}") + qualifications = binding.get("qualifications", []) + if isinstance(qualifications, list): + for index, qualification in enumerate(qualifications): + if isinstance(qualification, dict): + yield from source_url_errors( + qualification.get("source_url"), + f"{where}/providers/{name}/qualifications/{index}", + ) direct = card.get("provider") reference = card.get("reference_provider") if isinstance(reference, str) and reference not in providers and reference != direct: diff --git a/tests/test_provider_coverage.py b/tests/test_provider_coverage.py index b10eb91..6de3e89 100644 --- a/tests/test_provider_coverage.py +++ b/tests/test_provider_coverage.py @@ -232,3 +232,23 @@ def edit(card): @pytest.mark.parametrize("number", [float("nan"), float("inf"), -float("inf")]) def test_price_conditions_are_finite_json(number): assert price_mutation(lambda price: price["components"][0].update(when={"properties": {"quantity": {"maximum": number}}})) + + +@pytest.mark.parametrize("qualification", [ + {"notes": "Regular rate is not established"}, + {"source_url": SOURCE}, + {"source_url": "https://token:secret@example.com", "notes": "Unknown rate"}, + {"source_url": SOURCE, "notes": "Unknown rate", "guessed_price": 1}, +]) +def test_unknown_provider_fact_qualifications_need_a_public_source(qualification): + def edit(card): + add_binding(card) + card["providers"]["example"]["qualifications"] = [qualification] + assert mutate(edit) + + +def test_verified_provider_can_qualify_missing_regular_price(): + def edit(card): + add_binding(card) + card["providers"]["example"]["qualifications"] = [{"source_url": SOURCE, "notes": "The public endpoint is documented but no regular rate is published."}] + assert mutate(edit) == []