From 2b14448b6422d06278d38d37a0fadaf730f938e7 Mon Sep 17 00:00:00 2001 From: Samir Hanna Verza Date: Sun, 6 Sep 2026 11:53:24 -0300 Subject: [PATCH 1/2] fix(pricing): prefer a native provider over aggregator re-listings resolve_litellm_model() tie-breaks a bare model name that several LiteLLM providers list by returning the alphabetically first key whenever those listings agree on price. As LiteLLM's model_cost has grown, aggregators that re-key a model under a namespaced path now sort ahead of the model's own first-party provider: with litellm 1.90.1, "grok-4.5" resolves to "openrouter/x-ai/grok-4.5" instead of "xai/grok-4.5". The estimate stays correct (same price), but the reported provider is wrong, and it breaks test_resolves_common_bare_model_names on the pinned litellm. Prefer the first-party listing: the match whose key is exactly "/". When that is unique, return it before falling back to price consensus. This restores native-provider attribution and also resolves names the old rule gave up on when aggregators disagreed on price (e.g. a unique first-party listing among differently priced mirrors). Add deterministic, mock-backed regression tests for both cases so the behavior no longer rides on live LiteLLM catalog data. Co-Authored-By: Claude Opus 4.8 --- strix/report/pricing.py | 20 ++++++++++++++++- tests/test_pricing.py | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/strix/report/pricing.py b/strix/report/pricing.py index 57c8995921..dc4c89388a 100644 --- a/strix/report/pricing.py +++ b/strix/report/pricing.py @@ -7,7 +7,9 @@ @lru_cache(maxsize=512) -def resolve_litellm_model(model: str) -> str | None: +def resolve_litellm_model( # noqa: PLR0911 - each resolution path returns its own name + model: str, +) -> str | None: """Return a provider-qualified model name that LiteLLM can price.""" try: import litellm @@ -39,6 +41,22 @@ def resolve_litellm_model(model: str) -> str | None: matches = sorted(key for key in model_cost if key.endswith(f"/{name}")) if not matches: continue + # Prefer the first-party provider's own listing over an aggregator's + # re-listing of the same model. Aggregators (OpenRouter, DeepInfra, + # Perplexity, ...) re-key a model under a namespaced path such as + # "openrouter/x-ai/grok-4.5"; the native listing is the one whose key + # is exactly "/" ("xai/grok-4.5"). When that + # is unique it is the canonical name, so tie-break on it before + # falling back to price consensus (which would otherwise pick the + # alphabetically first key, an aggregator). + native = [ + key + for key in matches + if isinstance(model_cost.get(key), dict) + and key == f"{model_cost[key].get('litellm_provider')}/{name}" + ] + if len(native) == 1: + return native[0] prices = { ( model_cost[key].get("input_cost_per_token"), diff --git a/tests/test_pricing.py b/tests/test_pricing.py index c9e9e5dfb1..5170abfa41 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -121,3 +121,52 @@ def test_resolver_does_not_guess_between_differently_priced_providers() -> None: finally: litellm.model_cost = original resolve_litellm_model.cache_clear() + + +def test_resolver_prefers_native_provider_over_aggregator() -> None: + # The same model re-listed by an aggregator (which sorts first) and by its + # first-party provider at the same price. The resolver must return the + # native listing, not the alphabetically first aggregator key. + original = litellm.model_cost + litellm.model_cost = { + "openrouter/x-ai/grok-4.5": { + "litellm_provider": "openrouter", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + }, + "xai/grok-4.5": { + "litellm_provider": "xai", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + }, + } + try: + resolve_litellm_model.cache_clear() + assert resolve_litellm_model("grok-4.5") == "xai/grok-4.5" + finally: + litellm.model_cost = original + resolve_litellm_model.cache_clear() + + +def test_resolver_prefers_unique_native_provider_over_differently_priced_aggregators() -> None: + # A unique first-party listing wins even when aggregators price the model + # differently, where price consensus alone would give up and return None. + original = litellm.model_cost + litellm.model_cost = { + "deepinfra/x/example-model": { + "litellm_provider": "deepinfra", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "example/example-model": { + "litellm_provider": "example", + "input_cost_per_token": 5e-06, + "output_cost_per_token": 9e-06, + }, + } + try: + resolve_litellm_model.cache_clear() + assert resolve_litellm_model("example-model") == "example/example-model" + finally: + litellm.model_cost = original + resolve_litellm_model.cache_clear() From 61baca5336651423e72489b5064d95d409861c38 Mon Sep 17 00:00:00 2001 From: Samir Hanna Verza Date: Fri, 16 Oct 2026 17:47:33 -0300 Subject: [PATCH 2/2] fix(pricing): only prefer the native route when every route prices the model the same Review feedback on #1279: returning the native listing before the price check turned bare names whose routes disagree on price from None into a guess (91 of 1993 bare names on litellm 1.101). Move the preference inside the branch that already accepts the match (one route, or all routes at one price), so it only breaks ties the resolver was willing to resolve and changes nothing for differently priced routes. The guard test now uses fixtures with litellm_provider, as every real catalog entry has, and the test that asserted the wide behavior asserts None instead. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01TavwVMH7PvrD68JTDfd6Hb --- strix/report/pricing.py | 33 +++++++++++++-------------------- tests/test_pricing.py | 13 +++++++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/strix/report/pricing.py b/strix/report/pricing.py index dc4c89388a..9f449f2fec 100644 --- a/strix/report/pricing.py +++ b/strix/report/pricing.py @@ -7,9 +7,7 @@ @lru_cache(maxsize=512) -def resolve_litellm_model( # noqa: PLR0911 - each resolution path returns its own name - model: str, -) -> str | None: +def resolve_litellm_model(model: str) -> str | None: """Return a provider-qualified model name that LiteLLM can price.""" try: import litellm @@ -41,22 +39,6 @@ def resolve_litellm_model( # noqa: PLR0911 - each resolution path returns its o matches = sorted(key for key in model_cost if key.endswith(f"/{name}")) if not matches: continue - # Prefer the first-party provider's own listing over an aggregator's - # re-listing of the same model. Aggregators (OpenRouter, DeepInfra, - # Perplexity, ...) re-key a model under a namespaced path such as - # "openrouter/x-ai/grok-4.5"; the native listing is the one whose key - # is exactly "/" ("xai/grok-4.5"). When that - # is unique it is the canonical name, so tie-break on it before - # falling back to price consensus (which would otherwise pick the - # alphabetically first key, an aggregator). - native = [ - key - for key in matches - if isinstance(model_cost.get(key), dict) - and key == f"{model_cost[key].get('litellm_provider')}/{name}" - ] - if len(native) == 1: - return native[0] prices = { ( model_cost[key].get("input_cost_per_token"), @@ -66,7 +48,18 @@ def resolve_litellm_model( # noqa: PLR0911 - each resolution path returns its o if isinstance(model_cost.get(key), dict) } if len(matches) == 1 or len(prices) == 1: - return matches[0] + # Every route prices the model the same, so picking one is free; pick + # the first-party listing over an aggregator's re-listing. Aggregators + # (OpenRouter, DeepInfra, ...) re-key a model under a namespaced path + # such as "openrouter/x-ai/grok-4.5", which sorts ahead of the native + # "xai/grok-4.5", so the alphabetically first key would be misleading. + native = [ + key + for key in matches + if isinstance(model_cost.get(key), dict) + and key == f"{model_cost[key].get('litellm_provider')}/{name}" + ] + return native[0] if len(native) == 1 else matches[0] return None # noqa: TRY300 except Exception: # noqa: BLE001 return None diff --git a/tests/test_pricing.py b/tests/test_pricing.py index 5170abfa41..7216f8a1aa 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -105,12 +105,16 @@ def test_resolver_uses_provider_when_bare_entry_has_one() -> None: def test_resolver_does_not_guess_between_differently_priced_providers() -> None: original = litellm.model_cost + # Real catalog entries always carry ``litellm_provider``; without it the native + # route could never match and this guard would not exercise the preference. litellm.model_cost = { "provider-a/example": { + "litellm_provider": "provider-a", "input_cost_per_token": 1.0, "output_cost_per_token": 2.0, }, "provider-b/example": { + "litellm_provider": "provider-b", "input_cost_per_token": 3.0, "output_cost_per_token": 4.0, }, @@ -148,9 +152,10 @@ def test_resolver_prefers_native_provider_over_aggregator() -> None: resolve_litellm_model.cache_clear() -def test_resolver_prefers_unique_native_provider_over_differently_priced_aggregators() -> None: - # A unique first-party listing wins even when aggregators price the model - # differently, where price consensus alone would give up and return None. +def test_resolver_does_not_guess_when_a_native_route_exists_but_prices_differ() -> None: + # The native preference only breaks ties between routes that price the model the + # same. When an aggregator charges differently, the cost depends on the route and + # the resolver still declines to guess, as it did before the native preference. original = litellm.model_cost litellm.model_cost = { "deepinfra/x/example-model": { @@ -166,7 +171,7 @@ def test_resolver_prefers_unique_native_provider_over_differently_priced_aggrega } try: resolve_litellm_model.cache_clear() - assert resolve_litellm_model("example-model") == "example/example-model" + assert resolve_litellm_model("example-model") is None finally: litellm.model_cost = original resolve_litellm_model.cache_clear()