fix(ai): Use provider-specific model pricing

CHANGELOG.md

@@ -11,6 +11,7 @@
 
 **Bug Fixes**:
 
+- Use provider-specific AI model prices and context sizes when available, rather than another provider's metadata for the same model. ([sentry#128011](https://github.com/getsentry/sentry/issues/128011))
 - Use the dynamic sampling context to fill missing trace IDs on errors, preserving existing trace and span IDs. ([#6508](https://github.com/getsentry/relay/pull/6508))
 - Fix Perfetto sample timestamps recorded in a clock other than the primary trace clock. ([#6522](https://github.com/getsentry/relay/pull/6522))
 - Remove invalid `sentry.release` attributes from spans instead of forwarding them. ([#6547](https://github.com/getsentry/relay/pull/6547))

relay-event-normalization/src/eap/ai.rs

@@ -126,7 +126,9 @@ fn normalize_context_utilization(
         .get_value(GEN_AI__RESPONSE__MODEL)
         .and_then(|v| v.as_str());
 
-    let context_size = model_id.and_then(|id| model_metadata.and_then(|m| m.context_size(id)));
+    let provider = extract_string_value(attributes, GEN_AI__PROVIDER__NAME);
+    let context_size =
+        model_id.and_then(|id| model_metadata.and_then(|m| m.context_size(id, provider)));
 
     let Some(context_size) = context_size else {
         return;
@@ -173,7 +175,8 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_metadata: Option<&Model
         return;
     };
 
-    let Some(model_cost) = model_metadata.and_then(|m| m.cost_per_token(model_id)) else {
+    let provider = extract_string_value(attributes, GEN_AI__PROVIDER__NAME);
+    let Some(model_cost) = model_metadata.and_then(|m| m.cost_per_token(model_id, provider)) else {
         relay_statsd::metric!(
             counter(Counters::GenAiCostCalculationResult) += 1,
             result = "calculation_no_model_cost_available",
@@ -273,6 +276,7 @@ mod tests {
     fn model_metadata() -> ModelMetadata {
         ModelMetadata {
             version: 1,
+            providers: HashMap::new(),
             models: HashMap::from([
                 (
                     "claude-2.1".parse().unwrap(),
@@ -307,6 +311,7 @@ mod tests {
     fn model_metadata_with_context_size() -> ModelMetadata {
         ModelMetadata {
             version: 1,
+            providers: HashMap::new(),
             models: HashMap::from([(
                 "claude-2.1".parse().unwrap(),
                 ModelMetadataEntry {
@@ -323,6 +328,54 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_normalize_ai_provider_prices() {
+        let entry = |input_per_token| {
+            serde_json::json!({
+                "costs": {
+                    "inputPerToken": input_per_token,
+                    "outputPerToken": 0.0,
+                    "outputReasoningPerToken": 0.0,
+                    "inputCachedPerToken": 0.0,
+                    "inputCacheWritePerToken": 0.0
+                }
+            })
+        };
+        let metadata: ModelMetadata = serde_json::from_value(serde_json::json!({
+            "version": 1,
+            "models": {"clef": entry(0.042 / 1_000_000.0)},
+            "providers": {
+                "cloudflare-workers-ai": {"clef": entry(0.24 / 1_000_000.0)},
+                "openrouter": {"cloudflare/clef": entry(0.042 / 1_000_000.0)},
+                "anthropic": {"claude-sonnet-4": entry(3.0 / 1_000_000.0)},
+                "poe": {"claude-sonnet-4": entry(2.6 / 1_000_000.0)}
+            }
+        }))
+        .unwrap();
+        for (provider, model, expected) in [
+            ("cloudflare.workers_ai", "clef", 0.01323168),
+            ("openrouter", "cloudflare/clef", 0.002315544),
+            ("anthropic", "claude-sonnet-4", 0.165396),
+            ("poe", "claude-sonnet-4", 0.1433432),
+        ] {
+            let mut attributes = Annotated::new(attributes! {
+                "gen_ai.operation.name" => "chat".to_owned(),
+                "gen_ai.provider.name" => provider.to_owned(),
+                "gen_ai.response.model" => model.to_owned(),
+                "gen_ai.usage.input_tokens" => 55_132,
+            });
+            normalize_ai(&mut attributes, None, Some(&metadata));
+            let cost = attributes
+                .value()
+                .unwrap()
+                .get_value(GEN_AI__COST__TOTAL_TOKENS)
+                .unwrap()
+                .as_f64()
+                .unwrap();
+            assert!((cost - expected).abs() < 1e-10, "{provider}: {cost}");
+        }
+    }
+
     #[test]
     fn test_normalize_ai_type_fallback() {
         for model_key in [

relay-event-normalization/src/event.rs

@@ -2525,6 +2525,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::from([
                         (
                             "claude-2.1".parse().unwrap(),
@@ -2656,6 +2657,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::from([
                         (
                             "claude-2.1".parse().unwrap(),
@@ -2773,6 +2775,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::from([(
                         "claude-2.1".parse().unwrap(),
                         ModelMetadataEntry {
@@ -2848,6 +2851,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::from([
                         (
                             "claude-2.1".parse().unwrap(),
@@ -2947,6 +2951,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::new(),
                 }),
                 ..NormalizationConfig::default()
@@ -2991,6 +2996,7 @@ mod tests {
             &NormalizationConfig {
                 ai_model_metadata: Some(&ModelMetadata {
                     version: 1,
+                    providers: HashMap::new(),
                     models: HashMap::new(),
                 }),
                 ..NormalizationConfig::default()

relay-event-normalization/src/normalize/mod.rs

@@ -299,14 +299,20 @@ pub struct ModelMetadata {
     /// The mappings of model ID => metadata as a dictionary.
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub models: HashMap<ModelPattern, ModelMetadataEntry>,
+
+    /// Provider-specific metadata, keyed by models.dev provider IDs.
+    /// Older configurations omit this and use the model-only table.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub providers: HashMap<String, HashMap<ModelPattern, ModelMetadataEntry>>,
 }
 
 impl ModelMetadata {
     const SUPPORTED_VERSION: u16 = 1;
 
     /// `true` if the model metadata is empty or the version is unsupported.
     pub fn is_empty(&self) -> bool {
-        self.models.is_empty() || !self.is_enabled()
+        (self.models.is_empty() && self.providers.values().all(HashMap::is_empty))
+            || !self.is_enabled()
     }
 
     /// `false` if the version is unsupported.
@@ -315,40 +321,62 @@ impl ModelMetadata {
     }
 
     /// Gets the cost per token for a given model, if defined.
-    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
-        self.get(model_id).and_then(|entry| entry.costs.as_ref())
+    pub fn cost_per_token(&self, model_id: &str, provider: Option<&str>) -> Option<&ModelCostV2> {
+        self.get(model_id, provider)
+            .and_then(|entry| entry.costs.as_ref())
     }
 
     /// Gets the context window size for a given model, if defined.
     ///
     /// Returns `None` for a zero context size, as it is not a meaningful value.
-    pub fn context_size(&self, model_id: &str) -> Option<u64> {
-        self.get(model_id)
+    pub fn context_size(&self, model_id: &str, provider: Option<&str>) -> Option<u64> {
+        self.get(model_id, provider)
             .and_then(|entry| entry.context_size)
             .filter(|&size| size > 0)
     }
 
     /// Gets the metadata for a given model, if defined.
-    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
+    ///
+    /// With provider tables available, never fall back to another provider's
+    /// prices. Retain the legacy lookup for old configs or provider-less spans.
+    pub fn get(&self, model_id: &str, provider: Option<&str>) -> Option<&ModelMetadataEntry> {
         if !self.is_enabled() {
             return None;
         }
 
+        let models = match provider.filter(|provider| !provider.is_empty()) {
+            Some(provider) if !self.providers.is_empty() => {
+                // Map OpenTelemetry provider names to models.dev catalog IDs.
+                let provider = match provider {
+                    "cloudflare.workers_ai" => "cloudflare-workers-ai",
+                    "gcp.gemini" => "google",
+                    "gcp.vertex_ai" => "google-vertex",
+                    "aws.bedrock" => "amazon-bedrock",
+                    "azure.ai.openai" => "azure",
+                    provider => provider,
+                };
+                self.providers.get(provider)?
+            }
+            _ => &self.models,
+        };
         let normalized_model_id = normalize_ai_model_name(model_id);
 
-        // First try exact match.
-        if let Some(value) = self.models.get(normalized_model_id) {
+        // Prefer full IDs, including snapshots, before normalized aliases.
+        if let Some(value) = models
+            .get(model_id)
+            .or_else(|| models.get(normalized_model_id))
+        {
             return Some(value);
         }
 
-        // Fall back to glob matching.
-        self.models.iter().find_map(|(key, value)| {
-            if key.is_match(normalized_model_id) {
-                Some(value)
-            } else {
-                None
-            }
-        })
+        // Prefer the most specific matching pattern, independent of map order.
+        models
+            .iter()
+            .filter(|(key, _)| key.is_match(normalized_model_id))
+            .max_by(|(a, _), (b, _)| {
+                (a.original.len(), &a.original).cmp(&(b.original.len(), &b.original))
+            })
+            .map(|(_, value)| value)
     }
 }
 
@@ -459,6 +487,100 @@ mod tests {
 
     use super::*;
 
+    #[test]
+    fn test_provider_metadata_lookup() {
+        let mut metadata: ModelMetadata = serde_json::from_value(json!({
+            "version": 1,
+            "models": {"clef": {"contextSize": 100}},
+            "providers": {
+                "cloudflare-workers-ai": {
+                    "clef": {"contextSize": 200},
+                    "clef-20240101": {"contextSize": 300},
+                    "*clef": {"contextSize": 200},
+                    "*author/clef": {"contextSize": 400}
+                },
+                "openrouter": {"cloudflare/clef": {"contextSize": 500}}
+            }
+        }))
+        .unwrap();
+        assert!(!metadata.is_empty());
+        for (provider, model, expected) in [
+            (Some("cloudflare.workers_ai"), "clef", Some(200)),
+            (Some("cloudflare-workers-ai"), "clef", Some(200)),
+            (Some("cloudflare.workers_ai"), "clef-20240101", Some(300)),
+            (Some("cloudflare.workers_ai"), "clef-20240202", Some(200)),
+            (
+                Some("cloudflare.workers_ai"),
+                "@cf/cloudflare/clef",
+                Some(200),
+            ),
+            (
+                Some("cloudflare.workers_ai"),
+                "prefix/author/clef",
+                Some(400),
+            ),
+            (Some("openrouter"), "cloudflare/clef", Some(500)),
+            (Some("unknown"), "clef", None),
+            (Some("openrouter"), "clef", None),
+            (Some("cloudflare.workers_ai"), "unknown", None),
+            (None, "clef", Some(100)),
+            (Some(""), "clef", Some(100)),
+        ] {
+            assert_eq!(
+                metadata.context_size(model, provider),
+                expected,
+                "{provider:?}: {model}"
+            );
+        }
+        metadata.models.clear();
+        assert!(!metadata.is_empty());
+        metadata.version = 2;
+        assert!(metadata.is_empty());
+        assert!(
+            metadata
+                .get("clef", Some("cloudflare.workers_ai"))
+                .is_none()
+        );
+    }
+
+    #[test]
+    fn test_legacy_model_metadata() {
+        let metadata: ModelMetadata = serde_json::from_value(json!({
+            "version": 1,
+            "models": {"clef": {"contextSize": 100}}
+        }))
+        .unwrap();
+        assert_eq!(metadata.context_size("clef", None), Some(100));
+        assert_eq!(
+            metadata.context_size("clef", Some("cloudflare.workers_ai")),
+            Some(100)
+        );
+        assert!(
+            !serde_json::to_value(&metadata)
+                .unwrap()
+                .as_object()
+                .unwrap()
+                .contains_key("providers")
+        );
+    }
+
+    #[test]
+    fn test_provider_metadata_aliases() {
+        for (provider, catalog_id) in [
+            ("gcp.gemini", "google"),
+            ("gcp.vertex_ai", "google-vertex"),
+            ("aws.bedrock", "amazon-bedrock"),
+            ("azure.ai.openai", "azure"),
+        ] {
+            let metadata: ModelMetadata = serde_json::from_value(json!({
+                "version": 1,
+                "providers": {catalog_id: {"model": {"contextSize": 100}}}
+            }))
+            .unwrap();
+            assert_eq!(metadata.context_size("model", Some(provider)), Some(100));
+        }
+    }
+
     #[test]
     fn test_normalize_ai_model_name() {
         // Test date patterns