diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c30dcbefe6..0eda9eb674c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ **Bug Fixes**: +- Use provider-specific AI model prices and context sizes when available, rather than another provider's metadata for the same model. ([sentry#128011](https://github.com/getsentry/sentry/issues/128011)) - Use the dynamic sampling context to fill missing trace IDs on errors, preserving existing trace and span IDs. ([#6508](https://github.com/getsentry/relay/pull/6508)) - Fix Perfetto sample timestamps recorded in a clock other than the primary trace clock. ([#6522](https://github.com/getsentry/relay/pull/6522)) - Remove invalid `sentry.release` attributes from spans instead of forwarding them. ([#6547](https://github.com/getsentry/relay/pull/6547)) diff --git a/relay-event-normalization/src/eap/ai.rs b/relay-event-normalization/src/eap/ai.rs index d5bb4dc7147..1e7ac00c37e 100644 --- a/relay-event-normalization/src/eap/ai.rs +++ b/relay-event-normalization/src/eap/ai.rs @@ -126,7 +126,9 @@ fn normalize_context_utilization( .get_value(GEN_AI__RESPONSE__MODEL) .and_then(|v| v.as_str()); - let context_size = model_id.and_then(|id| model_metadata.and_then(|m| m.context_size(id))); + let provider = extract_string_value(attributes, GEN_AI__PROVIDER__NAME); + let context_size = + model_id.and_then(|id| model_metadata.and_then(|m| m.context_size(id, provider))); let Some(context_size) = context_size else { return; @@ -173,7 +175,8 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_metadata: Option<&Model return; }; - let Some(model_cost) = model_metadata.and_then(|m| m.cost_per_token(model_id)) else { + let provider = extract_string_value(attributes, GEN_AI__PROVIDER__NAME); + let Some(model_cost) = model_metadata.and_then(|m| m.cost_per_token(model_id, provider)) else { relay_statsd::metric!( counter(Counters::GenAiCostCalculationResult) += 1, result = "calculation_no_model_cost_available", @@ -273,6 +276,7 @@ mod tests { fn model_metadata() -> ModelMetadata { ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([ ( "claude-2.1".parse().unwrap(), @@ -307,6 +311,7 @@ mod tests { fn model_metadata_with_context_size() -> ModelMetadata { ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([( "claude-2.1".parse().unwrap(), ModelMetadataEntry { @@ -323,6 +328,54 @@ mod tests { } } + #[test] + fn test_normalize_ai_provider_prices() { + let entry = |input_per_token| { + serde_json::json!({ + "costs": { + "inputPerToken": input_per_token, + "outputPerToken": 0.0, + "outputReasoningPerToken": 0.0, + "inputCachedPerToken": 0.0, + "inputCacheWritePerToken": 0.0 + } + }) + }; + let metadata: ModelMetadata = serde_json::from_value(serde_json::json!({ + "version": 1, + "models": {"clef": entry(0.042 / 1_000_000.0)}, + "providers": { + "cloudflare-workers-ai": {"clef": entry(0.24 / 1_000_000.0)}, + "openrouter": {"cloudflare/clef": entry(0.042 / 1_000_000.0)}, + "anthropic": {"claude-sonnet-4": entry(3.0 / 1_000_000.0)}, + "poe": {"claude-sonnet-4": entry(2.6 / 1_000_000.0)} + } + })) + .unwrap(); + for (provider, model, expected) in [ + ("cloudflare.workers_ai", "clef", 0.01323168), + ("openrouter", "cloudflare/clef", 0.002315544), + ("anthropic", "claude-sonnet-4", 0.165396), + ("poe", "claude-sonnet-4", 0.1433432), + ] { + let mut attributes = Annotated::new(attributes! { + "gen_ai.operation.name" => "chat".to_owned(), + "gen_ai.provider.name" => provider.to_owned(), + "gen_ai.response.model" => model.to_owned(), + "gen_ai.usage.input_tokens" => 55_132, + }); + normalize_ai(&mut attributes, None, Some(&metadata)); + let cost = attributes + .value() + .unwrap() + .get_value(GEN_AI__COST__TOTAL_TOKENS) + .unwrap() + .as_f64() + .unwrap(); + assert!((cost - expected).abs() < 1e-10, "{provider}: {cost}"); + } + } + #[test] fn test_normalize_ai_type_fallback() { for model_key in [ diff --git a/relay-event-normalization/src/event.rs b/relay-event-normalization/src/event.rs index 2688c913b6d..f8517a1bf92 100644 --- a/relay-event-normalization/src/event.rs +++ b/relay-event-normalization/src/event.rs @@ -2525,6 +2525,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([ ( "claude-2.1".parse().unwrap(), @@ -2656,6 +2657,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([ ( "claude-2.1".parse().unwrap(), @@ -2773,6 +2775,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([( "claude-2.1".parse().unwrap(), ModelMetadataEntry { @@ -2848,6 +2851,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([ ( "claude-2.1".parse().unwrap(), @@ -2947,6 +2951,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::new(), }), ..NormalizationConfig::default() @@ -2991,6 +2996,7 @@ mod tests { &NormalizationConfig { ai_model_metadata: Some(&ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::new(), }), ..NormalizationConfig::default() diff --git a/relay-event-normalization/src/normalize/mod.rs b/relay-event-normalization/src/normalize/mod.rs index 7d40c069721..68d0a2ea874 100644 --- a/relay-event-normalization/src/normalize/mod.rs +++ b/relay-event-normalization/src/normalize/mod.rs @@ -299,6 +299,11 @@ pub struct ModelMetadata { /// The mappings of model ID => metadata as a dictionary. #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub models: HashMap, + + /// Provider-specific metadata, keyed by models.dev provider IDs. + /// Older configurations omit this and use the model-only table. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub providers: HashMap>, } impl ModelMetadata { @@ -306,7 +311,8 @@ impl ModelMetadata { /// `true` if the model metadata is empty or the version is unsupported. pub fn is_empty(&self) -> bool { - self.models.is_empty() || !self.is_enabled() + (self.models.is_empty() && self.providers.values().all(HashMap::is_empty)) + || !self.is_enabled() } /// `false` if the version is unsupported. @@ -315,40 +321,62 @@ impl ModelMetadata { } /// Gets the cost per token for a given model, if defined. - pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> { - self.get(model_id).and_then(|entry| entry.costs.as_ref()) + pub fn cost_per_token(&self, model_id: &str, provider: Option<&str>) -> Option<&ModelCostV2> { + self.get(model_id, provider) + .and_then(|entry| entry.costs.as_ref()) } /// Gets the context window size for a given model, if defined. /// /// Returns `None` for a zero context size, as it is not a meaningful value. - pub fn context_size(&self, model_id: &str) -> Option { - self.get(model_id) + pub fn context_size(&self, model_id: &str, provider: Option<&str>) -> Option { + self.get(model_id, provider) .and_then(|entry| entry.context_size) .filter(|&size| size > 0) } /// Gets the metadata for a given model, if defined. - pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> { + /// + /// With provider tables available, never fall back to another provider's + /// prices. Retain the legacy lookup for old configs or provider-less spans. + pub fn get(&self, model_id: &str, provider: Option<&str>) -> Option<&ModelMetadataEntry> { if !self.is_enabled() { return None; } + let models = match provider.filter(|provider| !provider.is_empty()) { + Some(provider) if !self.providers.is_empty() => { + // Map OpenTelemetry provider names to models.dev catalog IDs. + let provider = match provider { + "cloudflare.workers_ai" => "cloudflare-workers-ai", + "gcp.gemini" => "google", + "gcp.vertex_ai" => "google-vertex", + "aws.bedrock" => "amazon-bedrock", + "azure.ai.openai" => "azure", + provider => provider, + }; + self.providers.get(provider)? + } + _ => &self.models, + }; let normalized_model_id = normalize_ai_model_name(model_id); - // First try exact match. - if let Some(value) = self.models.get(normalized_model_id) { + // Prefer full IDs, including snapshots, before normalized aliases. + if let Some(value) = models + .get(model_id) + .or_else(|| models.get(normalized_model_id)) + { return Some(value); } - // Fall back to glob matching. - self.models.iter().find_map(|(key, value)| { - if key.is_match(normalized_model_id) { - Some(value) - } else { - None - } - }) + // Prefer the most specific matching pattern, independent of map order. + models + .iter() + .filter(|(key, _)| key.is_match(normalized_model_id)) + .max_by(|(a, _), (b, _)| { + (a.original.len(), &a.original).cmp(&(b.original.len(), &b.original)) + }) + .map(|(_, value)| value) } } @@ -459,6 +487,100 @@ mod tests { use super::*; + #[test] + fn test_provider_metadata_lookup() { + let mut metadata: ModelMetadata = serde_json::from_value(json!({ + "version": 1, + "models": {"clef": {"contextSize": 100}}, + "providers": { + "cloudflare-workers-ai": { + "clef": {"contextSize": 200}, + "clef-20240101": {"contextSize": 300}, + "*clef": {"contextSize": 200}, + "*author/clef": {"contextSize": 400} + }, + "openrouter": {"cloudflare/clef": {"contextSize": 500}} + } + })) + .unwrap(); + assert!(!metadata.is_empty()); + for (provider, model, expected) in [ + (Some("cloudflare.workers_ai"), "clef", Some(200)), + (Some("cloudflare-workers-ai"), "clef", Some(200)), + (Some("cloudflare.workers_ai"), "clef-20240101", Some(300)), + (Some("cloudflare.workers_ai"), "clef-20240202", Some(200)), + ( + Some("cloudflare.workers_ai"), + "@cf/cloudflare/clef", + Some(200), + ), + ( + Some("cloudflare.workers_ai"), + "prefix/author/clef", + Some(400), + ), + (Some("openrouter"), "cloudflare/clef", Some(500)), + (Some("unknown"), "clef", None), + (Some("openrouter"), "clef", None), + (Some("cloudflare.workers_ai"), "unknown", None), + (None, "clef", Some(100)), + (Some(""), "clef", Some(100)), + ] { + assert_eq!( + metadata.context_size(model, provider), + expected, + "{provider:?}: {model}" + ); + } + metadata.models.clear(); + assert!(!metadata.is_empty()); + metadata.version = 2; + assert!(metadata.is_empty()); + assert!( + metadata + .get("clef", Some("cloudflare.workers_ai")) + .is_none() + ); + } + + #[test] + fn test_legacy_model_metadata() { + let metadata: ModelMetadata = serde_json::from_value(json!({ + "version": 1, + "models": {"clef": {"contextSize": 100}} + })) + .unwrap(); + assert_eq!(metadata.context_size("clef", None), Some(100)); + assert_eq!( + metadata.context_size("clef", Some("cloudflare.workers_ai")), + Some(100) + ); + assert!( + !serde_json::to_value(&metadata) + .unwrap() + .as_object() + .unwrap() + .contains_key("providers") + ); + } + + #[test] + fn test_provider_metadata_aliases() { + for (provider, catalog_id) in [ + ("gcp.gemini", "google"), + ("gcp.vertex_ai", "google-vertex"), + ("aws.bedrock", "amazon-bedrock"), + ("azure.ai.openai", "azure"), + ] { + let metadata: ModelMetadata = serde_json::from_value(json!({ + "version": 1, + "providers": {catalog_id: {"model": {"contextSize": 100}}} + })) + .unwrap(); + assert_eq!(metadata.context_size("model", Some(provider)), Some(100)); + } + } + #[test] fn test_normalize_ai_model_name() { // Test date patterns diff --git a/relay-event-normalization/src/normalize/span/ai.rs b/relay-event-normalization/src/normalize/span/ai.rs index a8324e0c1bf..c457e1dc62c 100644 --- a/relay-event-normalization/src/normalize/span/ai.rs +++ b/relay-event-normalization/src/normalize/span/ai.rs @@ -343,7 +343,8 @@ fn set_total_tokens(data: &mut SpanData) { fn extract_context_utilization(data: &mut SpanData, model_metadata: &ModelMetadata) { let model_id = data.get_str(GEN_AI__RESPONSE__MODEL); - let context_size = model_id.and_then(|id| model_metadata.context_size(id)); + let provider = data.get_str(GEN_AI__PROVIDER__NAME); + let context_size = model_id.and_then(|id| model_metadata.context_size(id, provider)); let Some(context_size) = context_size else { return; @@ -394,7 +395,7 @@ fn extract_ai_data( // Extracts the total cost of the AI model used if let Some(model_id) = data.get_str(GEN_AI__RESPONSE__MODEL) { extract_ai_model_cost_data( - model_metadata.cost_per_token(model_id), + model_metadata.cost_per_token(model_id, data.get_str(GEN_AI__PROVIDER__NAME)), data, origin, platform, @@ -1362,9 +1363,52 @@ mod tests { "#); } + #[test] + fn test_provider_specific_metadata() { + let mut metadata = metadata_with_context_size(); + let mut provider_models = metadata.models.clone(); + let entry = provider_models.get_mut("claude-2.1").unwrap(); + entry.costs.as_mut().unwrap().input_per_token = 0.03; + entry.context_size = Some(200_000); + metadata + .providers + .insert("anthropic".to_owned(), provider_models); + + let mut span = Span { + op: "gen_ai.test".to_owned().into(), + data: SpanData::from_value( + json!({ + "gen_ai.provider.name": "anthropic", + "gen_ai.response.model": "claude-2.1", + "gen_ai.usage.input_tokens": 1000, + }) + .into(), + ), + ..Default::default() + }; + enrich_ai_span(&mut span, Some(&metadata)); + let data = span.data.value().unwrap(); + assert_eq!( + data.get_value(GEN_AI__COST__TOTAL_TOKENS) + .and_then(Value::as_f64), + Some(30.0) + ); + assert_eq!( + data.get_value(GEN_AI__CONTEXT__WINDOW_SIZE) + .and_then(Value::as_f64), + Some(200_000.0) + ); + assert_eq!( + data.get_value(GEN_AI__CONTEXT__UTILIZATION) + .and_then(Value::as_f64), + Some(0.005) + ); + } + fn metadata_with_context_size() -> ModelMetadata { ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([( "claude-2.1".parse().unwrap(), ModelMetadataEntry { @@ -1416,6 +1460,7 @@ mod tests { fn test_context_utilization_no_context_size() { let metadata = ModelMetadata { version: 1, + providers: HashMap::new(), models: HashMap::from([( "claude-2.1".parse().unwrap(), ModelMetadataEntry {