feat: ai tracing improvements

CLAUDE.md

@@ -685,6 +685,21 @@ Templates are DB rows seeded by migrations (`traceway-otel-agent` for the OTel h
 | POST | `/api/exception-stack-traces/by-id/:exceptionId` | App | Single exception by ID |
 | POST | `/api/exception-stack-traces/:hash` | App | Exception by hash |
 
+**AI Traces & Conversations**
+
+One `ai_traces` row per LLM call (any OTLP span with `gen_ai.*` attributes). Each row carries a `conversation_id` resolved at ingest (`gen_ai.conversation.id` -> `session.id` span/resource attr -> distributed trace id -> empty), `tool_call_count`/`tool_names` parsed from the completion payload (OpenAI `tool_calls`, Anthropic `tool_use`, OTel output messages, `gen_ai.tool.*` fallback), and `flagged`/`flagged_terms` from an ingest-time word-boundary scan of prompt+completion against per-project selected built-in language packs (`projects.ai_flagged_languages`, default `["en"]`; packs live in `backend/app/services/contentflag/terms/*.txt` — en/de/es/fr/it/pt/sr; empty array = custom terms only) plus per-project custom terms (`projects.ai_flagged_terms`); both edited in the project settings AI tab and cached via the project cache. All three project-creation paths (`Create`, `CreateWithOrganization`, `cmd/seed.go`) must set `AiFlaggedLanguages` explicitly since lit inserts every struct field. `tool_names`/`flagged_terms` are stored comma-separated (values sanitized at ingest); the content matcher lives in `backend/app/services/contentflag/`. Conversation analytics exclude rows with an empty `conversation_id`; user analytics additionally require a non-empty `user_id`.
+
+| Method | Endpoint | Auth | Purpose |
+|--------|----------|------|---------|
+| POST | `/api/ai-traces/grouped` | App | AI traces grouped by trace name |
+| POST | `/api/ai-traces/trace` | App | Calls for one trace name (`?traceName=`) |
+| POST | `/api/ai-traces/:traceId` | App | Single call detail + conversation blob |
+| POST | `/api/ai-conversations/grouped` | App | Conversations (GROUP BY conversation_id): turns, cost, tokens, tools, models, flagged; filters userId/model/toolName/flaggedOnly/search (search matches conversation id, user, model, tool names, and flagged terms; row-level filters are semi-joins on conversation_id so a match on any turn returns the whole conversation's aggregates); response also carries `thresholds` (range-wide P95 cost/turns for outlier highlighting) and `facets` (models, tools) |
+| POST | `/api/ai-conversations/conversation` | App | All turns of one conversation (id in body) ordered by recorded_at, each with its stored input/output payload (capped at 200 turns of payloads), plus stats |
+| POST | `/api/ai-users/grouped` | App | Per-user conversation analytics: conversation count, total calls, avg/min/median turns, avg cost per conversation, total cost, flagged conversation count |
+
+Frontend routes: `/ai-traces` (tabs: Traces, Conversations, Users), `/ai-traces/conversations/[conversationId]` (chat timeline with tool calls rendered). Trace names `conversations` and `users` are shadowed by these static routes. Notification rule types `ai_trace_cost` (per call), `ai_conversation_cost` (24h cumulative per conversation), and `ai_flagged_content` (flagged term match, optional term filter) are event-driven; remember both `notification_rule.repository.go` copies list event rule types explicitly.
+
 **Organization Management**
 | Method | Endpoint | Auth | Purpose |
 |--------|----------|------|---------|

README.md

@@ -50,7 +50,7 @@ Traceway is an **OpenTelemetry-native** observability platform. It combines **lo
 - **Exceptions**: Stack traces are normalized, given a SHA-256 fingerprint, and grouped into ranked issues. Source-mapped (webpack, esbuild, Vite).
 - **Profiling** _(experimental)_: Flame graphs for CPU, heap, and goroutines with version-to-version diffing and a top-functions table. Ingests native Go pprof and OTLP profiles.
 - **Session Replay**: Watch what the user did right before the error. Available for web (any JS framework) and Flutter.
-- **AI Observability**: LLM cost, tokens, latency, and full conversations across providers (OpenRouter and any OTel-compatible AI gateway).
+- **AI Observability**: LLM cost, tokens, latency, and full conversations across providers (OpenRouter and any OTel-compatible AI gateway). Calls group into conversations via `gen_ai.conversation.id`, tool calls are parsed from completions and rendered in the chat view, and multi-language content flagging catches conversations containing terms you care about. Per-customer analytics (conversation length, cost per conversation) key on `user.id`: set it to a stable customer identifier such as your account or tenant id, the same value across all of that user's conversations, never a session id.
 - **On-Call & Paging**: Rotation schedules with layers and overrides, escalation policies, and pages that escalate until someone acknowledges. Delivered via email, Slack, Pushover, Telegram, or SMS, with one-click acknowledge links that need no login.
 
 Plus: background-task (job) monitoring, configurable alerts (Slack / GitHub / email / webhook / Pushover / Telegram), multi-tenant orgs with role-based access, and a per-endpoint slow-threshold override.

backend/app/cache/project_cache.go

@@ -67,6 +67,8 @@ func copyProject(proj *models.Project) *models.Project {
 	cp := *proj
 	cp.HealthcheckPaths = append(models.StringSlice(nil), proj.HealthcheckPaths...)
 	cp.ProfileLabelAllowlist = append(models.StringSlice(nil), proj.ProfileLabelAllowlist...)
+	cp.AiFlaggedTerms = append(models.StringSlice(nil), proj.AiFlaggedTerms...)
+	cp.AiFlaggedLanguages = append(models.StringSlice(nil), proj.AiFlaggedLanguages...)
 	return &cp
 }
 
@@ -120,6 +122,8 @@ func (c *projectCache) UpdateProject(proj *models.Project) {
 	cached.DropHealthyHealthchecks = proj.DropHealthyHealthchecks
 	cached.HealthcheckPaths = proj.HealthcheckPaths
 	cached.ProfileLabelAllowlist = proj.ProfileLabelAllowlist
+	cached.AiFlaggedTerms = proj.AiFlaggedTerms
+	cached.AiFlaggedLanguages = proj.AiFlaggedLanguages
 }
 
 func (c *projectCache) RemoveProject(id uuid.UUID) {

backend/app/controllers/ai_trace.controller.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/url"
+	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -50,6 +51,57 @@ type AiTraceDetailResponse struct {
 	Conversation json.RawMessage `json:"conversation,omitempty"`
 }
 
+type AiConversationSearchRequest struct {
+	FromDate      time.Time        `json:"fromDate"`
+	ToDate        time.Time        `json:"toDate"`
+	OrderBy       string           `json:"orderBy"`
+	SortDirection string           `json:"sortDirection"`
+	Pagination    PaginationParams `json:"pagination"`
+	Search        string           `json:"search"`
+	UserId        string           `json:"userId"`
+	Model         string           `json:"model"`
+	ToolName      string           `json:"toolName"`
+	FlaggedOnly   bool             `json:"flaggedOnly"`
+}
+
+type AiConversationFacets struct {
+	Models []string `json:"models"`
+	Tools  []string `json:"tools"`
+}
+
+type AiConversationListResponse struct {
+	Data       []models.AiConversationStats     `json:"data"`
+	Pagination Pagination                       `json:"pagination"`
+	Thresholds *models.AiConversationThresholds `json:"thresholds"`
+	Facets     AiConversationFacets             `json:"facets"`
+}
+
+type AiConversationDetailRequest struct {
+	ConversationId string    `json:"conversationId"`
+	FromDate       time.Time `json:"fromDate"`
+	ToDate         time.Time `json:"toDate"`
+}
+
+type AiConversationTurn struct {
+	models.AiTrace
+	Input  string `json:"input"`
+	Output string `json:"output"`
+}
+
+type AiConversationDetailResponse struct {
+	Data  []AiConversationTurn              `json:"data"`
+	Stats *models.AiConversationDetailStats `json:"stats"`
+}
+
+type AiUserSearchRequest struct {
+	FromDate      time.Time        `json:"fromDate"`
+	ToDate        time.Time        `json:"toDate"`
+	OrderBy       string           `json:"orderBy"`
+	SortDirection string           `json:"sortDirection"`
+	Pagination    PaginationParams `json:"pagination"`
+	Search        string           `json:"search"`
+}
+
 func (a aiTraceController) FindGroupedByTraceName(c *gin.Context) {
 	projectId, err := middleware.GetProjectId(c)
 	if err != nil {
@@ -172,4 +224,148 @@ func (a aiTraceController) GetAiTraceDetail(c *gin.Context) {
 	c.JSON(http.StatusOK, response)
 }
 
+func (a aiTraceController) FindConversations(c *gin.Context) {
+	projectId, err := middleware.GetProjectId(c)
+	if err != nil {
+		c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
+		return
+	}
+
+	var request AiConversationSearchRequest
+	if err := c.ShouldBindJSON(&request); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	stats, total, thresholds, err := telemetry.AiTraceRepository.FindConversations(c, projectId, request.FromDate, request.ToDate, request.Pagination.Page, request.Pagination.PageSize, request.OrderBy, request.SortDirection, request.Search, request.UserId, request.Model, request.ToolName, request.FlaggedOnly)
+	if err != nil {
+		c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading ai conversations: %w", err))
+		return
+	}
+
+	facetModels, err := telemetry.AiTraceRepository.ListModels(c, projectId, request.FromDate, request.ToDate)
+	if err != nil {
+		facetModels = nil
+		traceway.CaptureException(traceway.NewStackTraceErrorf("error loading ai conversation model facets: %w", err))
+	}
+	facetTools, err := telemetry.AiTraceRepository.ListToolNames(c, projectId, request.FromDate, request.ToDate)
+	if err != nil {
+		facetTools = nil
+		traceway.CaptureException(traceway.NewStackTraceErrorf("error loading ai conversation tool facets: %w", err))
+	}
+
+	c.JSON(http.StatusOK, AiConversationListResponse{
+		Data: stats,
+		Pagination: Pagination{
+			Page:       request.Pagination.Page,
+			PageSize:   request.Pagination.PageSize,
+			Total:      total,
+			TotalPages: (total + int64(request.Pagination.PageSize) - 1) / int64(request.Pagination.PageSize),
+		},
+		Thresholds: thresholds,
+		Facets:     AiConversationFacets{Models: facetModels, Tools: facetTools},
+	})
+}
+
+// conversationPayloadCap bounds how many turns get their stored prompt and
+// completion blobs attached on the detail endpoint; each blob can carry a
+// full resent message history, so unbounded reads would balloon the response.
+const conversationPayloadCap = 200
+
+func (a aiTraceController) GetConversationDetail(c *gin.Context) {
+	projectId, err := middleware.GetProjectId(c)
+	if err != nil {
+		c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
+		return
+	}
+
+	var request AiConversationDetailRequest
+	if err := c.ShouldBindJSON(&request); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	if request.ConversationId == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "conversationId is required"})
+		return
+	}
+
+	traces, stats, err := telemetry.AiTraceRepository.FindByConversationId(c, projectId, request.ConversationId, request.FromDate, request.ToDate)
+	if err != nil {
+		c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading ai conversation: %w", err))
+		return
+	}
+	if len(traces) == 0 {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Conversation not found"})
+		return
+	}
+
+	turns := make([]AiConversationTurn, len(traces))
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, 8)
+	for i, trace := range traces {
+		turns[i] = AiConversationTurn{AiTrace: trace}
+		if trace.StorageKey == "" || i >= conversationPayloadCap {
+			continue
+		}
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, storageKey string) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			data, err := storage.Store.Read(c, storageKey)
+			if err != nil {
+				// The blob write is best-effort at ingest, so a missing payload
+				// is expected for some turns; the turn still renders from its
+				// row data.
+				return
+			}
+			var payload struct {
+				Input  string `json:"input"`
+				Output string `json:"output"`
+			}
+			if err := json.Unmarshal(data, &payload); err != nil {
+				return
+			}
+			turns[i].Input = payload.Input
+			turns[i].Output = payload.Output
+		}(i, trace.StorageKey)
+	}
+	wg.Wait()
+
+	c.JSON(http.StatusOK, AiConversationDetailResponse{
+		Data:  turns,
+		Stats: stats,
+	})
+}
+
+func (a aiTraceController) FindAiUsers(c *gin.Context) {
+	projectId, err := middleware.GetProjectId(c)
+	if err != nil {
+		c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
+		return
+	}
+
+	var request AiUserSearchRequest
+	if err := c.ShouldBindJSON(&request); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	stats, total, err := telemetry.AiTraceRepository.FindUserStats(c, projectId, request.FromDate, request.ToDate, request.Pagination.Page, request.Pagination.PageSize, request.OrderBy, request.SortDirection, request.Search)
+	if err != nil {
+		c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading ai user stats: %w", err))
+		return
+	}
+
+	c.JSON(http.StatusOK, PaginatedResponse[models.AiUserStats]{
+		Data: stats,
+		Pagination: Pagination{
+			Page:       request.Pagination.Page,
+			PageSize:   request.Pagination.PageSize,
+			Total:      total,
+			TotalPages: (total + int64(request.Pagination.PageSize) - 1) / int64(request.Pagination.PageSize),
+		},
+	})
+}
+
 var AiTraceController = aiTraceController{}

backend/app/controllers/notification_rule.controller.go

@@ -36,6 +36,8 @@ var validRuleTypes = map[string]bool{
 	"impact_score_high":       true,
 	"impact_score_medium":     true,
 	"ai_trace_cost":           true,
+	"ai_conversation_cost":    true,
+	"ai_flagged_content":      true,
 }
 
 func (ctrl *notificationRuleController) List(ctx *gin.Context) {

backend/app/controllers/otelcontrollers/otel.controller.go

@@ -165,7 +165,14 @@ func (o otelController) ExportTraces(c *gin.Context) {
 
 	var aiTraceInfos []hooks.AiTraceInfo
 	for _, at := range aiTraces {
-		aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{TraceName: at.TraceName, TotalCost: at.TotalCost})
+		aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{
+			TraceName:      at.TraceName,
+			TotalCost:      at.TotalCost,
+			ConversationId: at.ConversationId,
+			UserId:         at.UserId,
+			Flagged:        at.Flagged,
+			FlaggedTerms:   at.FlaggedTerms,
+		})
 	}
 
 	if hasOrg {

backend/app/controllers/otelcontrollers/testdata/anthropic_tool_use_ai_trace.json

@@ -0,0 +1,41 @@
+{
+  "resourceSpans": [
+    {
+      "resource": {
+        "attributes": [
+          {"key": "service.name", "value": {"stringValue": "support-bot"}},
+          {"key": "service.version", "value": {"stringValue": "0.9.0"}}
+        ]
+      },
+      "scopeSpans": [
+        {
+          "scope": {"name": "support-bot"},
+          "spans": [
+            {
+              "attributes": [
+                {"key": "trace.name", "value": {"stringValue": "Support Bot"}},
+                {"key": "user.id", "value": {"stringValue": "customer-77"}},
+                {"key": "session.id", "value": {"stringValue": "sess-9f81"}},
+                {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}},
+                {"key": "gen_ai.system", "value": {"stringValue": "anthropic"}},
+                {"key": "gen_ai.request.model", "value": {"stringValue": "claude-sonnet-5"}},
+                {"key": "gen_ai.usage.input_tokens", "value": {"intValue": 300}},
+                {"key": "gen_ai.usage.output_tokens", "value": {"intValue": 55}},
+                {"key": "gen_ai.response.finish_reasons", "value": {"arrayValue": {"values": [{"stringValue": "tool_use"}]}}},
+                {"key": "gen_ai.prompt", "value": {"stringValue": "{\"messages\":[{\"role\":\"user\",\"content\":\"My order never arrived, this is bullshit. Where is it?\"}]}"}},
+                {"key": "gen_ai.completion", "value": {"stringValue": "{\"content\":[{\"type\":\"text\",\"text\":\"I'm sorry about the delay. Let me look up your order.\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"lookup_order\",\"input\":{\"order_id\":\"A-123\"}}],\"stop_reason\":\"tool_use\"}"}}
+              ],
+              "endTimeUnixNano": "1774926644373000000",
+              "kind": 3,
+              "name": "chat claude-sonnet-5",
+              "spanId": "2b06f22e9f134e70",
+              "startTimeUnixNano": "1774926642873000000",
+              "status": {"code": 1},
+              "traceId": "bb5ad7a3070fecf76bc3180145d97611"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

backend/app/controllers/otelcontrollers/testdata/anthropic_tool_use_ai_trace.json.golden.json

@@ -0,0 +1,39 @@
+{
+  "endpointCount": 0,
+  "endpoints": [],
+  "taskCount": 0,
+  "spanCount": 0,
+  "spans": [],
+  "exceptionCount": 0,
+  "exceptions": [],
+  "aiTraceCount": 1,
+  "aiTraces": [
+    {
+      "traceName": "Support Bot",
+      "model": "claude-sonnet-5",
+      "provider": "anthropic",
+      "operation": "chat",
+      "inputTokens": 300,
+      "outputTokens": 55,
+      "totalTokens": 355,
+      "cachedTokens": 0,
+      "reasoningTokens": 0,
+      "inputCost": 0,
+      "outputCost": 0,
+      "totalCost": 0,
+      "finishReason": "tool_use",
+      "statusCode": 1,
+      "conversationId": "sess-9f81",
+      "toolCallCount": 1,
+      "toolNames": [
+        "lookup_order"
+      ],
+      "flagged": true,
+      "flaggedTerms": [
+        "bullshit"
+      ]
+    }
+  ],
+  "conversationCount": 1,
+  "allSpansLinked": true
+}