diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 27bd5229..eb77e813 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -470,6 +470,7 @@ export const tabNavigation: NavTab[] = [ { title: 'Conversation Hallucination', href: '/docs/evaluation/builtin/conversation-hallucination' }, { title: 'Tool Call Accuracy', href: '/docs/evaluation/builtin/tool-call-accuracy' }, { title: 'Trajectory Match', href: '/docs/evaluation/builtin/trajectory-match' }, + { title: 'SRE Root Cause Accuracy', href: '/docs/evaluation/builtin/sre-root-cause-accuracy' }, { title: 'Step Count', href: '/docs/evaluation/builtin/step-count' }, ] }, diff --git a/src/pages/docs/evaluation/builtin/index.mdx b/src/pages/docs/evaluation/builtin/index.mdx index a6dfc6db..d86f884f 100644 --- a/src/pages/docs/evaluation/builtin/index.mdx +++ b/src/pages/docs/evaluation/builtin/index.mdx @@ -79,6 +79,7 @@ Multi-turn behavior: coherence, goal completion, and how a customer-facing agent | [**Customer Agent: Task Completion**](/docs/evaluation/builtin/customer-agent-task-completion) | Checks whether an agent fully resolved the customer's request, including valid policy-based refusals. | `agent_prompt`, `conversation` | Conversation, Chat | LLM as Judge | | [**Tool Call Accuracy**](/docs/evaluation/builtin/tool-call-accuracy) | Compares an agent's actual tool calls against expected calls, scoring matches on function name and arguments. | `output`, `expected` | Agents, Tool Use | Code | | [**Trajectory Match**](/docs/evaluation/builtin/trajectory-match) | Compares an agent's actual action sequence against an expected trajectory using configurable matching modes. | `output`, `expected` | Agents, Tool Use | Code | +| [**SRE Root Cause Accuracy**](/docs/evaluation/builtin/sre-root-cause-accuracy) | Scores an AI SRE or investigation agent's final diagnosis against evidence such as logs, metrics, traces, and runbooks. | `diagnosis`, `context` | Agents, Root Cause Analysis | LLM as Judge | | [**Step Count**](/docs/evaluation/builtin/step-count) | Validates the number of steps in an agent trajectory against an exact count or a min/max range. | `output` | Agents | Code | ## Output quality & format diff --git a/src/pages/docs/evaluation/builtin/sre-root-cause-accuracy.mdx b/src/pages/docs/evaluation/builtin/sre-root-cause-accuracy.mdx new file mode 100644 index 00000000..1b565417 --- /dev/null +++ b/src/pages/docs/evaluation/builtin/sre-root-cause-accuracy.mdx @@ -0,0 +1,72 @@ +--- +title: "SRE Root Cause Accuracy" +description: "Scores an AI SRE or investigation agent's final diagnosis against evidence such as logs, metrics, traces, and runbooks." +--- + +SRE Root Cause Accuracy checks whether an AI SRE or investigation agent correctly identifies the root cause of an incident based on the provided evidence. + +## What it does + +This evaluator uses an LLM-as-a-judge to score the agent's diagnosis on a scale from 0.0 to 1.0. It penalizes hallucinated causes, claims that lack evidence, and destructive remediation suggestions. + +### Input + +| Required Input | Type | Description | +| --- | --- | --- | +| `diagnosis` | `string` | The agent's final root-cause diagnosis. | +| `context` | `string` \| `array` | The evidence gathered during the incident (logs, metrics, traces, runbooks). | +| `trajectory` | `string` \| `array` | (Optional) The agent's tool-use trajectory. | + +### Output + +| Field | Type | Description | +| --- | --- | --- | +| Result | `score` | A numeric score between 0.0 and 1.0. Higher scores indicate the agent's diagnosis correctly identified the root cause and is grounded in the provided context. | +| Reason | `string` | A plain-language explanation of the score. | + +### Run it from code + +Call `evaluate()` with the template name and the eval's required inputs. It returns the score and the reason. + + +Before running: [install the SDK and set `FI_API_KEY` / `FI_SECRET_KEY`](/docs/evaluation/reference/sdk-api). + + + + +```python Python +from fi.evals import evaluate + +result = evaluate( + "sre_root_cause_accuracy", + diagnosis="The database went down due to an OutOfMemory error.", + context="Logs show OOMKilled at 10:05 PM for the db-service container.", +) + +print(result.score) +print(result.reason) +``` + +```typescript TypeScript +import { evaluate } from "@future-agi/ai-evaluation"; + +const result = await evaluate( + "sre_root_cause_accuracy", + { + diagnosis: "The database went down due to an OutOfMemory error.", + context: "Logs show OOMKilled at 10:05 PM for the db-service container.", + } +); + +console.log(result); +``` + + + +## When to use + +Run SRE Root Cause Accuracy when you are building AI agents that investigate systems, diagnose outages, or suggest remediations based on telemetry and logs. + +## What to do when SRE Root Cause Accuracy is low + +Read the reason string to identify whether the agent is hallucinating causes or failing to ground its claims in the provided logs and metrics. If the agent frequently invents causes, consider providing stricter system prompts requiring citations for every claim.