feat(llm): require user-owned request credentials

lib/kodo/agent/evaluation_runner.ex

@@ -2,18 +2,20 @@ defmodule Kodo.Agent.EvaluationRunner do
   @moduledoc "Runs the pinned MVP evaluation suite against a live LLM adapter."
 
   alias Kodo.Agent.{EvaluationSuite, ModelMapping, ReviewResult, Roles, Tools}
+  alias Kodo.Accounts.Scope
+  alias Kodo.LLM
 
   @timeout 120_000
   @max_output 32_000
 
   @doc "Runs every task and returns a JSON-safe report. Errors are isolated per task."
-  def run(opts \\ []) do
+  def run(%Scope{} = scope, opts \\ []) do
     started_at = DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601()
     suite = Keyword.get_lazy(opts, :suite, &EvaluationSuite.load!/0)
     adapter = Keyword.get(opts, :adapter, Kodo.LLM.ReqLLM)
     mapping = Keyword.get(opts, :mapping, ModelMapping.balanced())
 
-    tasks = Enum.map(suite["tasks"], &safe_task(&1, adapter, mapping))
+    tasks = Enum.map(suite["tasks"], &safe_task(&1, scope, adapter, mapping))
 
     %{
       "run" => %{"started_at" => started_at, "revision" => revision()},
@@ -79,12 +81,12 @@ defmodule Kodo.Agent.EvaluationRunner do
     }
   end
 
-  defp safe_task(task, adapter, mapping) do
+  defp safe_task(task, scope, adapter, mapping) do
     started = System.monotonic_time(:millisecond)
 
     result =
       try do
-        execute(task, adapter, mapping)
+        execute(task, scope, adapter, mapping)
       rescue
         exception -> %{"error" => Exception.message(exception), "trace" => [], "usage" => %{}}
       catch
@@ -102,12 +104,14 @@ defmodule Kodo.Agent.EvaluationRunner do
     |> Map.put("latency_ms", System.monotonic_time(:millisecond) - started)
   end
 
-  defp execute(%{"type" => "search"} = task, adapter, mapping) do
+  defp execute(%{"type" => "search"} = task, scope, adapter, mapping) do
     with_workspace(task["fixture"]["files"], fn root ->
       role = ModelMapping.role!(mapping, :search)
       contract = Roles.fetch!(:search, role["role_contract"])
       prompt = task["prompt"] <> " Cite each finding as path:line and quote evidence."
-      {answer, trace, usage} = tool_loop(adapter, role, contract, prompt, root, [], [], mapping)
+
+      {answer, trace, usage} =
+        tool_loop(scope, adapter, role, contract, prompt, root, [], mapping)
 
       %{
         "answer" => answer,
@@ -118,17 +122,13 @@ defmodule Kodo.Agent.EvaluationRunner do
     end)
   end
 
-  defp execute(%{"type" => "review"} = task, adapter, mapping) do
+  defp execute(%{"type" => "review"} = task, scope, adapter, mapping) do
     role = ModelMapping.role!(mapping, :review)
     contract = Roles.fetch!(:review, role["role_contract"])
     messages = messages(contract.prompt, task["prompt"] <> "\n\n" <> task["fixture"]["diff"])
     started = System.monotonic_time(:millisecond)
 
-    {:ok, response} =
-      adapter.generate_object(role["model"], messages, ReviewResult.schema(),
-        timeout: @timeout,
-        reasoning: role["reasoning"]
-      )
+    {:ok, response} = generate_object(scope, adapter, role, messages, ReviewResult.schema())
 
     %{
       "object" => response.object,
@@ -138,7 +138,7 @@ defmodule Kodo.Agent.EvaluationRunner do
     }
   end
 
-  defp execute(%{"type" => "implementation"} = task, adapter, mapping) do
+  defp execute(%{"type" => "implementation"} = task, scope, adapter, mapping) do
     with_workspace(task["fixture"]["files"], fn root ->
       System.cmd("git", ["init", "-q"], cd: root, stderr_to_stdout: true)
       System.cmd("git", ["add", "."], cd: root, stderr_to_stdout: true)
@@ -149,10 +149,10 @@ defmodule Kodo.Agent.EvaluationRunner do
       prompt = implementation_prompt(task, checks)
 
       {answer, trace, usage} =
-        tool_loop(adapter, role, contract, prompt, root, checks, [], mapping)
+        tool_loop(scope, adapter, role, contract, prompt, root, checks, mapping)
 
       {diff, 0} = System.cmd("git", ["diff", "--no-ext-diff"], cd: root, stderr_to_stdout: true)
-      initial_review = structured_review(adapter, mapping, task["prompt"], diff)
+      initial_review = structured_review(scope, adapter, mapping, task["prompt"], diff)
 
       {answer, trace, usage, review} =
         if initial_review.object["clean"] do
@@ -162,12 +162,22 @@ defmodule Kodo.Agent.EvaluationRunner do
             correction_prompt(task, checks, initial_review.object["findings"])
 
           {corrected_answer, correction_trace, correction_usage} =
-            tool_loop(adapter, role, contract, correction_prompt, root, checks, [], mapping)
+            tool_loop(
+              scope,
+              adapter,
+              role,
+              contract,
+              correction_prompt,
+              root,
+              checks,
+              mapping
+            )
 
           {corrected_diff, 0} =
             System.cmd("git", ["diff", "--no-ext-diff"], cd: root, stderr_to_stdout: true)
 
-          corrected_review = structured_review(adapter, mapping, task["prompt"], corrected_diff)
+          corrected_review =
+            structured_review(scope, adapter, mapping, task["prompt"], corrected_diff)
 
           {corrected_answer,
            trace ++
@@ -201,12 +211,13 @@ defmodule Kodo.Agent.EvaluationRunner do
     end)
   end
 
-  defp tool_loop(adapter, role, contract, prompt, root, commands, trace, mapping) do
+  defp tool_loop(scope, adapter, role, contract, prompt, root, commands, mapping) do
     state = %{
+      scope: scope,
       root: root,
       commands: commands,
       history: messages(contract.prompt, prompt),
-      trace: trace,
+      trace: [],
       usage: %{},
       mapping: mapping
     }
@@ -220,16 +231,16 @@ defmodule Kodo.Agent.EvaluationRunner do
     else
       started = System.monotonic_time(:millisecond)
 
-      case adapter.generate(
-             role["model"],
+      case generate(
+             state.scope,
+             adapter,
+             role,
              state.history,
              tool_definitions_for_turn(
                contract.toolset_version,
                step + 1,
                contract.budget.max_continuations
-             ),
-             timeout: @timeout,
-             reasoning: role["reasoning"]
+             )
            ) do
         {:ok, response} ->
           continue_loop(adapter, role, contract, state, response, started, step)
@@ -287,7 +298,13 @@ defmodule Kodo.Agent.EvaluationRunner do
     Enum.map_reduce(calls, [], fn call, trace ->
       output =
         if call.name == "delegate_search" do
-          delegated_search(adapter, state.mapping, state.root, call.arguments["question"])
+          delegated_search(
+            state.scope,
+            adapter,
+            state.mapping,
+            state.root,
+            call.arguments["question"]
+          )
         else
           execute_tool(call.name, call.arguments, state.root, state.commands)
         end
@@ -440,33 +457,53 @@ defmodule Kodo.Agent.EvaluationRunner do
     end
   end
 
-  defp delegated_search(adapter, mapping, root, question) do
+  defp delegated_search(scope, adapter, mapping, root, question) do
     role = ModelMapping.role!(mapping, :search)
     contract = Roles.fetch!(:search, role["role_contract"])
 
     {answer, trace, usage} =
-      tool_loop(adapter, role, contract, question, root, [], [], mapping)
+      tool_loop(scope, adapter, role, contract, question, root, [], mapping)
 
     %{"question" => question, "evidence" => answer, "trace" => trace, "usage" => usage}
   end
 
-  defp structured_review(adapter, mapping, task, diff) do
+  defp structured_review(scope, adapter, mapping, task, diff) do
     role = ModelMapping.role!(mapping, :review)
     contract = Roles.fetch!(:review, role["role_contract"])
 
     {:ok, response} =
-      adapter.generate_object(
-        role["model"],
+      generate_object(
+        scope,
+        adapter,
+        role,
         messages(
           contract.prompt,
           "Original task:\n#{task}\n\nReview this final diff:\n\n#{diff}"
         ),
-        ReviewResult.schema(),
+        ReviewResult.schema()
+      )
+
+    %{response | object: ReviewResult.actionable(response.object, diff)}
+  end
+
+  defp generate(scope, adapter, role, messages, tools) do
+    with {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do
+      LLM.generate(scope, model, reference, messages, tools,
+        adapter: adapter,
         timeout: @timeout,
         reasoning: role["reasoning"]
       )
+    end
+  end
 
-    %{response | object: ReviewResult.actionable(response.object, diff)}
+  defp generate_object(scope, adapter, role, messages, schema) do
+    with {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do
+      LLM.generate_object(scope, model, reference, messages, schema,
+        adapter: adapter,
+        timeout: @timeout,
+        reasoning: role["reasoning"]
+      )
+    end
   end
 
   @doc false

lib/kodo/agent/loop.ex

@@ -169,6 +169,7 @@ defmodule Kodo.Agent.Loop do
 
     with {:ok, capability_validation} <-
            context.adapter.validate_model(review["model"], review, contract),
+         {:ok, request} <- resolve_request(context.session_id, review),
          :ok <-
            Phoenix.PubSub.subscribe(
              Kodo.PubSub,
@@ -201,8 +202,9 @@ defmodule Kodo.Agent.Loop do
            Sessions.dispatch_if_owner(context.ownership, fn ->
              task = original_task(context.events)
 
-             context.adapter.generate_object(
-               review["model"],
+             generate_object(
+               context.adapter,
+               request,
                [
                  %{"role" => "system", "content" => contract.prompt},
                  %{
@@ -466,6 +468,7 @@ defmodule Kodo.Agent.Loop do
     with :ok <- within_budget(invocation, usage(current_turn(events)), budgets),
          {:ok, capability_validation} <-
            adapter.validate_model(primary["model"], primary, contract),
+         {:ok, request} <- resolve_request(session_id, primary),
          {:ok, invocation_id} <-
            start_invocation(
              session_id,
@@ -478,7 +481,7 @@ defmodule Kodo.Agent.Loop do
          :ok <- rehoming_boundary(),
          {:ok, response} <-
            Sessions.dispatch_if_owner(ownership, fn ->
-             adapter.generate(primary["model"], transcript(events, contract), tools,
+             generate(adapter, request, transcript(events, contract), tools,
                timeout: budgets[:model_timeout],
                reasoning: primary["reasoning"]
              )
@@ -916,6 +919,7 @@ defmodule Kodo.Agent.Loop do
   defp run_search_continuation(messages, continuation, tokens, state) do
     with :ok <- within_budget(continuation, tokens, state.contract.budget),
          :ok <- rehoming_boundary(),
+         {:ok, request} <- resolve_request(state.context.session_id, state.search),
          {:ok, invocation_id} <-
            start_subagent_invocation(
              state.parent_call,
@@ -928,8 +932,9 @@ defmodule Kodo.Agent.Loop do
          :ok <- rehoming_boundary(),
          {:ok, response} <-
            Sessions.dispatch_if_owner(state.context.ownership, fn ->
-             state.context.adapter.generate(
-               state.search["model"],
+             generate(
+               state.context.adapter,
+               request,
                messages,
                Tools.definitions_for_turn(
                  state.contract.toolset_version,
@@ -1599,6 +1604,35 @@ defmodule Kodo.Agent.Loop do
   defp token_count(nil), do: 0
   defp token_count(usage), do: usage[:total_tokens] || usage["total_tokens"] || 0
 
+  defp resolve_request(session_id, role) do
+    with {:ok, scope} <- Sessions.owner_scope(session_id),
+         {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do
+      {:ok, %{scope: scope, model: model, reference: reference}}
+    end
+  end
+
+  defp generate(adapter, request, messages, tools, opts) do
+    LLM.generate(
+      request.scope,
+      request.model,
+      request.reference,
+      messages,
+      tools,
+      Keyword.put(opts, :adapter, adapter)
+    )
+  end
+
+  defp generate_object(adapter, request, messages, schema, opts) do
+    LLM.generate_object(
+      request.scope,
+      request.model,
+      request.reference,
+      messages,
+      schema,
+      Keyword.put(opts, :adapter, adapter)
+    )
+  end
+
   defp within_budget(invocations, tokens, budgets) do
     cond do
       invocations > budgets[:max_continuations] -> {:error, :continuation_budget_exceeded}