diff --git a/lib/kodo/agent/evaluation_runner.ex b/lib/kodo/agent/evaluation_runner.ex index 8b9a9e6..0a518ca 100644 --- a/lib/kodo/agent/evaluation_runner.ex +++ b/lib/kodo/agent/evaluation_runner.ex @@ -2,18 +2,20 @@ defmodule Kodo.Agent.EvaluationRunner do @moduledoc "Runs the pinned MVP evaluation suite against a live LLM adapter." alias Kodo.Agent.{EvaluationSuite, ModelMapping, ReviewResult, Roles, Tools} + alias Kodo.Accounts.Scope + alias Kodo.LLM @timeout 120_000 @max_output 32_000 @doc "Runs every task and returns a JSON-safe report. Errors are isolated per task." - def run(opts \\ []) do + def run(%Scope{} = scope, opts \\ []) do started_at = DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() suite = Keyword.get_lazy(opts, :suite, &EvaluationSuite.load!/0) adapter = Keyword.get(opts, :adapter, Kodo.LLM.ReqLLM) mapping = Keyword.get(opts, :mapping, ModelMapping.balanced()) - tasks = Enum.map(suite["tasks"], &safe_task(&1, adapter, mapping)) + tasks = Enum.map(suite["tasks"], &safe_task(&1, scope, adapter, mapping)) %{ "run" => %{"started_at" => started_at, "revision" => revision()}, @@ -79,12 +81,12 @@ defmodule Kodo.Agent.EvaluationRunner do } end - defp safe_task(task, adapter, mapping) do + defp safe_task(task, scope, adapter, mapping) do started = System.monotonic_time(:millisecond) result = try do - execute(task, adapter, mapping) + execute(task, scope, adapter, mapping) rescue exception -> %{"error" => Exception.message(exception), "trace" => [], "usage" => %{}} catch @@ -102,12 +104,14 @@ defmodule Kodo.Agent.EvaluationRunner do |> Map.put("latency_ms", System.monotonic_time(:millisecond) - started) end - defp execute(%{"type" => "search"} = task, adapter, mapping) do + defp execute(%{"type" => "search"} = task, scope, adapter, mapping) do with_workspace(task["fixture"]["files"], fn root -> role = ModelMapping.role!(mapping, :search) contract = Roles.fetch!(:search, role["role_contract"]) prompt = task["prompt"] <> " Cite each finding as path:line and quote evidence." - {answer, trace, usage} = tool_loop(adapter, role, contract, prompt, root, [], [], mapping) + + {answer, trace, usage} = + tool_loop(scope, adapter, role, contract, prompt, root, [], mapping) %{ "answer" => answer, @@ -118,17 +122,13 @@ defmodule Kodo.Agent.EvaluationRunner do end) end - defp execute(%{"type" => "review"} = task, adapter, mapping) do + defp execute(%{"type" => "review"} = task, scope, adapter, mapping) do role = ModelMapping.role!(mapping, :review) contract = Roles.fetch!(:review, role["role_contract"]) messages = messages(contract.prompt, task["prompt"] <> "\n\n" <> task["fixture"]["diff"]) started = System.monotonic_time(:millisecond) - {:ok, response} = - adapter.generate_object(role["model"], messages, ReviewResult.schema(), - timeout: @timeout, - reasoning: role["reasoning"] - ) + {:ok, response} = generate_object(scope, adapter, role, messages, ReviewResult.schema()) %{ "object" => response.object, @@ -138,7 +138,7 @@ defmodule Kodo.Agent.EvaluationRunner do } end - defp execute(%{"type" => "implementation"} = task, adapter, mapping) do + defp execute(%{"type" => "implementation"} = task, scope, adapter, mapping) do with_workspace(task["fixture"]["files"], fn root -> System.cmd("git", ["init", "-q"], cd: root, stderr_to_stdout: true) System.cmd("git", ["add", "."], cd: root, stderr_to_stdout: true) @@ -149,10 +149,10 @@ defmodule Kodo.Agent.EvaluationRunner do prompt = implementation_prompt(task, checks) {answer, trace, usage} = - tool_loop(adapter, role, contract, prompt, root, checks, [], mapping) + tool_loop(scope, adapter, role, contract, prompt, root, checks, mapping) {diff, 0} = System.cmd("git", ["diff", "--no-ext-diff"], cd: root, stderr_to_stdout: true) - initial_review = structured_review(adapter, mapping, task["prompt"], diff) + initial_review = structured_review(scope, adapter, mapping, task["prompt"], diff) {answer, trace, usage, review} = if initial_review.object["clean"] do @@ -162,12 +162,22 @@ defmodule Kodo.Agent.EvaluationRunner do correction_prompt(task, checks, initial_review.object["findings"]) {corrected_answer, correction_trace, correction_usage} = - tool_loop(adapter, role, contract, correction_prompt, root, checks, [], mapping) + tool_loop( + scope, + adapter, + role, + contract, + correction_prompt, + root, + checks, + mapping + ) {corrected_diff, 0} = System.cmd("git", ["diff", "--no-ext-diff"], cd: root, stderr_to_stdout: true) - corrected_review = structured_review(adapter, mapping, task["prompt"], corrected_diff) + corrected_review = + structured_review(scope, adapter, mapping, task["prompt"], corrected_diff) {corrected_answer, trace ++ @@ -201,12 +211,13 @@ defmodule Kodo.Agent.EvaluationRunner do end) end - defp tool_loop(adapter, role, contract, prompt, root, commands, trace, mapping) do + defp tool_loop(scope, adapter, role, contract, prompt, root, commands, mapping) do state = %{ + scope: scope, root: root, commands: commands, history: messages(contract.prompt, prompt), - trace: trace, + trace: [], usage: %{}, mapping: mapping } @@ -220,16 +231,16 @@ defmodule Kodo.Agent.EvaluationRunner do else started = System.monotonic_time(:millisecond) - case adapter.generate( - role["model"], + case generate( + state.scope, + adapter, + role, state.history, tool_definitions_for_turn( contract.toolset_version, step + 1, contract.budget.max_continuations - ), - timeout: @timeout, - reasoning: role["reasoning"] + ) ) do {:ok, response} -> continue_loop(adapter, role, contract, state, response, started, step) @@ -287,7 +298,13 @@ defmodule Kodo.Agent.EvaluationRunner do Enum.map_reduce(calls, [], fn call, trace -> output = if call.name == "delegate_search" do - delegated_search(adapter, state.mapping, state.root, call.arguments["question"]) + delegated_search( + state.scope, + adapter, + state.mapping, + state.root, + call.arguments["question"] + ) else execute_tool(call.name, call.arguments, state.root, state.commands) end @@ -440,33 +457,53 @@ defmodule Kodo.Agent.EvaluationRunner do end end - defp delegated_search(adapter, mapping, root, question) do + defp delegated_search(scope, adapter, mapping, root, question) do role = ModelMapping.role!(mapping, :search) contract = Roles.fetch!(:search, role["role_contract"]) {answer, trace, usage} = - tool_loop(adapter, role, contract, question, root, [], [], mapping) + tool_loop(scope, adapter, role, contract, question, root, [], mapping) %{"question" => question, "evidence" => answer, "trace" => trace, "usage" => usage} end - defp structured_review(adapter, mapping, task, diff) do + defp structured_review(scope, adapter, mapping, task, diff) do role = ModelMapping.role!(mapping, :review) contract = Roles.fetch!(:review, role["role_contract"]) {:ok, response} = - adapter.generate_object( - role["model"], + generate_object( + scope, + adapter, + role, messages( contract.prompt, "Original task:\n#{task}\n\nReview this final diff:\n\n#{diff}" ), - ReviewResult.schema(), + ReviewResult.schema() + ) + + %{response | object: ReviewResult.actionable(response.object, diff)} + end + + defp generate(scope, adapter, role, messages, tools) do + with {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do + LLM.generate(scope, model, reference, messages, tools, + adapter: adapter, timeout: @timeout, reasoning: role["reasoning"] ) + end + end - %{response | object: ReviewResult.actionable(response.object, diff)} + defp generate_object(scope, adapter, role, messages, schema) do + with {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do + LLM.generate_object(scope, model, reference, messages, schema, + adapter: adapter, + timeout: @timeout, + reasoning: role["reasoning"] + ) + end end @doc false diff --git a/lib/kodo/agent/loop.ex b/lib/kodo/agent/loop.ex index 3aebf73..a446468 100644 --- a/lib/kodo/agent/loop.ex +++ b/lib/kodo/agent/loop.ex @@ -169,6 +169,7 @@ defmodule Kodo.Agent.Loop do with {:ok, capability_validation} <- context.adapter.validate_model(review["model"], review, contract), + {:ok, request} <- resolve_request(context.session_id, review), :ok <- Phoenix.PubSub.subscribe( Kodo.PubSub, @@ -201,8 +202,9 @@ defmodule Kodo.Agent.Loop do Sessions.dispatch_if_owner(context.ownership, fn -> task = original_task(context.events) - context.adapter.generate_object( - review["model"], + generate_object( + context.adapter, + request, [ %{"role" => "system", "content" => contract.prompt}, %{ @@ -466,6 +468,7 @@ defmodule Kodo.Agent.Loop do with :ok <- within_budget(invocation, usage(current_turn(events)), budgets), {:ok, capability_validation} <- adapter.validate_model(primary["model"], primary, contract), + {:ok, request} <- resolve_request(session_id, primary), {:ok, invocation_id} <- start_invocation( session_id, @@ -478,7 +481,7 @@ defmodule Kodo.Agent.Loop do :ok <- rehoming_boundary(), {:ok, response} <- Sessions.dispatch_if_owner(ownership, fn -> - adapter.generate(primary["model"], transcript(events, contract), tools, + generate(adapter, request, transcript(events, contract), tools, timeout: budgets[:model_timeout], reasoning: primary["reasoning"] ) @@ -916,6 +919,7 @@ defmodule Kodo.Agent.Loop do defp run_search_continuation(messages, continuation, tokens, state) do with :ok <- within_budget(continuation, tokens, state.contract.budget), :ok <- rehoming_boundary(), + {:ok, request} <- resolve_request(state.context.session_id, state.search), {:ok, invocation_id} <- start_subagent_invocation( state.parent_call, @@ -928,8 +932,9 @@ defmodule Kodo.Agent.Loop do :ok <- rehoming_boundary(), {:ok, response} <- Sessions.dispatch_if_owner(state.context.ownership, fn -> - state.context.adapter.generate( - state.search["model"], + generate( + state.context.adapter, + request, messages, Tools.definitions_for_turn( state.contract.toolset_version, @@ -1599,6 +1604,35 @@ defmodule Kodo.Agent.Loop do defp token_count(nil), do: 0 defp token_count(usage), do: usage[:total_tokens] || usage["total_tokens"] || 0 + defp resolve_request(session_id, role) do + with {:ok, scope} <- Sessions.owner_scope(session_id), + {:ok, model, reference} <- LLM.resolve_integration(scope, role["model"]) do + {:ok, %{scope: scope, model: model, reference: reference}} + end + end + + defp generate(adapter, request, messages, tools, opts) do + LLM.generate( + request.scope, + request.model, + request.reference, + messages, + tools, + Keyword.put(opts, :adapter, adapter) + ) + end + + defp generate_object(adapter, request, messages, schema, opts) do + LLM.generate_object( + request.scope, + request.model, + request.reference, + messages, + schema, + Keyword.put(opts, :adapter, adapter) + ) + end + defp within_budget(invocations, tokens, budgets) do cond do invocations > budgets[:max_continuations] -> {:error, :continuation_budget_exceeded} diff --git a/lib/kodo/llm.ex b/lib/kodo/llm.ex index 016f17d..17a3f2a 100644 --- a/lib/kodo/llm.ex +++ b/lib/kodo/llm.ex @@ -1,5 +1,18 @@ defmodule Kodo.LLM do - @moduledoc "Provider-independent boundary for one explicit primary-agent model step." + @moduledoc """ + Provider-independent boundary for one explicit agent model step. + + Generation goes through this facade rather than directly through ReqLLM so a + request must resolve a user-owned integration before ReqLLM can consult its + process-wide or environment credential fallbacks. + """ + + alias Kodo.Accounts.Scope + alias Kodo.LLM.Credential + alias Kodo.LLM.CredentialResolver + alias Kodo.LLM.IntegrationRef + + @credential_option_keys ~w(api_key access_token auth_mode oauth_file auth_file provider_options chatgpt_account_id)a @type tool :: %{ required(:name) => String.t(), @@ -15,13 +28,70 @@ defmodule Kodo.LLM do optional(:assistant) => map() } - @callback generate(String.t(), [map()], [tool()], keyword()) :: + @callback generate(LLMDB.Model.t(), [map()], [tool()], Credential.t(), keyword()) :: {:ok, result()} | {:error, term()} - @callback generate_object(String.t(), [map()], map(), keyword()) :: + @callback generate_object(LLMDB.Model.t(), [map()], map(), Credential.t(), keyword()) :: {:ok, %{object: map(), usage: map() | nil}} | {:error, term()} @callback validate_model(String.t(), map(), map()) :: {:ok, map()} | {:error, term()} + @doc "Resolves a model and captures non-secret integration metadata for later admission." + def resolve_integration(%Scope{} = scope, model) do + with {:ok, resolved_model} <- resolve_model(model), + {:ok, reference} <- CredentialResolver.reference(scope, resolved_model) do + {:ok, resolved_model, reference} + end + end + + @doc "Generates text after rechecking and decrypting the referenced credential." + def generate( + %Scope{} = scope, + %LLMDB.Model{} = model, + %IntegrationRef{} = reference, + messages, + tools, + opts + ) do + adapter = Keyword.get(opts, :adapter, adapter()) + adapter_opts = Keyword.delete(opts, :adapter) + + with :ok <- reject_credential_options(adapter_opts), + {:ok, credential} <- CredentialResolver.resolve(scope, model, reference) do + adapter.generate(model, messages, tools, credential, adapter_opts) + end + end + + @doc "Generates a structured object after scoped credential admission." + def generate_object( + %Scope{} = scope, + %LLMDB.Model{} = model, + %IntegrationRef{} = reference, + messages, + schema, + opts + ) do + adapter = Keyword.get(opts, :adapter, adapter()) + adapter_opts = Keyword.delete(opts, :adapter) + + with :ok <- reject_credential_options(adapter_opts), + {:ok, credential} <- CredentialResolver.resolve(scope, model, reference) do + adapter.generate_object(model, messages, schema, credential, adapter_opts) + end + end + def adapter, do: Application.get_env(:kodo, :llm_adapter, Kodo.LLM.ReqLLM) + + defp resolve_model(model) do + case ReqLLM.model(model) do + {:ok, %LLMDB.Model{} = resolved} -> {:ok, resolved} + {:error, _reason} -> {:error, :malformed_model} + end + end + + defp reject_credential_options(opts) do + if Enum.any?(@credential_option_keys, &Keyword.has_key?(opts, &1)), + do: {:error, :credential_options_not_allowed}, + else: :ok + end end diff --git a/lib/kodo/llm/credential.ex b/lib/kodo/llm/credential.ex new file mode 100644 index 0000000..a595344 --- /dev/null +++ b/lib/kodo/llm/credential.ex @@ -0,0 +1,39 @@ +defmodule Kodo.LLM.Credential do + @moduledoc false + + @derive {Inspect, + only: [ + :integration_id, + :provider, + :authentication_type, + :credential_generation, + :billing_path + ]} + @enforce_keys [ + :integration_id, + :provider, + :authentication_type, + :credential_generation, + :billing_path, + :token + ] + defstruct [ + :integration_id, + :provider, + :authentication_type, + :credential_generation, + :billing_path, + :token, + :account_id + ] + + @opaque t :: %__MODULE__{ + integration_id: Ecto.UUID.t(), + provider: String.t(), + authentication_type: String.t(), + credential_generation: non_neg_integer(), + billing_path: :platform | :subscription | :aggregator, + token: String.t(), + account_id: String.t() | nil + } +end diff --git a/lib/kodo/llm/credential_resolver.ex b/lib/kodo/llm/credential_resolver.ex new file mode 100644 index 0000000..901ccd4 --- /dev/null +++ b/lib/kodo/llm/credential_resolver.ex @@ -0,0 +1,133 @@ +defmodule Kodo.LLM.CredentialResolver do + @moduledoc """ + Resolves a scoped integration reference into one operation-local credential. + + This Kodo-owned boundary deliberately wraps ReqLLM credential handling. ReqLLM + supports process-wide keys, environment keys, and shared OAuth files; resolving + here first prevents those fallbacks from selecting another user's credential. + Only the resulting current access token or API key may cross into the adapter. + """ + + alias Kodo.Accounts.Scope + alias Kodo.Integrations + alias Kodo.Integrations.CredentialEncryption + alias Kodo.Integrations.Integration + alias Kodo.LLM.Credential + alias Kodo.LLM.IntegrationRef + + @providers ~w(openai openai_codex anthropic openrouter) + + @doc false + def reference(%Scope{} = scope, %LLMDB.Model{provider: model_provider}) do + provider = Atom.to_string(model_provider) + + with :ok <- require_supported_provider(provider), + {:ok, integration} <- Integrations.get_integration_by_provider(scope, provider), + :ok <- require_usable(integration) do + {:ok, IntegrationRef.from_integration(integration)} + end + end + + def reference(_scope, _model), do: {:error, :invalid_integration_reference} + + @doc false + def resolve( + %Scope{} = scope, + %LLMDB.Model{provider: model_provider}, + %IntegrationRef{} = reference + ) do + provider = Atom.to_string(model_provider) + + with :ok <- require_supported_provider(provider), + :ok <- require_reference(reference), + {:ok, integration} <- Integrations.get_integration(scope, reference.integration_id), + :ok <- require_generation(integration, reference.credential_generation), + :ok <- require_provider(reference.provider, provider), + :ok <- require_provider(integration.provider, provider), + :ok <- require_usable(integration), + {:ok, payload} <- CredentialEncryption.decrypt(integration) do + build_credential(integration, payload) + end + end + + def resolve(_scope, _model, _reference), do: {:error, :invalid_integration_reference} + + defp require_supported_provider(provider) when provider in @providers, do: :ok + defp require_supported_provider(_provider), do: {:error, :unsupported_model_provider} + + defp require_reference(%IntegrationRef{ + integration_id: id, + provider: provider, + credential_generation: generation + }) + when is_binary(provider) and provider in @providers and is_integer(generation) and + generation >= 0 do + case Ecto.UUID.cast(id) do + {:ok, _id} -> :ok + :error -> {:error, :invalid_integration_reference} + end + end + + defp require_reference(%IntegrationRef{}), do: {:error, :invalid_integration_reference} + + defp require_provider(provider, provider), do: :ok + defp require_provider(_actual, _expected), do: {:error, :integration_provider_mismatch} + + defp require_generation(%Integration{credential_generation: generation}, generation), do: :ok + + defp require_generation(%Integration{}, _generation), + do: {:error, :stale_credential_generation} + + defp require_usable(%Integration{connection_status: "disconnected"}), + do: {:error, :integration_disconnected} + + defp require_usable(%Integration{connection_status: "reauthorization_required"}), + do: {:error, :integration_reauthorization_required} + + defp require_usable(%Integration{connection_status: "connected", validation_status: "invalid"}), + do: {:error, :integration_invalid} + + defp require_usable(%Integration{connection_status: "connected"}), do: :ok + + defp build_credential(%Integration{authentication_type: "api_key"} = integration, payload) do + with {:ok, api_key} <- fetch_secret(payload, "api_key") do + {:ok, credential(integration, api_key, nil)} + end + end + + defp build_credential( + %Integration{provider: "openai_codex", authentication_type: "oauth"} = integration, + payload + ) do + with {:ok, access_token} <- fetch_secret(payload, "access_token"), + {:ok, account_id} <- fetch_secret(payload, "account_id") do + {:ok, credential(integration, access_token, account_id)} + end + end + + defp build_credential(%Integration{}, _payload), + do: {:error, :credential_payload_invalid} + + defp credential(integration, token, account_id) do + %Credential{ + integration_id: integration.id, + provider: integration.provider, + authentication_type: integration.authentication_type, + credential_generation: integration.credential_generation, + billing_path: billing_path(integration.provider), + token: token, + account_id: account_id + } + end + + defp fetch_secret(payload, field) do + case payload[field] do + value when is_binary(value) and value != "" -> {:ok, value} + _missing -> {:error, :credential_payload_invalid} + end + end + + defp billing_path("openai_codex"), do: :subscription + defp billing_path("openrouter"), do: :aggregator + defp billing_path(_provider), do: :platform +end diff --git a/lib/kodo/llm/integration_ref.ex b/lib/kodo/llm/integration_ref.ex new file mode 100644 index 0000000..91438b3 --- /dev/null +++ b/lib/kodo/llm/integration_ref.ex @@ -0,0 +1,28 @@ +defmodule Kodo.LLM.IntegrationRef do + @moduledoc """ + Non-secret reference to one generation of a user-owned provider integration. + + Generation is part of the reference so replacing or disconnecting credentials + invalidates references captured by earlier preflight work. + """ + + alias Kodo.Integrations.Integration + + @enforce_keys [:integration_id, :provider, :credential_generation] + defstruct [:integration_id, :provider, :credential_generation] + + @opaque t :: %__MODULE__{ + integration_id: Ecto.UUID.t(), + provider: String.t(), + credential_generation: non_neg_integer() + } + + @doc "Builds a reference from scoped integration metadata." + def from_integration(%Integration{} = integration) do + %__MODULE__{ + integration_id: integration.id, + provider: integration.provider, + credential_generation: integration.credential_generation + } + end +end diff --git a/lib/kodo/llm/req_llm.ex b/lib/kodo/llm/req_llm.ex index a61fa17..1bbe253 100644 --- a/lib/kodo/llm/req_llm.ex +++ b/lib/kodo/llm/req_llm.ex @@ -4,6 +4,7 @@ defmodule Kodo.LLM.ReqLLM do @behaviour Kodo.LLM alias Kodo.Agent.ModelCapabilities + alias Kodo.LLM.Credential alias ReqLLM.Context alias ReqLLM.Message alias ReqLLM.Message.ContentPart @@ -30,13 +31,13 @@ defmodule Kodo.LLM.ReqLLM do end @impl true - def generate(model, messages, tools, opts) do + def generate(model, messages, tools, %Credential{} = credential, opts) do context = build_context(messages) - request(model, context, tools, opts) + request(model, context, tools, credential, opts) end @impl true - def generate_object(model, messages, schema, opts) do + def generate_object(model, messages, schema, %Credential{} = credential, opts) do request_opts = [ receive_timeout: Keyword.fetch!(opts, :timeout), @@ -44,6 +45,7 @@ defmodule Kodo.LLM.ReqLLM do output_validation: :strict ] |> put_reasoning_effort(opts[:reasoning]) + |> put_credential(credential) with {:ok, response} <- ReqLLM.generate_object(model, build_context(messages), schema, request_opts) do @@ -61,6 +63,13 @@ defmodule Kodo.LLM.ReqLLM do |> put_reasoning_effort(opts[:reasoning]) end + @doc false + def request_options(tools, %Credential{} = credential, opts) do + tools + |> request_options(opts) + |> put_credential(credential) + end + @doc false def build_context(messages) do messages @@ -81,8 +90,8 @@ defmodule Kodo.LLM.ReqLLM do end) end - defp request(model, context, tools, opts) do - request_opts = request_options(tools, opts) + defp request(model, context, tools, credential, opts) do + request_opts = request_options(tools, credential, opts) with {:ok, response} <- ReqLLM.generate_text(model, context, request_opts) do classified = Response.classify(response) @@ -105,6 +114,21 @@ defmodule Kodo.LLM.ReqLLM do Keyword.put(opts, :reasoning_effort, Map.fetch!(@reasoning_efforts, reasoning)) end + defp put_credential(opts, %Credential{authentication_type: "api_key", token: token}) do + Keyword.put(opts, :api_key, token) + end + + defp put_credential(opts, %Credential{ + authentication_type: "oauth", + token: token, + account_id: account_id + }) do + opts + |> Keyword.put(:auth_mode, :oauth) + |> Keyword.put(:access_token, token) + |> Keyword.put(:chatgpt_account_id, account_id) + end + defp resolve_dispatchable_model(model) do with {:ok, resolved} <- ReqLLM.model(model), {:ok, _provider} <- ReqLLM.provider(resolved.provider) do diff --git a/lib/kodo/sessions.ex b/lib/kodo/sessions.ex index 62ee7c2..48698c6 100644 --- a/lib/kodo/sessions.ex +++ b/lib/kodo/sessions.ex @@ -10,6 +10,7 @@ defmodule Kodo.Sessions do alias Kodo.Cluster.Placement alias Kodo.ControlPlaneTelemetry alias Kodo.Agent.ModelSettings + alias Kodo.Accounts.User alias Kodo.Repo alias Kodo.Sessions.Event alias Kodo.Sessions.Ownership @@ -54,6 +55,23 @@ defmodule Kodo.Sessions do def get_session!(id), do: Repo.get!(Session, id) + @doc false + def owner_scope(session_id) do + case Ecto.UUID.cast(session_id) do + {:ok, session_id} -> + user = + User + |> join(:inner, [user], session in Session, on: session.user_id == user.id) + |> where([_user, session], session.id == ^session_id) + |> Repo.one() + + if user, do: {:ok, Scope.for_user(user)}, else: {:error, :session_not_found} + + :error -> + {:error, :session_not_found} + end + end + def list_sessions(%Scope{user: user}) do Session |> where([session], session.user_id == ^user.id) diff --git a/lib/mix/tasks/kodo.eval.run.ex b/lib/mix/tasks/kodo.eval.run.ex index 7b52da6..a075538 100644 --- a/lib/mix/tasks/kodo.eval.run.ex +++ b/lib/mix/tasks/kodo.eval.run.ex @@ -3,20 +3,25 @@ defmodule Mix.Tasks.Kodo.Eval.Run do @shortdoc "Run the pinned Kodo live evaluation" use Mix.Task + alias Kodo.Accounts + alias Kodo.Accounts.Scope alias Kodo.Agent.EvaluationRunner @impl Mix.Task def run(args) do - {opts, rest, invalid} = OptionParser.parse(args, strict: [output: :string]) + {opts, rest, invalid} = + OptionParser.parse(args, strict: [output: :string, user_email: :string]) - if rest != [] or invalid != [] or is_nil(opts[:output]) do - Mix.raise("usage: mix kodo.eval.run --output PATH") + if rest != [] or invalid != [] or is_nil(opts[:output]) or is_nil(opts[:user_email]) do + Mix.raise("usage: mix kodo.eval.run --output PATH --user-email EMAIL") end - {:ok, _applications} = Application.ensure_all_started(:req_llm) + Mix.Task.run("app.start") + user = Accounts.get_user_by_email(opts[:user_email]) || Mix.raise("evaluation user not found") + scope = Scope.for_user(user) output = Path.expand(opts[:output]) File.mkdir_p!(Path.dirname(output)) - File.write!(output, Jason.encode!(EvaluationRunner.run(), pretty: true)) + File.write!(output, Jason.encode!(EvaluationRunner.run(scope), pretty: true)) Mix.shell().info("Evaluation report written to #{output}") end end diff --git a/test/e2e/hermetic_full_stack_test.exs b/test/e2e/hermetic_full_stack_test.exs index 9585793..296bdf5 100644 --- a/test/e2e/hermetic_full_stack_test.exs +++ b/test/e2e/hermetic_full_stack_test.exs @@ -2,6 +2,7 @@ defmodule Kodo.E2E.HermeticFullStackTest do use Kodo.DataCase, async: false alias Kodo.Sessions + alias Kodo.Integrations alias Kodo.Test.FullStackCase, as: Stack import Kodo.AccountsFixtures @@ -9,7 +10,7 @@ defmodule Kodo.E2E.HermeticFullStackTest do @moduletag timeout: 120_000 @prompt "KODO_HERMETIC_FULL_STACK_FIX_GREETING" - @model "test:hermetic-full-stack" + @model "openai:gpt-4o-mini" @required_tools ["apply_patch", "read_file", "git_diff"] @http_created_status 201 @http_accepted_status 202 @@ -26,7 +27,13 @@ defmodule Kodo.E2E.HermeticFullStackTest do stack = Stack.start_stack!() workspace = Stack.fixture!() - token = user_fixture() |> Kodo.Accounts.generate_user_agent_token() + user = user_fixture() + scope = Kodo.Accounts.Scope.for_user(user) + + {:ok, _integration} = + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "e2e-test-key"}) + + token = Kodo.Accounts.generate_user_agent_token(user) runner = Stack.start_runner!(stack.base_url, workspace, token) %{stack: stack, workspace: workspace, runner: runner, token: token} end diff --git a/test/kodo/agent/evaluation_runner_test.exs b/test/kodo/agent/evaluation_runner_test.exs index 0dbaf94..7698ac2 100644 --- a/test/kodo/agent/evaluation_runner_test.exs +++ b/test/kodo/agent/evaluation_runner_test.exs @@ -3,6 +3,17 @@ defmodule Kodo.Agent.EvaluationRunnerTest do alias Kodo.Agent.EvaluationRunner + test "evaluation runs require an explicit authenticated user scope" do + scope = %Kodo.Accounts.Scope{user: %Kodo.Accounts.User{id: 123}} + suite = %{"name" => "empty", "version" => 1, "tasks" => []} + + report = EvaluationRunner.run(scope, suite: suite) + + assert report["suite"]["name"] == "empty" + assert report["tasks"] == [] + refute function_exported?(EvaluationRunner, :run, 0) + end + test "search scoring is deterministic and penalizes irrelevant files" do expected = %{ "relevant_files" => ["lib/a.ex"], diff --git a/test/kodo/agent/loop_test.exs b/test/kodo/agent/loop_test.exs index f09739b..0687889 100644 --- a/test/kodo/agent/loop_test.exs +++ b/test/kodo/agent/loop_test.exs @@ -3,6 +3,7 @@ defmodule Kodo.Agent.LoopTest do alias Kodo.Agent.Loop alias Kodo.Agent.Tools + alias Kodo.Integrations alias Kodo.Runners alias Kodo.Sessions @@ -11,6 +12,9 @@ defmodule Kodo.Agent.LoopTest do setup do scope = user_scope_fixture() + {:ok, _integration} = + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "loop-test-key"}) + {:ok, runner} = Runners.register(scope, %{ workspace_root: "/work/#{Ecto.UUID.generate()}", @@ -25,12 +29,12 @@ defmodule Kodo.Agent.LoopTest do Sessions.create_session(scope, %{ runner_id: runner.id, title: "Budgeted turn", - model: "test:model" + model: "openai:gpt-4o-mini" }) {:ok, ownership} = Sessions.claim_ownership(session.id, nil) - %{runner: runner, session: session, ownership: ownership} + %{runner: runner, session: session, ownership: ownership, scope: scope} end test "stops before accepting a response that exceeds the token budget", %{ @@ -56,6 +60,34 @@ defmodule Kodo.Agent.LoopTest do ) end + test "fails before recording an invocation when the required integration is disconnected", %{ + session: session, + ownership: ownership, + scope: scope + } do + {:ok, integration} = Integrations.get_integration_by_provider(scope, "openai") + + assert {:ok, _disconnected} = + Integrations.disconnect(scope, integration.id, integration.credential_generation) + + {:ok, _event} = + Sessions.append_event( + session.id, + "user_message", + %{"role" => "user", "content" => "final answer"}, + ownership: ownership + ) + + assert {:error, :integration_disconnected} = + Loop.run(session.id, + adapter: Kodo.Test.FakeLLM, + budgets: budgets([]), + ownership: ownership + ) + + refute Enum.any?(Sessions.events_after(session.id), &(&1.type == "model_invocation_started")) + end + test "records the resolved role and model mapping for an invocation", %{ session: session, ownership: ownership @@ -79,8 +111,8 @@ defmodule Kodo.Agent.LoopTest do Enum.find(Sessions.events_after(session.id), &(&1.type == "model_invocation_started")) assert invocation.payload["role"] == "primary" - assert invocation.payload["provider"] == "test" - assert invocation.payload["model"] == "test:model" + assert invocation.payload["provider"] == "openai" + assert invocation.payload["model"] == "openai:gpt-4o-mini" assert invocation.payload["reasoning"] == "none" assert invocation.version == 3 assert invocation.payload["role_contract"] == "alpha-v1" @@ -126,7 +158,7 @@ defmodule Kodo.Agent.LoopTest do ) end) - assert_receive {:llm_request, "test:model", system, tools, opts} + assert_receive {:llm_request, %LLMDB.Model{provider: :openai}, system, tools, opts}, 5_000 assert system["content"] == Kodo.Agent.Roles.fetch!(:primary).prompt assert tools == Tools.definitions("workspace-v5") assert opts[:reasoning] == "none" @@ -146,7 +178,7 @@ defmodule Kodo.Agent.LoopTest do Enum.find(Sessions.events_after(session.id), &(&1.type == "model_invocation_started")) assert invocation.version == 3 - assert invocation.payload["model"] == "test:model" + assert invocation.payload["model"] == "openai:gpt-4o-mini" assert invocation.payload["role_contract"] == "alpha-v1" assert invocation.payload["toolset_version"] == "workspace-v5" @@ -200,7 +232,7 @@ defmodule Kodo.Agent.LoopTest do ) end) - assert_receive {:llm_request, "test:model", system, tools, opts} + assert_receive {:llm_request, %LLMDB.Model{provider: :openai}, system, tools, opts}, 5_000 assert system["content"] == Kodo.Agent.Roles.fetch!(:primary).prompt assert tools == Tools.definitions("workspace-v5") assert opts[:reasoning] == "none" @@ -218,7 +250,7 @@ defmodule Kodo.Agent.LoopTest do invocation = Enum.find(Sessions.events_after(session.id), &(&1.type == "model_invocation_started")) - assert invocation.payload["model"] == "test:model" + assert invocation.payload["model"] == "openai:gpt-4o-mini" assert invocation.payload["role_contract"] == "alpha-v1" assert invocation.payload["toolset_version"] == "workspace-v5" end diff --git a/test/kodo/cluster/distributed_handoff_test.exs b/test/kodo/cluster/distributed_handoff_test.exs index f53a2a1..b8145a6 100644 --- a/test/kodo/cluster/distributed_handoff_test.exs +++ b/test/kodo/cluster/distributed_handoff_test.exs @@ -6,6 +6,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do alias Kodo.Cluster.InstanceManager alias Kodo.Cluster.Instances alias Kodo.Cluster.Placement + alias Kodo.Integrations alias Kodo.Repo alias Kodo.Runners alias Kodo.Sessions @@ -59,6 +60,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do user = user_fixture() scope = Scope.for_user(user) + {:ok, _integration} = connect_integration(scope) {:ok, runner} = Runners.register(scope, %{ @@ -74,7 +76,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do Sessions.create_session(scope, %{ runner_id: runner.id, title: "Distributed handoff", - model: "test:model", + model: "openai:gpt-4o-mini", approval_policy: "safe" }) @@ -142,6 +144,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do user = user_fixture() scope = Scope.for_user(user) + {:ok, _integration} = connect_integration(scope) {:ok, runner} = Runners.register(scope, %{ @@ -159,7 +162,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do Sessions.create_session(scope, %{ runner_id: runner.id, title: "Node-loss replay", - model: "test:model", + model: "openai:gpt-4o-mini", approval_policy: "safe" }) @@ -202,6 +205,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do defp assert_competing_claims_are_fenced(source, target, peer_node) do user = user_fixture() scope = Scope.for_user(user) + {:ok, _integration} = connect_integration(scope) {:ok, runner} = Runners.register(scope, %{ @@ -217,7 +221,7 @@ defmodule Kodo.Cluster.DistributedHandoffTest do Sessions.create_session(scope, %{ runner_id: runner.id, title: "Competing distributed claims", - model: "test:model" + model: "openai:gpt-4o-mini" }) local_claim = Task.async(fn -> Sessions.claim_ownership(session.id, source.boot_id) end) @@ -321,11 +325,15 @@ defmodule Kodo.Cluster.DistributedHandoffTest do options = instance_options(revision, generation, Atom.to_string(peer_node)) agent_budgets = Application.fetch_env!(:kodo, :agent_budgets) + credential_encryption = + Application.fetch_env!(:kodo, Kodo.Integrations.CredentialEncryption) + {:ok, supervisor} = :erpc.call(peer_node, Kodo.Test.ClusterPeer, :start, [ repo_config, options, agent_budgets, + credential_encryption, self() ]) @@ -355,6 +363,10 @@ defmodule Kodo.Cluster.DistributedHandoffTest do ] end + defp connect_integration(scope) do + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "distributed-test-key"}) + end + defp cleanup_cluster_rows do Repo.delete_all(Kodo.Cluster.PlacementOverride) Repo.delete_all(Kodo.Sessions.Event) diff --git a/test/kodo/llm/credential_resolver_test.exs b/test/kodo/llm/credential_resolver_test.exs new file mode 100644 index 0000000..667c088 --- /dev/null +++ b/test/kodo/llm/credential_resolver_test.exs @@ -0,0 +1,242 @@ +defmodule Kodo.LLM.CredentialResolverTest do + use Kodo.DataCase, async: true + + alias Kodo.AccountsFixtures + alias Kodo.Integrations + alias Kodo.LLM + alias Kodo.LLM.CredentialResolver + alias Kodo.LLM.IntegrationRef + + setup do + scope = AccountsFixtures.user_scope_fixture() + + {:ok, integration} = + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "scoped-secret"}) + + %{ + scope: scope, + integration: integration, + reference: IntegrationRef.from_integration(integration) + } + end + + test "resolves only the referenced owned credential for a matching model", context do + assert {:ok, credential} = resolve(context) + assert credential.integration_id == context.integration.id + assert credential.provider == "openai" + assert credential.authentication_type == "api_key" + assert credential.credential_generation == context.integration.credential_generation + assert credential.billing_path == :platform + assert credential.token == "scoped-secret" + assert credential.account_id == nil + refute inspect(credential) =~ "scoped-secret" + end + + test "the public facade rejects credential options and stale preflight references", context do + assert {:ok, resolved_model, reference} = + LLM.resolve_integration(context.scope, "openai:gpt-4o-mini") + + assert {:error, :credential_options_not_allowed} = + LLM.generate( + context.scope, + resolved_model, + reference, + [%{"role" => "user", "content" => "final answer"}], + [], + adapter: Kodo.Test.FakeLLM, + timeout: 1_000, + api_key: "caller-secret" + ) + + assert {:ok, _replaced} = + Integrations.replace_credentials( + context.scope, + context.integration.id, + context.integration.credential_generation, + %{"api_key" => "replacement"} + ) + + assert {:error, :stale_credential_generation} = + LLM.generate( + context.scope, + resolved_model, + reference, + [%{"role" => "user", "content" => "final answer"}], + [], + adapter: Kodo.Test.FakeLLM, + timeout: 1_000 + ) + end + + test "rejects forged and cross-user references", context do + other_scope = AccountsFixtures.user_scope_fixture() + + assert {:error, :integration_not_found} = + resolve(%{context | scope: other_scope}) + + forged = %{context.reference | integration_id: Ecto.UUID.generate()} + assert {:error, :integration_not_found} = resolve(%{context | reference: forged}) + end + + test "rejects stale references after replacement or disconnection", context do + assert {:ok, replaced} = + Integrations.replace_credentials( + context.scope, + context.integration.id, + context.integration.credential_generation, + %{"api_key" => "replacement"} + ) + + assert {:error, :stale_credential_generation} = resolve(context) + + replacement_reference = IntegrationRef.from_integration(replaced) + + assert {:ok, _disconnected} = + Integrations.disconnect( + context.scope, + replaced.id, + replaced.credential_generation + ) + + assert {:error, :stale_credential_generation} = + resolve(%{context | reference: replacement_reference}) + end + + test "rejects disconnected and confirmed-invalid integrations", context do + assert {:ok, disconnected} = + Integrations.disconnect( + context.scope, + context.integration.id, + context.integration.credential_generation + ) + + assert {:error, :integration_disconnected} = + resolve(%{context | reference: IntegrationRef.from_integration(disconnected)}) + + other_scope = AccountsFixtures.user_scope_fixture() + + assert {:ok, invalid} = + Integrations.connect(other_scope, "openai", "api_key", %{"api_key" => "invalid"}) + + assert {:ok, invalid} = + Integrations.validation_invalid( + other_scope, + invalid.id, + invalid.credential_generation + ) + + assert {:error, :integration_invalid} = + resolve(%{ + context + | scope: other_scope, + integration: invalid, + reference: IntegrationRef.from_integration(invalid) + }) + end + + test "admits unverified and validation-unavailable integrations", context do + assert {:ok, _credential} = resolve(context) + + assert {:ok, unavailable} = + Integrations.validation_unavailable( + context.scope, + context.integration.id, + context.integration.credential_generation, + "timeout" + ) + + assert {:ok, _credential} = + resolve(%{context | reference: IntegrationRef.from_integration(unavailable)}) + end + + test "captures only usable integration metadata during preflight", context do + assert {:ok, reference} = CredentialResolver.reference(context.scope, model()) + assert reference == context.reference + + assert {:ok, invalid} = + Integrations.validation_invalid( + context.scope, + context.integration.id, + context.integration.credential_generation + ) + + assert {:error, :integration_invalid} = + CredentialResolver.reference(context.scope, model()) + + assert invalid.credential_generation == reference.credential_generation + end + + test "requires exact model, reference, and stored providers", context do + anthropic_model = LLMDB.Model.new!(%{id: "claude", provider: :anthropic}) + + assert {:error, :integration_provider_mismatch} = + CredentialResolver.resolve(context.scope, anthropic_model, context.reference) + + forged = %{context.reference | provider: "anthropic"} + + assert {:error, :integration_provider_mismatch} = + resolve(%{context | reference: forged}) + + unsupported_model = LLMDB.Model.new!(%{id: "local", provider: :ollama}) + + assert {:error, :unsupported_model_provider} = + CredentialResolver.resolve(context.scope, unsupported_model, context.reference) + end + + test "rejects malformed decrypted payloads with a bounded error" do + other_scope = AccountsFixtures.user_scope_fixture() + + assert {:ok, malformed} = + Integrations.connect(other_scope, "openai", "api_key", %{"unexpected" => "secret"}) + + assert {:error, :credential_payload_invalid} = + CredentialResolver.resolve( + other_scope, + model(), + IntegrationRef.from_integration(malformed) + ) + end + + test "resolves Codex access and account identity without exposing refresh state" do + scope = AccountsFixtures.user_scope_fixture() + + integration = %Kodo.Integrations.Integration{ + id: Ecto.UUID.generate(), + user_id: scope.user.id, + provider: "openai_codex", + authentication_type: "oauth" + } + + integration = Kodo.Repo.insert!(integration) + + assert {:ok, connected} = + Integrations.oauth_succeeded(scope, integration.id, 0, %{ + "access_token" => "access-secret", + "refresh_token" => "refresh-secret", + "account_id" => "account-secret" + }) + + codex_model = LLMDB.Model.new!(%{id: "codex", provider: :openai_codex}) + + assert {:ok, credential} = + CredentialResolver.resolve( + scope, + codex_model, + IntegrationRef.from_integration(connected) + ) + + assert credential.token == "access-secret" + assert credential.account_id == "account-secret" + assert credential.billing_path == :subscription + inspected = inspect(credential) + refute inspected =~ "access-secret" + refute inspected =~ "refresh-secret" + refute inspected =~ "account-secret" + end + + defp resolve(context) do + CredentialResolver.resolve(context.scope, model(), context.reference) + end + + defp model, do: LLMDB.Model.new!(%{id: "gpt", provider: :openai}) +end diff --git a/test/kodo/llm/req_llm_test.exs b/test/kodo/llm/req_llm_test.exs index eb8de57..cb8709e 100644 --- a/test/kodo/llm/req_llm_test.exs +++ b/test/kodo/llm/req_llm_test.exs @@ -3,6 +3,15 @@ defmodule Kodo.LLM.ReqLLMTest do alias Kodo.LLM.ReqLLM, as: Adapter + @credential %Kodo.LLM.Credential{ + integration_id: "00000000-0000-0000-0000-000000000001", + provider: "openai", + authentication_type: "api_key", + credential_generation: 1, + billing_path: :platform, + token: "request-local-key" + } + test "translates Kodo tool definitions into strict ReqLLM tools" do [tool] = Adapter.build_tools([ @@ -27,7 +36,7 @@ defmodule Kodo.LLM.ReqLLMTest do adapter = Kodo.LLM.adapter() assert Code.ensure_loaded?(adapter) - assert function_exported?(adapter, :generate, 4) + assert function_exported?(adapter, :generate, 5) end test "applies the model budget to provider receive and total timeouts" do @@ -37,6 +46,35 @@ defmodule Kodo.LLM.ReqLLMTest do assert options[:total_timeout] == 12_345 end + test "overrides ambient API keys with the operation-local credential" do + options = + Adapter.request_options([], @credential, + timeout: 12_345, + reasoning: "none", + api_key: "caller-supplied-key" + ) + + assert options[:api_key] == "request-local-key" + end + + test "passes only current Codex access and account credentials" do + credential = %{ + @credential + | provider: "openai_codex", + authentication_type: "oauth", + billing_path: :subscription, + token: "request-local-access", + account_id: "request-local-account" + } + + options = Adapter.request_options([], credential, timeout: 12_345, reasoning: "none") + + assert options[:auth_mode] == :oauth + assert options[:access_token] == "request-local-access" + assert options[:chatgpt_account_id] == "request-local-account" + refute Keyword.has_key?(options, :refresh_token) + end + test "rejects catalog models whose provider cannot be dispatched" do assert {:error, {:model_not_available, _reason}} = Adapter.validate_model( diff --git a/test/kodo/sessions/active_session_test.exs b/test/kodo/sessions/active_session_test.exs index 6f47a84..3266357 100644 --- a/test/kodo/sessions/active_session_test.exs +++ b/test/kodo/sessions/active_session_test.exs @@ -5,6 +5,7 @@ defmodule Kodo.Sessions.ActiveSessionTest do alias Kodo.Cluster.Discovery alias Kodo.Cluster.InstanceManager alias Kodo.Cluster.Instances + alias Kodo.Integrations alias Kodo.Sessions alias Kodo.Sessions.ActiveSession alias Kodo.Sessions.Recovery @@ -19,6 +20,9 @@ defmodule Kodo.Sessions.ActiveSessionTest do scope = user_scope_fixture() + {:ok, _integration} = + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "active-test-key"}) + {:ok, runner} = Runners.register(scope, %{ workspace_root: "/work/#{Ecto.UUID.generate()}", diff --git a/test/kodo_web/controllers/session_controller_test.exs b/test/kodo_web/controllers/session_controller_test.exs index 0de40b0..2cfd375 100644 --- a/test/kodo_web/controllers/session_controller_test.exs +++ b/test/kodo_web/controllers/session_controller_test.exs @@ -3,6 +3,7 @@ defmodule KodoWeb.SessionControllerTest do alias Kodo.Runners alias Kodo.Sessions + alias Kodo.Integrations import Kodo.AccountsFixtures @@ -29,6 +30,9 @@ defmodule KodoWeb.SessionControllerTest do user = user_fixture() scope = Kodo.Accounts.Scope.for_user(user) + {:ok, _integration} = + Integrations.connect(scope, "openai", "api_key", %{"api_key" => "controller-test-key"}) + {:ok, runner} = Runners.register(scope, %{ workspace_root: "/work/#{Ecto.UUID.generate()}", @@ -76,7 +80,7 @@ defmodule KodoWeb.SessionControllerTest do assert response == %{"status" => "running"} - assert_receive {:tool_request, request} + assert_receive {:tool_request, request}, 5_000 assert request["request"]["tool"] == "apply_patch" assert request["authority"]["session_id"] == session["id"] assert request["authority"]["ownership_epoch"] > 0 @@ -146,7 +150,7 @@ defmodule KodoWeb.SessionControllerTest do |> post_json(~p"/api/sessions/#{session["id"]}/messages", %{content: "wait"}) |> json_response(202) - assert_receive :fake_llm_waiting + assert_receive :fake_llm_waiting, 5_000 response = conn @@ -182,7 +186,8 @@ defmodule KodoWeb.SessionControllerTest do %{ type: "approval_requested", payload: %{"approval_id" => approval_id, "name" => "apply_patch"} - }} + }}, + 5_000 refute_received {:tool_request, _request} @@ -195,7 +200,7 @@ defmodule KodoWeb.SessionControllerTest do |> json_response(200) assert response == %{"decision" => "approved", "status" => "running"} - assert_receive {:tool_request, request} + assert_receive {:tool_request, request}, 5_000 assert conn |> recycle(["accept", "authorization"]) @@ -240,7 +245,8 @@ defmodule KodoWeb.SessionControllerTest do |> json_response(202) assert_receive {:session_event, - %{type: "session_status_changed", payload: %{"status" => "failed"}}} + %{type: "session_status_changed", payload: %{"status" => "failed"}}}, + 5_000 assert Enum.any?(Sessions.events_after(session["id"]), &(&1.type == "session_failed")) end @@ -286,7 +292,7 @@ defmodule KodoWeb.SessionControllerTest do |> post_json(~p"/api/sessions", %{ runner_id: runner.id, title: "Fix greeting", - model: "test:model", + model: "openai:gpt-4o-mini", approval_policy: Keyword.get(opts, :approval_policy, "standard") }) |> json_response(201) @@ -309,7 +315,7 @@ defmodule KodoWeb.SessionControllerTest do end defp respond_to_review(runner_id) do - assert_receive {:tool_request, request} + assert_receive {:tool_request, request}, 5_000 assert request["request"]["tool"] == "git_diff" Phoenix.PubSub.broadcast( diff --git a/test/support/cluster_peer.ex b/test/support/cluster_peer.ex index 123ec4d..6e6070c 100644 --- a/test/support/cluster_peer.ex +++ b/test/support/cluster_peer.ex @@ -1,9 +1,10 @@ defmodule Kodo.Test.ClusterPeer do @moduledoc false - def start(repo_config, instance_options, agent_budgets, test_pid) do + def start(repo_config, instance_options, agent_budgets, credential_encryption, test_pid) do Application.put_env(:kodo, Kodo.Repo, repo_config) Application.put_env(:kodo, Kodo.Cluster.InstanceManager, instance_options) + Application.put_env(:kodo, Kodo.Integrations.CredentialEncryption, credential_encryption) Application.put_env(:kodo, :agent_budgets, agent_budgets) Application.put_env(:kodo, :llm_adapter, Kodo.Test.FakeLLM) Application.put_env(:kodo, :fake_llm_test_pid, test_pid) diff --git a/test/support/fake_llm.ex b/test/support/fake_llm.ex index e6b996f..7606f7b 100644 --- a/test/support/fake_llm.ex +++ b/test/support/fake_llm.ex @@ -32,7 +32,7 @@ defmodule Kodo.Test.FakeLLM do end @impl true - def generate(model, messages, tools, opts) do + def generate(model, messages, tools, _credential, opts) do last = List.last(messages) force_final_turn? = @@ -61,7 +61,7 @@ defmodule Kodo.Test.FakeLLM do end @impl true - def generate_object(_model, messages, _schema, _opts) do + def generate_object(_model, messages, _schema, _credential, _opts) do if test_pid = Application.get_env(:kodo, :fake_llm_review_pid) do send(test_pid, {:review_messages, messages}) end