Reach OpenRouter through workload identity federation in CI

.github/workflows/weekly-maintenance-prompt.md

@@ -2,7 +2,7 @@ You are operating in a GitHub Actions runner for the kernel-tinker-rl repository
 
 The GitHub CLI is available as `gh`, and Git is authenticated as the `kernel-internal[bot]` GitHub App. Their credential helpers renew the installation token before it expires; use the normal `gh` and `git` commands, and never bypass the helpers or print credentials. You may push branches, create pull requests, reply to review comments, and merge pull requests. Pi Web Access is installed; call `web_enable` before using `web_search`, `fetch_content`, or `get_search_content` for release notes and migration guidance.
 
-`uv`, Node, and the Kernel CLI (`kernel`) are installed. `KERNEL_API_KEY`, `TINKER_API_KEY`, and `OPENROUTER_API_KEY` are set for smoke tests and model baselines.
+`uv`, Node, and the Kernel CLI (`kernel`) are installed. `KERNEL_API_KEY` and `TINKER_API_KEY` are set for smoke tests and model baselines. WebJudge reaches OpenRouter with a short-lived token from `OPENROUTER_ACCESS_TOKEN_FILE`; there is no `OPENROUTER_API_KEY`, so don't ask for one.
 
 # Context
 

.github/workflows/weekly-maintenance.yml

@@ -185,14 +185,15 @@ jobs:
           openai-identity-provider-id: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }}
           openai-service-account-id: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }}
           openai-audience: ${{ vars.OPENAI_WIF_AUDIENCE }}
+          # WebJudge calls OpenRouter; it reads the token from OPENROUTER_ACCESS_TOKEN_FILE.
+          openrouter-federation-policy-id: ${{ vars.OPENROUTER_FEDERATION_POLICY_ID }}
 
       - name: Run weekly maintenance
         env:
           CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
           GITHUB_CHECKS_TOKEN: ${{ github.token }}
           GIT_TERMINAL_PROMPT: "0"
           KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }}
-          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
           PI_PREFERRED_MODEL: ${{ vars.PI_PREFERRED_MODEL }}
           PI_SKIP_VERSION_CHECK: "1"
@@ -202,7 +203,7 @@ jobs:
           : "${PI_PREFERRED_MODEL:?Set the PI_PREFERRED_MODEL organization variable}"
           : "${KERNEL_API_KEY:?Set KERNEL_API_KEY}"
           : "${TINKER_API_KEY:?Set TINKER_API_KEY}"
-          : "${OPENROUTER_API_KEY:?Set OPENROUTER_API_KEY}"
+          : "${OPENROUTER_ACCESS_TOKEN_FILE:?Set the OPENROUTER_FEDERATION_POLICY_ID organization variable}"
           rendered_prompt="$RUNNER_TEMP/weekly-maintenance-prompt.md"
           envsubst '${GITHUB_REPOSITORY} ${OUTCOME_FILE} ${POOL_NAME}' \
             < .github/workflows/weekly-maintenance-prompt.md \

core/reward_models/webjudge.py

@@ -21,8 +21,9 @@
 import os
 import re
 from dataclasses import dataclass, field
+from pathlib import Path
 
-from openai import AsyncOpenAI
+from openai import AsyncOpenAI, DefaultAsyncHttpxClient
 from PIL import Image
 
 from .base import RewardModel, Trajectory
@@ -80,6 +81,37 @@ class WebJudgeResult:
     screenshot_scores: list[dict] = field(default_factory=list)
 
 
+class _TokenFileAuth:
+    """Sends the token in a file as the bearer, reading the file for every request.
+
+    In CI the file holds a short-lived OpenRouter access token that a background process
+    replaces before it expires, so it can't be read once at startup. A plain callable rather
+    than an httpx.Auth subclass, because the SDK's HTTP client is not always httpx itself.
+    """
+
+    def __init__(self, path: str):
+        self.path = Path(path)
+
+    def __call__(self, request):
+        request.headers["Authorization"] = f"Bearer {self.path.read_text().strip()}"
+        return request
+
+
+def _openrouter_client(api_key: str | None, base_url: str) -> AsyncOpenAI:
+    """An OpenRouter client using, in order: api_key, OPENROUTER_API_KEY, or the access token
+    in OPENROUTER_ACCESS_TOKEN_FILE (set by workload identity federation in CI)."""
+    key = api_key or os.getenv("OPENROUTER_API_KEY")
+    token_file = os.getenv("OPENROUTER_ACCESS_TOKEN_FILE")
+    if key or not token_file:
+        return AsyncOpenAI(api_key=key, base_url=base_url)
+    # The SDK requires an api_key; the auth hook replaces the header it builds from it.
+    return AsyncOpenAI(
+        api_key="federated",
+        base_url=base_url,
+        http_client=DefaultAsyncHttpxClient(auth=_TokenFileAuth(token_file)),
+    )
+
+
 class WebJudge(RewardModel):
     """
     WebJudge evaluator for web navigation trajectories.
@@ -107,16 +139,14 @@ def __init__(
 
         Args:
             model: Model name (OpenRouter format, e.g., "openai/gpt-5-mini")
-            api_key: API key for OpenRouter. Falls back to OPENROUTER_API_KEY env var.
+            api_key: API key for OpenRouter. Falls back to the OPENROUTER_API_KEY env var, then
+                to the access token in OPENROUTER_ACCESS_TOKEN_FILE.
             base_url: API base URL. Defaults to OpenRouter.
             evaluation_criteria: Custom evaluation criteria for the judgment phase.
                 If None, uses DEFAULT_EVALUATION_CRITERIA (designed for e-commerce tasks).
                 Override this for different task types (e.g., authentication, navigation).
         """
-        self.client = AsyncOpenAI(
-            api_key=api_key or os.getenv("OPENROUTER_API_KEY"),
-            base_url=base_url,
-        )
+        self.client = _openrouter_client(api_key, base_url)
         self.model = model
         self.evaluation_criteria = evaluation_criteria
 

pyproject.toml

@@ -35,3 +35,4 @@ ignore = ["E501"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+pythonpath = ["."]

tests/test_webjudge_auth.py

@@ -0,0 +1,58 @@
+import httpx2
+import pytest
+
+from core.reward_models import webjudge
+
+
+def test_token_file_is_read_for_every_request(tmp_path):
+    token = tmp_path / "token"
+    token.write_text("first\n")
+    auth = webjudge._TokenFileAuth(str(token))
+
+    request = auth(httpx2.Request("POST", "https://openrouter.ai/api/v1/x"))
+    assert request.headers["Authorization"] == "Bearer first"
+
+    token.write_text("second")
+    request = auth(httpx2.Request("POST", "https://openrouter.ai/api/v1/x"))
+    assert request.headers["Authorization"] == "Bearer second"
+
+
+@pytest.mark.parametrize(
+    ("api_key", "env_key", "expect_federated"),
+    [("explicit", None, False), (None, "from-env", False), (None, None, True)],
+)
+def test_client_prefers_a_key_over_the_token_file(
+    tmp_path, monkeypatch, api_key, env_key, expect_federated
+):
+    token = tmp_path / "token"
+    token.write_text("access")
+    monkeypatch.setenv("OPENROUTER_ACCESS_TOKEN_FILE", str(token))
+    if env_key:
+        monkeypatch.setenv("OPENROUTER_API_KEY", env_key)
+    else:
+        monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    client = webjudge._openrouter_client(api_key, "https://openrouter.ai/api/v1")
+
+    assert (client.api_key == "federated") == expect_federated
+    if not expect_federated:
+        assert client.api_key == (api_key or env_key)
+
+
+async def test_federated_client_sends_the_current_token(tmp_path, monkeypatch):
+    token = tmp_path / "token"
+    token.write_text("access-1")
+    monkeypatch.setenv("OPENROUTER_ACCESS_TOKEN_FILE", str(token))
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    seen = []
+
+    def handler(request):
+        seen.append(request.headers["Authorization"])
+        return httpx2.Response(200, json={"object": "list", "data": []})
+
+    client = webjudge._openrouter_client(None, "https://openrouter.ai/api/v1")
+    client._client._transport = httpx2.MockTransport(handler)
+    await client.models.list()
+    token.write_text("access-2")
+    await client.models.list()
+    assert seen == ["Bearer access-1", "Bearer access-2"]