diff --git a/.github/actions/llm-credentials/action.yml b/.github/actions/llm-credentials/action.yml new file mode 100644 index 0000000..0437f12 --- /dev/null +++ b/.github/actions/llm-credentials/action.yml @@ -0,0 +1,81 @@ +name: LLM credentials +description: >- + Short-lived Anthropic or OpenAI credentials for pi through workload identity federation, so + the job needs no stored API key. The job needs `permissions: id-token: write`. Credentials are + refreshed in the background, so later steps run pi with no key in their env. Run the action + again with `stop: true` once the model is done, to delete the credentials before later steps. + +inputs: + model: + description: >- + Pi model as provider/id, such as the PI_PREFERRED_MODEL variable. Its provider, anthropic or + openai, picks the credentials. Required unless stop is true. + stop: + description: Stop refreshing, delete the credentials and print the refresh log. + default: 'false' + # The federation IDs are not secrets: they name the provider-side rules that this org's GitHub + # OIDC tokens match. Pass the kernel org variables; action metadata can't read `vars` itself. + # Only the model's provider's IDs are needed. + anthropic-federation-rule-id: + description: Anthropic federation rule ID (fdrl_...) + anthropic-organization-id: + description: Anthropic organization ID + anthropic-service-account-id: + description: Anthropic service account ID (svac_...) + openai-identity-provider-id: + description: OpenAI workload identity provider ID (idp_...) + openai-service-account-id: + description: OpenAI service account ID + openai-audience: + description: Audience the OpenAI workload identity provider requires + +runs: + using: composite + steps: + - name: Model credentials + shell: bash + env: + MODEL: ${{ inputs.model }} + STOP: ${{ inputs.stop }} + ANTHROPIC_FEDERATION_RULE_ID: ${{ inputs.anthropic-federation-rule-id }} + ANTHROPIC_ORGANIZATION_ID: ${{ inputs.anthropic-organization-id }} + ANTHROPIC_SERVICE_ACCOUNT_ID: ${{ inputs.anthropic-service-account-id }} + ANTHROPIC_IDENTITY_TOKEN_FILE: ${{ runner.temp }}/anthropic-identity-token + OPENAI_IDENTITY_PROVIDER_ID: ${{ inputs.openai-identity-provider-id }} + OPENAI_SERVICE_ACCOUNT_ID: ${{ inputs.openai-service-account-id }} + OPENAI_WIF_AUDIENCE: ${{ inputs.openai-audience }} + run: | + set -euo pipefail + if [ "$STOP" = true ]; then + exec bash "$GITHUB_ACTION_PATH/llm_credentials.sh" stop + fi + : "${MODEL:?Set the model input to a pi model as provider/id, such as the PI_PREFERRED_MODEL variable}" + provider=${MODEL%%/*} + # require + require() { + if [ -z "${!1}" ]; then + echo "Set the $2 input from the $3 org variable. If it is set, check that the variable is visible to this repository." >&2 + exit 1 + fi + } + case "$provider" in + anthropic) + require ANTHROPIC_FEDERATION_RULE_ID anthropic-federation-rule-id ANTHROPIC_FEDERATION_RULE_ID + require ANTHROPIC_ORGANIZATION_ID anthropic-organization-id ANTHROPIC_ORGANIZATION_ID + require ANTHROPIC_SERVICE_ACCOUNT_ID anthropic-service-account-id ANTHROPIC_SERVICE_ACCOUNT_ID + ;; + openai) + require OPENAI_IDENTITY_PROVIDER_ID openai-identity-provider-id OPENAI_IDENTITY_PROVIDER_ID + require OPENAI_SERVICE_ACCOUNT_ID openai-service-account-id OPENAI_SERVICE_ACCOUNT_ID + require OPENAI_WIF_AUDIENCE openai-audience OPENAI_WIF_AUDIENCE + ;; + esac + bash "$GITHUB_ACTION_PATH/llm_credentials.sh" start "$provider" + # The Anthropic SDK in pi does the exchange itself, so later steps need the federation + # settings. OpenAI's token is already in pi's auth.json. + if [ "$provider" = anthropic ]; then + for name in ANTHROPIC_FEDERATION_RULE_ID ANTHROPIC_ORGANIZATION_ID \ + ANTHROPIC_SERVICE_ACCOUNT_ID ANTHROPIC_IDENTITY_TOKEN_FILE; do + echo "$name=${!name}" >> "$GITHUB_ENV" + done + fi diff --git a/.github/actions/llm-credentials/llm_credentials.sh b/.github/actions/llm-credentials/llm_credentials.sh new file mode 100644 index 0000000..12008c5 --- /dev/null +++ b/.github/actions/llm-credentials/llm_credentials.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Short-lived model credentials for pi in GitHub Actions, through workload identity federation +# instead of stored API keys. The job needs `permissions: id-token: write`. +# +# llm_credentials.sh start fetch once, then refresh in the background +# llm_credentials.sh refresh fetch once +# llm_credentials.sh stop stop refreshing, delete the credentials and +# print the refresh log +# +# GitHub's OIDC tokens last about five minutes, and each provider's credential expires no later +# than the token it was exchanged for, so a run longer than that needs the background refresh. +# It runs every $LLM_CREDENTIALS_INTERVAL seconds (default 30) until `stop` or the end of the job. +# +# anthropic: writes a fresh GitHub token to $ANTHROPIC_IDENTITY_TOKEN_FILE. The Anthropic SDK in +# pi exchanges it using ANTHROPIC_FEDERATION_RULE_ID, ANTHROPIC_ORGANIZATION_ID and +# ANTHROPIC_SERVICE_ACCOUNT_ID, and rereads the file before each refresh. Each token can be +# exchanged only once, and the SDK retries a failed refresh by rereading the file, so the file +# rotates well inside the 90 seconds between the SDK's first refresh attempt and its deadline. +# openai: exchanges a GitHub token for an OpenAI access token using OPENAI_IDENTITY_PROVIDER_ID, +# OPENAI_SERVICE_ACCOUNT_ID and OPENAI_WIF_AUDIENCE, and sets it as the openai entry of pi's +# auth.json, which pi rereads when the file changes. Other entries are kept. +set -euo pipefail +umask 077 + +state=${RUNNER_TEMP:-/tmp} +pidfile=$state/llm-credentials.pid +log=$state/llm-credentials.log +auth=${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}/auth.json + +# The functions below return on failure explicitly: `set -e` does not apply inside a function +# called from `refresh || ...`, and a failed fetch must never overwrite a working credential. + +github_token() { + local audience + audience=$(jq -rn --arg a "$1" '$a | @uri') || return 1 + curl -fsS --retry 3 -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$audience" | jq -re .value +} + +# write : replace the file in one step so a reader never sees a partial write. +write() { + printf '%s' "$2" > "$1.tmp" && mv "$1.tmp" "$1" +} + +refresh_anthropic() { + local token + token=$(github_token https://api.anthropic.com) || return 1 + write "${ANTHROPIC_IDENTITY_TOKEN_FILE:?}" "$token" +} + +refresh_openai() { + local token access current + token=$(github_token "${OPENAI_WIF_AUDIENCE:?}") || return 1 + access=$(jq -n --arg token "$token" --arg provider "${OPENAI_IDENTITY_PROVIDER_ID:?}" \ + --arg account "${OPENAI_SERVICE_ACCOUNT_ID:?}" \ + '{grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", + subject_token_type: "urn:ietf:params:oauth:token-type:jwt", subject_token: $token, + identity_provider_id: $provider, service_account_id: $account}' | + curl -fsS --retry 3 https://auth.openai.com/oauth/token -H 'Content-Type: application/json' -d @- | + jq -re .access_token) || return 1 + mkdir -p "$(dirname "$auth")" || return 1 + current='{}' + if [ -s "$auth" ]; then + current=$(cat "$auth") || return 1 + fi + # A file that isn't valid JSON is an error rather than something to overwrite. + current=$(jq -e --arg key "$access" '.openai = {type: "api_key", key: $key}' <<<"$current") || return 1 + write "$auth" "$current" +} + +refresh() { + case "$1" in + anthropic | openai) "refresh_$1" ;; + *) echo "no federated credentials for provider '$1'" >&2; return 1 ;; + esac +} + +stop_loop() { + local group _ + if [ -f "$pidfile" ]; then + # The loop leads its own process group, so this also stops a refresh in flight before the + # credentials are deleted below. + group=$(cat "$pidfile") + kill -TERM -- "-$group" 2>/dev/null || true + for _ in $(seq 50); do + kill -0 -- "-$group" 2>/dev/null || break + sleep 0.1 + done + kill -KILL -- "-$group" 2>/dev/null || true + rm -f "$pidfile" + fi +} + +stop() { + local status=0 updated + stop_loop + if [ -n "${ANTHROPIC_IDENTITY_TOKEN_FILE:-}" ]; then + rm -f "$ANTHROPIC_IDENTITY_TOKEN_FILE" "$ANTHROPIC_IDENTITY_TOKEN_FILE.tmp" + fi + rm -f "$auth.tmp" + if [ -s "$auth" ]; then + if updated=$(jq -e 'del(.openai)' "$auth"); then + write "$auth" "$updated" + else + echo "$auth is not valid JSON; left it as is" >&2 + status=1 + fi + fi + if [ -s "$log" ]; then + echo "Credential refresh log:" + cat "$log" + fi + return "$status" +} + +usage="usage: llm_credentials.sh start|refresh | stop" +case "${1:-}" in + start) + # A second start replaces the first loop rather than orphaning it. + stop_loop + refresh "${2:?$usage}" + setsid bash "${BASH_SOURCE[0]}" loop "$2" >"$log" 2>&1 & + for _ in $(seq 50); do + [ -s "$pidfile" ] && exit 0 + sleep 0.1 + done + echo "credential refresh loop did not start" >&2 + exit 1 + ;; + refresh) refresh "${2:?$usage}" ;; + stop) stop ;; + loop) + echo "$$" > "$pidfile" + while sleep "${LLM_CREDENTIALS_INTERVAL:-30}"; do + refresh "$2" || echo "$(date -u +%FT%TZ) $2 credential refresh failed" + done + ;; + *) echo "$usage" >&2; exit 1 ;; +esac diff --git a/.github/workflows/pr-risk-tests.yml b/.github/workflows/pr-risk-tests.yml index 78aef4e..a9cddc2 100644 --- a/.github/workflows/pr-risk-tests.yml +++ b/.github/workflows/pr-risk-tests.yml @@ -8,6 +8,8 @@ on: - .github/workflows/pr-risk/** - scripts/pr_risk.py - scripts/test_pr_risk.py + - .github/actions/llm-credentials/** + - scripts/test_llm_credentials.py push: branches: [main] paths: @@ -16,6 +18,8 @@ on: - .github/workflows/pr-risk/** - scripts/pr_risk.py - scripts/test_pr_risk.py + - .github/actions/llm-credentials/** + - scripts/test_llm_credentials.py permissions: contents: read @@ -29,3 +33,23 @@ jobs: python3 -m venv /tmp/pr-risk-tests /tmp/pr-risk-tests/bin/pip install PyYAML==6.0.3 /tmp/pr-risk-tests/bin/python -m unittest discover -s scripts -p test_pr_risk.py + /tmp/pr-risk-tests/bin/python -m unittest discover -s scripts -p test_llm_credentials.py + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: shellcheck and actionlint + run: | + set -euo pipefail + tarball=actionlint_1.7.7_linux_amd64.tar.gz + curl -fsSLO "https://github.com/rhysd/actionlint/releases/download/v1.7.7/$tarball" + echo "023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 $tarball" | sha256sum -c - + tar -xzf "$tarball" actionlint + shellcheck --version + shellcheck .github/actions/llm-credentials/llm_credentials.sh + # actionlint doesn't check the `run` block of a composite action. + yq '.runs.steps[0].run' .github/actions/llm-credentials/action.yml > "$RUNNER_TEMP/action-run.sh" + shellcheck --shell=bash "$RUNNER_TEMP/action-run.sh" + # actionlint also runs shellcheck on each `run` block. + ./actionlint .github/workflows/pr-risk.yml .github/workflows/pr-risk-tests.yml diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 91ec2f3..b88fced 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -13,6 +13,9 @@ name: PR Risk # Credentials are split by step. The model runs with no tools and no GitHub token, reading # only the PR data gathered with the caller's read-only token. The kernel-internal token is # minted after the model has finished, scoped to this repo and to pull-request writes. +# The model provider's credentials come from workload identity federation: the job's GitHub +# OIDC token is exchanged for short-lived provider credentials, so callers pass no API keys +# but must grant `id-token: write`. on: workflow_call: @@ -39,7 +42,8 @@ on: required: true ADMIN_APP_PRIVATE_KEY: required: true - # Pass the key for the model's provider. Both are optional so either provider works. + # Ignored. Declared so callers that still pass them keep working; remove them from the + # callers, then from here. ANTHROPIC_API_KEY: required: false OPENAI_API_KEY: @@ -53,6 +57,8 @@ jobs: permissions: contents: read pull-requests: read + # For the llm-credentials action to exchange the job's OIDC token for model credentials. + id-token: write concurrency: group: pr-risk-${{ github.repository }}-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -91,17 +97,40 @@ jobs: npm install -g --ignore-scripts @earendil-works/pi-coding-agent pi --version + # Checked out rather than referenced as kernel/security-workflows/...@, because `uses` + # can't take an expression and the action must follow security-workflows-ref like the + # classifier script does. + - name: Fetch credentials action + id: credentials-action + if: steps.gather.outputs.eligible == 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: kernel/security-workflows + ref: ${{ inputs.security-workflows-ref }} + path: .security-workflows + sparse-checkout: .github/actions/llm-credentials + persist-credentials: false + + - name: Model credentials + if: steps.gather.outputs.eligible == 'true' + uses: ./.security-workflows/.github/actions/llm-credentials + with: + model: ${{ inputs.model }} + anthropic-federation-rule-id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }} + anthropic-organization-id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }} + anthropic-service-account-id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }} + openai-identity-provider-id: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }} + openai-service-account-id: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }} + openai-audience: ${{ vars.OPENAI_WIF_AUDIENCE }} + - name: Classify if: steps.gather.outputs.eligible == 'true' env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} MODEL: ${{ inputs.model }} PI_SKIP_VERSION_CHECK: '1' PI_TELEMETRY: '0' run: | set -euo pipefail - : "${MODEL:?Set the PI_PREFERRED_MODEL organization variable or the model input}" # An empty directory, and no context files or project settings, so nothing but the # prompt and the PR data reaches the model. mkdir -p "$WORKDIR/cwd" && cd "$WORKDIR/cwd" @@ -111,6 +140,14 @@ jobs: < "$WORKDIR/input.md" > "$WORKDIR/pi-events.jsonl" python3 /tmp/pr_risk.py verdict --dir "$WORKDIR" + # Before the app's private key reaches a third-party action, so no model credential is on + # disk or still being refreshed from here on. Also prints the refresh log. + - name: Stop model credentials + if: ${{ always() && steps.credentials-action.outcome == 'success' }} + uses: ./.security-workflows/.github/actions/llm-credentials + with: + stop: true + - name: Generate app token id: app-token if: ${{ !cancelled() && (steps.gather.outputs.eligible == 'true' || steps.gather.outputs.commented == 'true') }} diff --git a/README.md b/README.md index 1e5831e..c4dd936 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ on: permissions: contents: read pull-requests: read + id-token: write jobs: classify: uses: kernel/security-workflows/.github/workflows/pr-risk.yml@main @@ -95,12 +96,51 @@ jobs: secrets: ADMIN_APP_ID: ${{ secrets.ADMIN_APP_ID }} ADMIN_APP_PRIVATE_KEY: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ``` The model defaults to the `PI_PREFERRED_MODEL` variable and can be overridden with the `model` -input. Pass the API key for that model's provider; both keys are optional secrets. +input. It must be an `anthropic/` or `openai/` model. There is no API key to pass: the +[`llm-credentials`](#llm-credentials-action) action exchanges the job's GitHub OIDC token for +short-lived provider credentials, which is why the caller grants `id-token: write`. + +## Actions + +### `llm-credentials` action + +Gives pi short-lived Anthropic or OpenAI credentials through workload identity federation, so a +job needs no stored API key. It refreshes them every 30 seconds in the background, so long agent +runs keep working past the five-minute life of a GitHub OIDC token. Run it again with `stop: true` +once the model is done: that stops the refresh, deletes the credentials, and prints the refresh +log, which is where a mid-run refresh failure shows up. + +```yaml +permissions: + contents: read + id-token: write +steps: + - uses: kernel/security-workflows/.github/actions/llm-credentials@main + with: + model: ${{ vars.PI_PREFERRED_MODEL }} + anthropic-federation-rule-id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }} + anthropic-organization-id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }} + anthropic-service-account-id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }} + openai-identity-provider-id: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }} + openai-service-account-id: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }} + openai-audience: ${{ vars.OPENAI_WIF_AUDIENCE }} + - env: + PI_PREFERRED_MODEL: ${{ vars.PI_PREFERRED_MODEL }} + run: pi --print --model "$PI_PREFERRED_MODEL" ... # no ANTHROPIC_API_KEY or OPENAI_API_KEY + - if: always() + uses: kernel/security-workflows/.github/actions/llm-credentials@main + with: + stop: true +``` + +The federation IDs come from the org variables listed under [Required variables](#required-variables). +An action can't read `vars` itself, so the caller passes them; only the model's provider's IDs are +used. +Don't also set `ANTHROPIC_API_KEY` or `OPENAI_API_KEY` in the pi step: a key takes precedence over +federation. For OpenAI the action sets only the `openai` entry of pi's `auth.json`. ## Enrollment @@ -135,7 +175,7 @@ projectIgnorePaths: Consumer repos need these secrets (set at org or repo level): -- `ANTHROPIC_API_KEY` — for the `semgrep.yml` triage agent (Pi) and the `pr-risk.yml` classifier (Claude Code) +- `ANTHROPIC_API_KEY` — for the `semgrep.yml` triage agent (Pi) - `OPENAI_API_KEY` — for the `semgrep.yml` triage agent (Pi), when `PI_PREFERRED_MODEL` is an OpenAI model - `CURSOR_API_KEY` — for the other fix/remediation agents (Cursor) - `ADMIN_APP_ID` + `ADMIN_APP_PRIVATE_KEY` — GitHub App for write access @@ -145,6 +185,8 @@ Consumer repos need these secrets (set at org or repo level): Consumer repos need these variables (set at org or repo level): -- `PI_PREFERRED_MODEL` — model for the `semgrep.yml` triage agent (Pi) -- `CLAUDE_CODE_PREFERRED_MODEL` — model for the `pr-risk.yml` classifier (Claude Code) +- `PI_PREFERRED_MODEL` — model for the `semgrep.yml` triage agent and the `pr-risk.yml` classifier (Pi) +- `ANTHROPIC_FEDERATION_RULE_ID`, `ANTHROPIC_ORGANIZATION_ID`, `ANTHROPIC_SERVICE_ACCOUNT_ID`, + `OPENAI_IDENTITY_PROVIDER_ID`, `OPENAI_SERVICE_ACCOUNT_ID`, `OPENAI_WIF_AUDIENCE` — workload + identity federation for the `llm-credentials` action, set at the org level - `CURSOR_PREFERRED_MODEL` — model for the other Cursor agent invocations diff --git a/scripts/test_llm_credentials.py b/scripts/test_llm_credentials.py new file mode 100644 index 0000000..d73ba64 --- /dev/null +++ b/scripts/test_llm_credentials.py @@ -0,0 +1,227 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + +import yaml + +ACTION = Path(__file__).resolve().parents[1] / ".github/actions/llm-credentials" +SCRIPT = ACTION / "llm_credentials.sh" + +# Stands in for curl and logs each request. GitHub's OIDC endpoint returns a numbered token naming +# its audience; OpenAI's token endpoint returns $FAKE_OPENAI_RESPONSE, or by default an access +# token naming the GitHub token it was given. +FAKE_CURL = r'''#!/usr/bin/env python3 +import json, os, sys, urllib.parse +from pathlib import Path +args = sys.argv[1:] +url = next(a for a in args if a.startswith("http")) +headers = [args[i + 1] for i, a in enumerate(args) if a == "-H"] +body = sys.stdin.read() if "@-" in args else None +with open(os.environ["FAKE_CURL_LOG"], "a") as log: + log.write(json.dumps({"url": url, "headers": headers, "body": body}) + "\n") +if os.environ.get("FAKE_CURL_FAIL"): + sys.exit(22) +if url.startswith("https://auth.openai.com/"): + default = json.dumps({"access_token": "access-for-" + json.loads(body)["subject_token"]}) + print(os.environ.get("FAKE_OPENAI_RESPONSE", default)) +else: + counter = Path(os.environ["FAKE_CURL_LOG"] + ".n") + n = int(counter.read_text()) + 1 if counter.exists() else 1 + counter.write_text(str(n)) + audience = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["audience"][0] + print(json.dumps({"value": f"jwt{n}-for-{audience}"})) +''' + + +class LlmCredentialsTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + (self.root / "bin").mkdir() + (self.root / "bin/curl").write_text(FAKE_CURL) + (self.root / "bin/curl").chmod(0o755) + self.token_file = self.root / "anthropic-token" + self.auth = self.root / "pi/auth.json" + self.env = { + "PATH": f"{self.root / 'bin'}:{os.environ['PATH']}", + "RUNNER_TEMP": str(self.root), + "FAKE_CURL_LOG": str(self.root / "curl.log"), + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://github.invalid/token?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "ANTHROPIC_IDENTITY_TOKEN_FILE": str(self.token_file), + "OPENAI_IDENTITY_PROVIDER_ID": "idp_test", + "OPENAI_SERVICE_ACCOUNT_ID": "user-test", + "OPENAI_WIF_AUDIENCE": "https://api.openai.com/v1", + "PI_CODING_AGENT_DIR": str(self.root / "pi"), + } + + def run_script(self, *args, **env): + return subprocess.run(["bash", str(SCRIPT), *args], env={**self.env, **env}, + capture_output=True, text=True) + + def requests(self): + log = self.root / "curl.log" + return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + + def test_anthropic_writes_identity_token(self): + self.assertEqual(self.run_script("refresh", "anthropic").returncode, 0) + self.assertEqual(self.token_file.read_text(), "jwt1-for-https://api.anthropic.com") + [oidc] = self.requests() + self.assertEqual(oidc["url"], "https://github.invalid/token?api-version=2.0&audience=https%3A%2F%2Fapi.anthropic.com") + self.assertEqual(oidc["headers"], ["Authorization: Bearer request-token"]) + + def test_openai_exchanges_the_github_token(self): + self.assertEqual(self.run_script("refresh", "openai").returncode, 0) + oidc, exchange = self.requests() + self.assertTrue(oidc["url"].endswith("&audience=https%3A%2F%2Fapi.openai.com%2Fv1")) + self.assertEqual(oidc["headers"], ["Authorization: Bearer request-token"]) + self.assertEqual(exchange["url"], "https://auth.openai.com/oauth/token") + self.assertEqual(exchange["headers"], ["Content-Type: application/json"]) + self.assertEqual(json.loads(exchange["body"]), { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "subject_token": "jwt1-for-https://api.openai.com/v1", + "identity_provider_id": "idp_test", + "service_account_id": "user-test", + }) + self.assertEqual(json.loads(self.auth.read_text()), + {"openai": {"type": "api_key", "key": "access-for-jwt1-for-https://api.openai.com/v1"}}) + self.assertEqual(self.auth.stat().st_mode & 0o777, 0o600) + + def test_openai_keeps_other_auth_entries(self): + self.auth.parent.mkdir() + self.auth.write_text(json.dumps({"anthropic": {"type": "api_key", "key": "other"}, + "openai": {"type": "api_key", "key": "old"}})) + self.assertEqual(self.run_script("refresh", "openai").returncode, 0) + auth = json.loads(self.auth.read_text()) + self.assertEqual(auth["anthropic"], {"type": "api_key", "key": "other"}) + self.assertEqual(auth["openai"]["key"], "access-for-jwt1-for-https://api.openai.com/v1") + + def test_failed_refresh_keeps_working_credentials(self): + self.run_script("refresh", "openai") + self.run_script("refresh", "anthropic") + before = {p: p.read_text() for p in (self.auth, self.token_file)} + for provider in ("openai", "anthropic"): + self.assertNotEqual(self.run_script("refresh", provider, FAKE_CURL_FAIL="1").returncode, 0) + # The GitHub token is issued but OpenAI's response has no access token. + for response in ("{}", '{"error": "invalid_grant"}'): + self.assertNotEqual(self.run_script("refresh", "openai", FAKE_OPENAI_RESPONSE=response).returncode, 0) + self.assertEqual({p: p.read_text() for p in before}, before) + + def test_start_refreshes_until_stopped(self): + self.addCleanup(self.run_script, "stop") + result = self.run_script("start", "anthropic", LLM_CREDENTIALS_INTERVAL="1") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.token_file.read_text(), "jwt1-for-https://api.anthropic.com") + group = int((self.root / "llm-credentials.pid").read_text()) + deadline = time.time() + 10 + while self.token_file.read_text().startswith("jwt1-") and time.time() < deadline: + time.sleep(0.2) + self.assertNotEqual(self.token_file.read_text(), "jwt1-for-https://api.anthropic.com") + + stopped = self.run_script("stop") + self.assertEqual(stopped.returncode, 0, stopped.stderr) + with self.assertRaises(ProcessLookupError): + os.killpg(group, 0) + self.assertFalse(self.token_file.exists()) + self.assertFalse((self.root / "llm-credentials.pid").exists()) + + def test_second_start_replaces_the_first_loop(self): + self.addCleanup(self.run_script, "stop") + self.assertEqual(self.run_script("start", "anthropic", LLM_CREDENTIALS_INTERVAL="1").returncode, 0) + first = int((self.root / "llm-credentials.pid").read_text()) + self.assertEqual(self.run_script("start", "anthropic", LLM_CREDENTIALS_INTERVAL="1").returncode, 0) + with self.assertRaises(ProcessLookupError): + os.killpg(first, 0) + self.assertEqual(self.run_script("stop").returncode, 0) + time.sleep(1.5) + self.assertFalse(self.token_file.exists()) + + def test_invalid_auth_json_is_an_error(self): + self.auth.parent.mkdir() + self.auth.write_text("not json") + self.assertNotEqual(self.run_script("refresh", "openai").returncode, 0) + stopped = self.run_script("stop") + self.assertNotEqual(stopped.returncode, 0) + self.assertIn("not valid JSON", stopped.stderr) + self.assertEqual(self.auth.read_text(), "not json") + + def test_empty_auth_json_gets_the_openai_entry(self): + self.auth.parent.mkdir() + self.auth.write_text("") + self.assertEqual(self.run_script("refresh", "openai").returncode, 0) + self.assertIn("openai", json.loads(self.auth.read_text())) + + def test_stop_removes_openai_entry_and_prints_log(self): + self.auth.parent.mkdir() + self.auth.write_text(json.dumps({"anthropic": {"type": "api_key", "key": "other"}, + "openai": {"type": "api_key", "key": "access"}})) + (self.root / "llm-credentials.log").write_text("2026-11-09T00:00:00Z openai credential refresh failed\n") + stopped = self.run_script("stop") + self.assertEqual(stopped.returncode, 0, stopped.stderr) + self.assertEqual(json.loads(self.auth.read_text()), {"anthropic": {"type": "api_key", "key": "other"}}) + self.assertIn("openai credential refresh failed", stopped.stdout) + + def test_stop_without_start_succeeds(self): + self.assertEqual(self.run_script("stop").returncode, 0) + + def run_action(self, model, **overrides): + step = yaml.safe_load((ACTION / "action.yml").read_text())["runs"]["steps"][0] + fake = self.root / "fake-action" + fake.mkdir(exist_ok=True) + (fake / "llm_credentials.sh").write_text('echo "$@" > "$FAKE_CREDENTIALS_LOG"\n') + env = {name: "id-" + name for name in step["env"]} + env.update(PATH=os.environ["PATH"], MODEL=model, STOP="false", GITHUB_ACTION_PATH=str(fake), + GITHUB_ENV=str(self.root / "github-env"), FAKE_CREDENTIALS_LOG=str(self.root / "calls")) + env.update(overrides) + for name in ("calls", "github-env"): + (self.root / name).unlink(missing_ok=True) + result = subprocess.run(["bash", "-c", step["run"]], env=env, capture_output=True, text=True) + read = lambda name: (self.root / name).read_text() if (self.root / name).exists() else "" + return result, read("calls"), read("github-env") + + def test_action_starts_the_model_provider(self): + _, calls, github_env = self.run_action("anthropic/claude-x") + self.assertEqual(calls, "start anthropic\n") + # Later steps run pi, whose Anthropic SDK needs the federation settings. + self.assertEqual(github_env.splitlines(), [ + f"{name}=id-{name}" for name in ("ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_SERVICE_ACCOUNT_ID", "ANTHROPIC_IDENTITY_TOKEN_FILE")]) + + _, calls, github_env = self.run_action("openai/gpt-x") + self.assertEqual(calls, "start openai\n") + self.assertEqual(github_env, "") + + def test_action_stops(self): + result, calls, _ = self.run_action("", STOP="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(calls, "stop\n") + + def test_action_requires_a_model(self): + result, calls, _ = self.run_action("") + self.assertNotEqual(result.returncode, 0) + self.assertIn("PI_PREFERRED_MODEL", result.stderr) + self.assertEqual(calls, "") + + def test_action_requires_the_providers_ids(self): + for model, name, hint in (("anthropic/claude-x", "ANTHROPIC_FEDERATION_RULE_ID", "anthropic-federation-rule-id"), + ("openai/gpt-x", "OPENAI_WIF_AUDIENCE", "openai-audience")): + result, calls, _ = self.run_action(model, **{name: ""}) + self.assertNotEqual(result.returncode, 0) + self.assertIn(hint, result.stderr) + self.assertIn(f"{name} org variable", result.stderr) + self.assertEqual(calls, "") + + def test_unknown_provider_fails(self): + result = self.run_script("start", "google") + self.assertNotEqual(result.returncode, 0) + self.assertIn("google", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_pr_risk.py b/scripts/test_pr_risk.py index 5147715..d28c454 100644 --- a/scripts/test_pr_risk.py +++ b/scripts/test_pr_risk.py @@ -357,11 +357,26 @@ def test_verdict_from_events_feeds_comment(self): def test_workflow_isolates_the_model(self): workflow = yaml.safe_load(WORKFLOW.read_text()) job = workflow["jobs"]["classify"] - self.assertEqual(job["permissions"], {"contents": "read", "pull-requests": "read"}) + self.assertEqual(job["permissions"], {"contents": "read", "pull-requests": "read", "id-token": "write"}) + self.assertNotIn("secrets.", json.dumps(job["env"])) steps = {s["name"]: s for s in job["steps"]} classify = steps["Classify"] - self.assertEqual(set(classify["env"]), {"ANTHROPIC_API_KEY", "OPENAI_API_KEY", "MODEL", - "PI_SKIP_VERSION_CHECK", "PI_TELEMETRY"}) + self.assertEqual(set(classify["env"]), {"MODEL", "PI_SKIP_VERSION_CHECK", "PI_TELEMETRY"}) + action = "./.security-workflows/.github/actions/llm-credentials" + fetch = steps["Fetch credentials action"] + self.assertEqual(fetch["with"]["ref"], "${{ inputs.security-workflows-ref }}") + self.assertEqual(fetch["with"]["path"], ".security-workflows") + self.assertFalse(fetch["with"]["persist-credentials"]) + credentials = steps["Model credentials"] + self.assertEqual(credentials["uses"], action) + self.assertEqual(credentials["with"]["model"], "${{ inputs.model }}") + self.assertNotIn("secrets.", json.dumps(credentials["with"])) + self.assertEqual(credentials["if"], classify["if"]) + self.assertEqual(fetch["if"], classify["if"]) + stop = steps["Stop model credentials"] + self.assertEqual(stop["uses"], action) + self.assertEqual(stop["with"], {"stop": True}) + self.assertIn("always()", stop["if"]) self.assertEqual(classify["env"]["MODEL"], "${{ inputs.model }}") for flag in ("--no-tools", "--no-mcp", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files", "--no-approve", "--no-session"): @@ -369,7 +384,9 @@ def test_workflow_isolates_the_model(self): self.assertNotIn("--approve ", classify["run"]) self.assertNotIn("GH_TOKEN", json.dumps({k: steps[k] for k in ("Install Pi", "Classify")})) names = list(steps) - self.assertLess(names.index("Classify"), names.index("Generate app token")) + order = ["Fetch credentials action", "Model credentials", "Classify", "Stop model credentials", + "Generate app token"] + self.assertEqual(order, sorted(order, key=names.index)) self.assertEqual(steps["Generate app token"]["with"]["permission-pull-requests"], "write") self.assertEqual(steps["Gather"]["env"]["GH_TOKEN"], "${{ github.token }}") triggers = workflow[True]["workflow_call"] @@ -395,15 +412,11 @@ def test_classify_step_runs_pi(self): (root / "bin/pi").chmod(0o755) (root / "events.jsonl").write_text(pi_events(json.dumps(VERDICT))) env = {"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "WORKDIR": str(work), - "MODEL": "openai/some-model", "OPENAI_API_KEY": "sk-test", + "MODEL": "openai/some-model", "FAKE_PI_LOG": str(root / "pi.json"), "FAKE_PI_EVENTS": str(root / "events.jsonl")} subprocess.run(["bash", "-c", script], env=env, check=True) call = json.loads((root / "pi.json").read_text()) result = json.loads((work / "result.json").read_text()) - - empty = subprocess.run(["bash", "-c", script], env={**env, "MODEL": ""}, capture_output=True, text=True) - self.assertNotEqual(empty.returncode, 0) - self.assertIn("PI_PREFERRED_MODEL", empty.stderr) self.assertEqual(result, {"is_error": False, "result": json.dumps(VERDICT), "structured_output": VERDICT}) self.assertEqual(call["stdin"], "# Pull request\n") self.assertEqual(call["cwd"], str(work / "cwd"))