From 42221d2ca9c5c367eee4b9fb41b374f0a2fccb4b Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Fri, 23 Oct 2026 19:22:55 +0800 Subject: [PATCH 1/4] Store one API key per judgment backend and let /typesafe login take a backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every backend's key came from its own environment variable, and only TypeSafe could be stored, because the store held one TypeSafe key. Now the store keeps one key per backend — a registry name, or an endpoint's keyEnv — and keySituation, authState, and ensureApiKey read it for every backend. The environment still wins. /typesafe login [backend] prompts with that backend's name, proves the key before saving, and stores it under that backend. A backend whose model list checks the key is proved with the list; a public list accepts any key, so SubQ, OpenRouter, and Command Code are proved with one judge request instead. /typesafe logout [backend] removes one key and leaves the others. The store reads the old {apiKey} shape as the TypeSafe key and rewrites it on the next save. A key stored for one backend is never read for another. --- .env.example | 6 ++-- README.md | 8 ++--- docs/api.md | 4 +-- src/auth.ts | 9 ++--- src/backends.ts | 8 ----- src/client.ts | 4 +-- src/credentials.ts | 72 +++++++++++++++++++++++++++++-------- src/extension.ts | 60 ++++++++++++++++++------------- src/key-prompt.ts | 12 +++---- src/login.ts | 49 ++++++++++++++------------ tests/auth.test.ts | 4 +-- tests/backends.test.ts | 12 +++++-- tests/client.test.ts | 2 +- tests/credentials.test.ts | 44 ++++++++++++++++++++++- tests/extension.test.ts | 74 +++++++++++++++++++++++++++++++++------ tests/login.test.ts | 19 +++++++--- 16 files changed, 276 insertions(+), 111 deletions(-) diff --git a/.env.example b/.env.example index 9d6ea56..1301643 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,9 @@ TYPESAFE_API_KEY= # Optional, explicit consent for automated/headless Pi tool calls: # PI_TYPESAFE_ENABLED=1 # Optional, route judgments through another backend that serves the same protocol. -# One of: typesafe (default), openrouter, commandcode, liquid, subq. The key then comes -# from that backend's own variable (OPENROUTER_API_KEY, COMMANDCODE_API_KEY, LIQUID_API_KEY, -# SUBQ_API_KEY), not TYPESAFE_API_KEY, and /typesafe login does not apply. +# One of: typesafe (default), openrouter, commandcode, liquid, subq. The key comes from that +# backend's own variable (OPENROUTER_API_KEY, COMMANDCODE_API_KEY, LIQUID_API_KEY, +# SUBQ_API_KEY), or from /typesafe login ; a set variable wins over a stored key. # PI_TYPESAFE_BACKEND=openrouter # Optional, override the backend's default model. Mapped to the backend's id form, so # jev-1.14 becomes typesafe/jev-1.14 on OpenRouter and jev-latest becomes ~typesafe/jev-latest. diff --git a/README.md b/README.md index 7f7863f..2d0ace4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ pi install npm:pi-typesafe Then, inside Pi: -1. `/typesafe login` and paste a key from [console.typesafe.ai](https://console.typesafe.ai). Input is hidden; the key is verified against the API and saved to `~/.pi/agent/pi-typesafe/auth.json` with owner-only permissions. +1. `/typesafe login` and paste a key from [console.typesafe.ai](https://console.typesafe.ai). Input is hidden; the key is verified against the API and saved to `~/.pi/agent/pi-typesafe/auth.json` with owner-only permissions. `/typesafe login ` does the same for OpenRouter, Command Code, Liquid AI, SubQ, or an endpoint you name, keeping each key under its own name in the same store. 2. `/typesafe test` sends one built-in sample request and shows the answers. 3. `/typesafe enable` lets the agent call the tool for this session, after you confirm the data notice. @@ -91,8 +91,8 @@ Need more than one request? `evaluateAll(request)` asks any number of questions | Command | Effect | | --- | --- | -| `/typesafe login` | Enter and verify an API key (hidden input), then store it | -| `/typesafe logout` | Delete the stored key and disable the tool | +| `/typesafe login [backend]` | Enter and verify a backend's API key (hidden input), then store it under that backend; no argument means the session's backend. Refuses while that backend's environment variable is set, because the variable wins | +| `/typesafe logout [backend]` | Delete that backend's stored key, and turn judgments off when it was the session's backend; no argument means the session's backend | | `/typesafe setup` | Check which key is in use; starts login if none | | `/typesafe status` | Opt-in state, key state, session and today's counters, cost estimate, any reached cap | | `/typesafe enable` | Confirm the data notice and allow agent tool calls this session | @@ -142,7 +142,7 @@ if (!answer.ok) return { skipped: answer.errorCode === "budget" }; // never thr Your extension owns its own user consent and budget; `/typesafe enable` applies only to this package's tool. Check `authState()` rather than your own consent flag before you report that judgments are on. -Judgments can also go through OpenRouter: `createTypeSafe({ backend: "openrouter" })` sends them to `openrouter.ai` with the key from `OPENROUTER_API_KEY`. Command Code serves the same protocol: `createTypeSafe({ backend: "commandcode" })` sends them to `api.commandcode.ai` with the key from `COMMANDCODE_API_KEY` and the model `typesafe/jev`. So does Liquid AI: `createTypeSafe({ backend: "liquid" })` sends them to `api.liquid.ai` with the key from `LIQUID_API_KEY` and the model `d1:free`, which is free; pass `model: "d1"` for the paid model, priced at $0.040 per million input tokens. Liquid counts the state once per question, where TypeSafe counts it once per request, so a request with N questions costs about N times the state on Liquid. Speed, measured on 2026-10-03 with one request at a time: `d1:free` took a median of about 14 seconds and sometimes failed with HTTP 504 after about 46 seconds, while `d1` took about 0.5 seconds. With `d1:free`, raise `timeoutMs` above its 15-second default, or pass `model: "d1"`. SubQ too: `createTypeSafe({ backend: "subq" })` sends them to `api.subq.ai` with the key from `SUBQ_API_KEY` and the model `subq-snap-preview`, priced at $0.05 per million input tokens. SubQ takes at most 16 questions in one request, a Choice of at least 2 options, and a Score of at most 10 levels: `evaluate` refuses a larger request before it sends it, `evaluateAll` splits to 16, and `client.limits` tells you the limit without a request. Or pass a `BackendEndpoint` object as `backend` to name any host that serves the protocol: it brings its own key variable, never reads `TYPESAFE_API_KEY`, and `backendHost()` reports its destination host for your consent text. These backends have no login store, so `/typesafe login` does not apply to them. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey`, or the status you report describes the TypeSafe key while the requests use another one. The `/typesafe` commands and the `typesafe_evaluate` tool follow `PI_TYPESAFE_BACKEND` (`typesafe` by default) and name that backend's host, key variable, and model in their consent and status text — so the extension sends judgments where it says it sends them. A name the registry does not know is a `configuration` error at session start, not a silent fallback. `PI_TYPESAFE_MODEL` overrides the backend's default model, written as that backend's own id is written (`jev-1.14`; it is mapped to the backend's id form, so OpenRouter sends `typesafe/jev-1.14`). `/typesafe login` and `/typesafe logout` apply only to a backend that uses the TypeSafe key store and say so on any other. Every export — the client, `ask`, batching, the usage ledger, auth state, the `PI_TYPESAFE_BACKEND` and `PI_TYPESAFE_MODEL` readers, and the `pi-typesafe/calibrate` and `pi-typesafe/ui` entry points — is in [docs/api.md](docs/api.md). +Judgments can also go through OpenRouter: `createTypeSafe({ backend: "openrouter" })` sends them to `openrouter.ai` with the key from `OPENROUTER_API_KEY`. Command Code serves the same protocol: `createTypeSafe({ backend: "commandcode" })` sends them to `api.commandcode.ai` with the key from `COMMANDCODE_API_KEY` and the model `typesafe/jev`. So does Liquid AI: `createTypeSafe({ backend: "liquid" })` sends them to `api.liquid.ai` with the key from `LIQUID_API_KEY` and the model `d1:free`, which is free; pass `model: "d1"` for the paid model, priced at $0.040 per million input tokens. Liquid counts the state once per question, where TypeSafe counts it once per request, so a request with N questions costs about N times the state on Liquid. Speed, measured on 2026-10-03 with one request at a time: `d1:free` took a median of about 14 seconds and sometimes failed with HTTP 504 after about 46 seconds, while `d1` took about 0.5 seconds. With `d1:free`, raise `timeoutMs` above its 15-second default, or pass `model: "d1"`. SubQ too: `createTypeSafe({ backend: "subq" })` sends them to `api.subq.ai` with the key from `SUBQ_API_KEY` and the model `subq-snap-preview`, priced at $0.05 per million input tokens. SubQ takes at most 16 questions in one request, a Choice of at least 2 options, and a Score of at most 10 levels: `evaluate` refuses a larger request before it sends it, `evaluateAll` splits to 16, and `client.limits` tells you the limit without a request. Or pass a `BackendEndpoint` object as `backend` to name any host that serves the protocol: it brings its own key variable, never reads `TYPESAFE_API_KEY`, and `backendHost()` reports its destination host for your consent text. Each backend's key can also be stored with `/typesafe login `: it prompts once, proves the key with the backend's model list where that list checks the key and otherwise with one judge request, and saves it under that backend's name, so a key stored for one backend is never sent to another. A backend's own environment variable still wins over its stored key. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey`, or the status you report describes the TypeSafe key while the requests use another one. The `/typesafe` commands and the `typesafe_evaluate` tool follow `PI_TYPESAFE_BACKEND` (`typesafe` by default) and name that backend's host, key variable, and model in their consent and status text — so the extension sends judgments where it says it sends them. A name the registry does not know is a `configuration` error at session start, not a silent fallback. `PI_TYPESAFE_MODEL` overrides the backend's default model, written as that backend's own id is written (`jev-1.14`; it is mapped to the backend's id form, so OpenRouter sends `typesafe/jev-1.14`). `/typesafe login [backend]` and `/typesafe logout [backend]` act on the backend they name, or on the session's own backend when they name none, and on a backend whose environment variable is set they say that the variable wins. Every export — the client, `ask`, batching, the usage ledger, auth state, the `PI_TYPESAFE_BACKEND` and `PI_TYPESAFE_MODEL` readers, and the `pi-typesafe/calibrate` and `pi-typesafe/ui` entry points — is in [docs/api.md](docs/api.md). ## Development diff --git a/docs/api.md b/docs/api.md index 7b56d85..8737297 100644 --- a/docs/api.md +++ b/docs/api.md @@ -63,7 +63,7 @@ SubQ (`subq`) serves the same protocol at `api.subq.ai` under the SDK's own `/v1 Anything that is neither a registry name nor such an object is refused with `backend must be a registry name or a backend object.` -The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend — registry or endpoint — reads only its own `keyEnv` variable: the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere, and the TypeSafe key is only ever sent to the typesafe backend. `createTypeSafe` with no `apiKey` and no key in the backend's variable fails with `No API key. Set in the environment.` before any request is built. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. +Every backend takes its key from its own environment variable first, then from the `/typesafe login` store. The store keeps one key per backend: a registry key under the backend name, an endpoint key under its `keyEnv`. A key stored for one backend is never read for another, and the TypeSafe key is only ever sent to the typesafe backend. `createTypeSafe` with no `apiKey` and no key for the backend fails with `No API key. Run /typesafe login in Pi, or set in the environment.` before any request is built. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. `evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, keys, or your submitted state, and no header value except a numeric `Retry-After` count in seconds (quoted by the 429 advice as `Retry after seconds.`). The advice is backend-aware: a 401 names the backend's own key variable (`Check TYPESAFE_API_KEY.`, `Check OPENROUTER_API_KEY.`, `Check COMMANDCODE_API_KEY.`, or `Check .` for an endpoint), and a 402 says `Check your account balance.` except on OpenRouter, which says `Insufficient credits. Add credits at https://openrouter.ai/credits.` `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`, or an endpoint that does not set `modelsVerifyKey: true`), which accepts any key and leaves the auth state unverified. @@ -156,7 +156,7 @@ console.log(formatCalibration(calibrate("action guard", samplesOf(results).sampl ## Login helpers: `pi-typesafe/ui` -`ensureApiKey(ctx, { backend })`, `loginWithPrompt(ctx)`, and `promptForApiKey(ctx)` use the same hidden input as `/typesafe login`. `ensureApiKey(ctx)` returns the existing key source, or prompts, verifies, and stores a new TypeSafe key (`undefined` when the user cancels). For any other backend it returns the environment source or throws `configuration` naming the variable to set; it never opens the prompt, because the prompt verifies against api.typesafe.ai and writes the TypeSafe store. These need Pi's TUI, so call them only from extension command handlers. +`ensureApiKey(ctx, { backend })`, `loginWithPrompt(ctx, { backend })`, and `promptForApiKey(ctx, label)` use the same hidden input as `/typesafe login`. `ensureApiKey` returns the existing key source, or prompts, proves, and stores a key for that backend (`undefined` when the user cancels). The proof is the backend's model list when that list checks the key; otherwise it is one judge request, because a public list accepts any key. `loginWithPrompt` resolves to `{ path, models? }` — `models` is absent when one request proved the key. These need Pi's TUI, so call them only from extension command handlers. ## One agent tool diff --git a/src/auth.ts b/src/auth.ts index 4f06377..876912e 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -1,6 +1,6 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, resolveBackend, usesTypesafeKey } from "./backends.js"; +import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, resolveBackend } from "./backends.js"; import type { BackendSpec } from "./backends.js"; import { credentialsPath, keySituation, keySourceLabel, piTypesafeDir } from "./credentials.js"; import type { KeySource } from "./credentials.js"; @@ -40,8 +40,9 @@ export interface AuthState { /** Short human label for the key source: `TYPESAFE_API_KEY`, `OPENROUTER_API_KEY`, `/typesafe login`, `no key`, `unusable key`. */ readonly keyName: string; /** - * True when the key in effect was accepted by the backend (login verifies a TypeSafe key; a successful request proves - * any key). The record is shared across backends: switching backends keeps the last outcome until the next request. + * True when the key in effect was accepted by the backend (a login proves the backend it was asked for; a successful + * request proves any key). The record is shared across backends: switching backends keeps the last outcome until the + * next request. */ readonly verified: boolean; readonly verifiedAt?: string; @@ -154,7 +155,7 @@ export function describeAuth(state: AuthState = authState()): AuthReport { const label = `${config.label} key`; const since = state.lastFailure ? ` Last failure: ${state.lastFailure.message}${state.lastFailure.at ? ` (${state.lastFailure.at})` : ""}` : ""; if (state.kind === "missing") { - const how = usesTypesafeKey(config) ? `a key is configured (/typesafe login or ${TYPESAFE_KEY_ENV})` : `${config.keyEnv} is set in the environment`; + const how = `a key is configured (/typesafe login${config.name && config.name !== DEFAULT_BACKEND ? ` ${config.name}` : ""} or ${config.keyEnv})`; return { level: "error", text: `${label}: missing — every Jev judgment is skipped until ${how}.${since}` }; } if (state.kind === "unusable") { diff --git a/src/backends.ts b/src/backends.ts index dea76ff..24cdfe1 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -187,14 +187,6 @@ export function backendPrice(backend: ResolvedBackend, model: string): number | return prices !== undefined && Object.hasOwn(prices, model) ? prices[model] : undefined; } -/** - * Whether a backend's key comes from the TypeSafe resolution (`TYPESAFE_API_KEY`, then the login store) or only from - * its own environment variable. Only the TypeSafe backend has a login store; every other backend is environment-only. - */ -export function usesTypesafeKey(backend: BackendConfig): boolean { - return (backend.keyEnv ?? TYPESAFE_KEY_ENV) === TYPESAFE_KEY_ENV; -} - const KEY_ENV_MESSAGE = "Backend keyEnv must name an environment variable: letters, digits, and underscores, not starting with a digit."; const HOST_MESSAGE = "Backend host must be an absolute https: URL with no user info, path, query, or fragment (http: is allowed only for localhost, 127.0.0.0/8, and [::1])."; diff --git a/src/client.ts b/src/client.ts index 0352f13..ca1737a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,7 +1,7 @@ import { TypeSafeClient } from "@typesafe-ai/sdk"; import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk"; import { recordAuthFailure, recordAuthVerified } from "./auth.js"; -import { TYPESAFE_KEY_ENV, backendPrice, resolveBackend, sentModelId, usesTypesafeKey } from "./backends.js"; +import { backendPrice, resolveBackend, sentModelId } from "./backends.js"; import type { BackendConfig, BackendSpec, RegistryBackendConfig, ResolvedBackend } from "./backends.js"; import type { BatchEvaluation, BatchOptions } from "./batch.js"; import { evaluateAll, evaluateMany } from "./batch.js"; @@ -205,7 +205,7 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { if (situation.kind === "environment" || situation.kind === "stored") apiKey = situation.key; } if (!apiKey) { - const how = usesTypesafeKey(backend) ? `Run /typesafe login in Pi, or set ${TYPESAFE_KEY_ENV}` : `Set ${backend.keyEnv}`; + const how = `Run /typesafe login${backend.name ? ` ${backend.name}` : ""} in Pi, or set ${backend.keyEnv}`; throw new TypeSafeIntegrationError("configuration", `No API key. ${how} in the environment.`); } const timeout = positiveInteger(options.timeoutMs ?? 15_000, "timeoutMs"); diff --git a/src/credentials.ts b/src/credentials.ts index d5215c2..3728297 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -1,7 +1,7 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join } from "node:path"; -import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, resolveBackend, usesTypesafeKey } from "./backends.js"; +import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, resolveBackend } from "./backends.js"; import type { BackendSpec } from "./backends.js"; import { TypeSafeIntegrationError } from "./errors.js"; @@ -41,20 +41,41 @@ export function normalizeApiKey(value: unknown): string { return key; } +/** Where a stored key belongs: the registry name, or an endpoint's `keyEnv`. Environment variables still win. */ +export function credentialId(backend: BackendSpec = DEFAULT_BACKEND): string { + const config = resolveBackend(backend); + return config.name ?? config.keyEnv; +} + +/** The stored keys, by credential id. The pre-0.12 shape, a lone `apiKey`, reads as the TypeSafe key. */ +function readStoredKeys(): Record { + const path = credentialsPath(); + const raw: unknown = JSON.parse(readFileSync(path, "utf8")); + const keys: Record = {}; + if (raw && typeof raw === "object") { + const record = raw as { apiKey?: unknown; keys?: unknown }; + if (record.keys && typeof record.keys === "object" && !Array.isArray(record.keys)) { + for (const [id, value] of Object.entries(record.keys as Record)) { + if (typeof value === "string" && value.trim()) keys[id] = value.trim(); + } + } + if (typeof record.apiKey === "string" && record.apiKey.trim()) keys[DEFAULT_BACKEND] = record.apiKey.trim(); + } + return keys; +} + /** - * The stored key, or `undefined` when the file is missing, unreadable, or holds no usable value. Throws + * The stored key for one backend, or `undefined` when the file is missing, unreadable, or holds no usable value. Throws * `configuration` when the file is readable by other users; keySituation() reports that case as `unusable` instead. */ -export function readStoredApiKey(): string | undefined { +export function readStoredApiKey(backend: BackendSpec = DEFAULT_BACKEND): string | undefined { const path = credentialsPath(); try { if (process.platform !== "win32" && (statSync(path).mode & 0o077) !== 0) { // Refuse to use a key other local users can read; the user must fix permissions or log in again. throw new TypeSafeIntegrationError("configuration", `Refusing to read ${path}: it is readable by other users. Run chmod 600 on it, or run /typesafe logout and /typesafe login.`); } - const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); - const key = parsed && typeof parsed === "object" ? (parsed as { apiKey?: unknown }).apiKey : undefined; - return typeof key === "string" && key.trim() ? key.trim() : undefined; + return readStoredKeys()[credentialId(backend)]; } catch (error) { if (error instanceof TypeSafeIntegrationError) throw error; return undefined; @@ -66,18 +87,16 @@ export function readStoredApiKey(): string | undefined { * backend never throws: the "unusable" kind carries the user-facing reason (a stored key that other local users can * read). An invalid backend — an unknown name or an endpoint object that fails validation — throws the same * `configuration` error as resolveBackend(), so validate a user-supplied endpoint with resolveBackend() first. - * The TypeSafe backend reads `TYPESAFE_API_KEY`, then the login store. Every other backend reads only its own - * environment variable, because the store holds a TypeSafe key and a login verifies against api.typesafe.ai. + * One rule for every backend: its own environment variable, then the key stored for it, then missing. */ export function keySituation(backend: BackendSpec = DEFAULT_BACKEND): KeySituation { const config = resolveBackend(backend); const keyEnv = config.keyEnv; const fromEnvironment = process.env[keyEnv]?.trim(); if (fromEnvironment) return { kind: "environment", key: fromEnvironment, keyEnv }; - if (!usesTypesafeKey(config)) return { kind: "missing" }; const path = credentialsPath(); try { - const key = readStoredApiKey(); + const key = readStoredApiKey(backend); return key ? { kind: "stored", key, path } : { kind: "missing" }; } catch (error) { // readStoredApiKey only throws for a store this process must not use; report it instead of failing the caller. @@ -109,28 +128,51 @@ export function resolveApiKey(backend: BackendSpec = DEFAULT_BACKEND): { key: st return situation.kind === "environment" || situation.kind === "stored" ? { key: situation.key, source: situation.kind } : undefined; } -export function storeApiKey(value: unknown): string { +export function storeApiKey(value: unknown, backend: BackendSpec = DEFAULT_BACKEND): string { const key = normalizeApiKey(value); const path = credentialsPath(); + const id = credentialId(backend); + // A store that cannot be read (wrong permissions, bad JSON) is replaced rather than blocking the login. + let keys: Record = {}; + try { keys = readStoredKeys(); } catch { keys = {}; } + keys[id] = key; try { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + // Written under a temporary name and renamed, so a crash mid-write cannot truncate the other backends' keys. const temporary = `${path}.${process.pid}.tmp`; - writeFileSync(temporary, `${JSON.stringify({ apiKey: key }, null, 2)}\n`, { mode: 0o600, flag: "w" }); + writeFileSync(temporary, `${JSON.stringify({ version: 1, keys }, null, 2)}\n`, { mode: 0o600, flag: "w" }); chmodSync(temporary, 0o600); renameSync(temporary, path); } catch { - throw new TypeSafeIntegrationError("configuration", `Could not write ${path}. Check directory permissions, or set TYPESAFE_API_KEY in the environment instead.`); + throw new TypeSafeIntegrationError("configuration", `Could not write ${path}. Check directory permissions, or set ${resolveBackend(backend).keyEnv} in the environment instead.`); } return path; } -export function clearStoredApiKey(): boolean { +/** Remove one backend's stored key, or with no argument every stored key. Returns true when a key was removed. */ +export function clearStoredApiKey(backend?: BackendSpec): boolean { const path = credentialsPath(); try { statSync(path); } catch { return false; } - rmSync(path, { force: true }); + if (backend === undefined) { + rmSync(path, { force: true }); + return true; + } + let keys: Record; + try { keys = readStoredKeys(); } catch { rmSync(path, { force: true }); return true; } + const id = credentialId(backend); + if (!(id in keys)) return false; + delete keys[id]; + if (!Object.keys(keys).length) rmSync(path, { force: true }); + else { + // Same temporary-name-then-rename write as storeApiKey: the remaining backends' keys survive a crash. + const temporary = `${path}.${process.pid}.tmp`; + writeFileSync(temporary, `${JSON.stringify({ version: 1, keys }, null, 2)}\n`, { mode: 0o600, flag: "w" }); + chmodSync(temporary, 0o600); + renameSync(temporary, path); + } return true; } diff --git a/src/extension.ts b/src/extension.ts index a38be64..a9b3566 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2,8 +2,8 @@ import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@earendil- import type { Static } from "typebox"; import { Text } from "@earendil-works/pi-tui"; import type { Questions } from "@typesafe-ai/sdk"; -import { BACKEND_ENV, DEFAULT_BACKEND, MODEL_ENV, backendFromEnvironment, backendHost, modelFromEnvironment, resolveBackend, sentModelId } from "./backends.js"; -import type { TypeSafeBackend } from "./backends.js"; +import { BACKEND_ENV, DEFAULT_BACKEND, DECISIONS_BACKENDS, MODEL_ENV, backendFromEnvironment, backendHost, modelFromEnvironment, resolveBackend, sentModelId } from "./backends.js"; +import type { BackendSpec, TypeSafeBackend } from "./backends.js"; import { createTypeSafe, DEFAULT_MAX_REQUESTS } from "./client.js"; import type { Evaluation, TypeSafe } from "./client.js"; import { authState, clearAuthState, describeAuth } from "./auth.js"; @@ -72,7 +72,6 @@ export default function typesafeExtension(pi: ExtensionAPI): void { backend = DEFAULT_BACKEND; backendError = safeError(error).message; } - const typesafeKey = backend === DEFAULT_BACKEND; const consent = consentFor(backend, model); let client: TypeSafe | undefined; // One callout per distinct degradation per session: a long run must not bury the reason in repeated notices. @@ -152,11 +151,19 @@ export default function typesafeExtension(pi: ExtensionAPI): void { pi.registerCommand("typesafe", { description: "TypeSafe login, consent, usage, sample test, and JSON playground", getArgumentCompletions(prefix) { + const [verb, ...rest] = prefix.split(/\s+/); + if (rest.length && (verb === "login" || verb === "logout")) { + const typed = rest.join(" "); + const names = (Object.keys(DECISIONS_BACKENDS) as TypeSafeBackend[]).filter(name => name.startsWith(typed)); + return names.length ? names.map(name => ({ value: `${verb} ${name}`, label: name })) : null; + } const matches = actions.filter(action => action.startsWith(prefix)).map(action => ({ value: action, label: action })); return matches.length ? matches : null; }, async handler(args, ctx) { - const action = args.trim() || "status"; + const [action = "status", ...rest] = args.trim().split(/\s+/).filter(Boolean); + const backendArg = rest[0]; + const backendChoices = Object.keys(DECISIONS_BACKENDS).join(", "); const report = (text: string, level: "info" | "warning" | "error" = "info") => { if (ctx.hasUI) ctx.ui.notify(text, level); else pi.sendMessage({ customType: "typesafe-status", content: text, display: true }); @@ -168,6 +175,12 @@ export default function typesafeExtension(pi: ExtensionAPI): void { return; } try { + // A backend the command names, or this session's own backend when it names none, so the commands act on + // what judgments actually use. resolveBackend refuses a name the registry does not know. + const spec: TypeSafeBackend = backendArg === undefined ? backend : backendArg as TypeSafeBackend; + const config = resolveBackend(spec); + const isActive = config.keyEnv === resolveBackend(backend).keyEnv; + const situation = keySituation(backend); if (action === "status") { const spend = client?.getSpend(); const auth = describeAuth(authState({ backend })); @@ -201,16 +214,16 @@ export default function typesafeExtension(pi: ExtensionAPI): void { return; } if (action === "logout") { - // The store holds a TypeSafe key, so it is not this session's credential on another backend. - if (!typesafeKey) { - report(`${resolveBackend(backend).label} is the active backend (${BACKEND_ENV}=${backend}), and it reads its key from ${resolveBackend(backend).keyEnv}, not the /typesafe login store. Unset ${BACKEND_ENV} to manage the TypeSafe key.`, "warning"); - return; - } - const removed = clearStoredApiKey(); + const removed = clearStoredApiKey(spec); clearAuthState(); client = undefined; - enabled = false; - report(removed ? `Removed the stored key at ${credentialsPath()}. TypeSafe is disabled.` : "No stored key to remove." + (process.env.TYPESAFE_API_KEY?.trim() ? " TYPESAFE_API_KEY is still set in the environment." : "")); + const env = process.env[config.keyEnv]?.trim(); + // A session with consent but no key can only fail the next call; logging out another backend's key changes nothing. + if (isActive && !env) enabled = false; + const tail = env + ? ` ${config.keyEnv} is still set in the environment and takes precedence over a stored key.` + : (isActive ? " Judgments are off for this session." : ""); + report(`${removed ? `Removed the stored ${config.label} key at ${credentialsPath()}.` : `No stored ${config.label} key to remove.`}${tail}`); return; } if (action === "disable") { @@ -219,28 +232,27 @@ export default function typesafeExtension(pi: ExtensionAPI): void { return; } if (!actions.includes(action)) { - report(`Usage: /typesafe ${actions.join(" | ")}`, "warning"); + report(`Usage: /typesafe ${actions.join(" | ")}. login and logout take a backend name: ${backendChoices}.`, "warning"); return; } if (!ctx.hasUI) { - report("This command needs interactive Pi. For headless tool use, explicitly set PI_TYPESAFE_ENABLED=1 and TYPESAFE_API_KEY before launching Pi.", "warning"); - return; - } - const situation = keySituation(backend); - // `/typesafe login` verifies a key against api.typesafe.ai and stores it for the TypeSafe backend only. - if ((action === "login" || (action === "setup" && situation.kind === "missing")) && !typesafeKey) { - report(`${resolveBackend(backend).label} is the active backend (${BACKEND_ENV}=${backend}): it reads its key from ${resolveBackend(backend).keyEnv} and never from the /typesafe login store, so login does not apply. Unset ${BACKEND_ENV} to use TypeSafe directly.`, "warning"); + report(`This command needs interactive Pi. For headless tool use, explicitly set PI_TYPESAFE_ENABLED=1 and ${config.keyEnv} before launching Pi.`, "warning"); return; } if (action === "login" || (action === "setup" && situation.kind === "missing")) { - if (process.env.TYPESAFE_API_KEY?.trim()) { - report("TYPESAFE_API_KEY is set in the environment and takes precedence over a stored key. Unset it before using /typesafe login.", "warning"); + // The one rule for every backend: the stored key is used unless the backend's variable is set, and then it wins. + if (process.env[config.keyEnv]?.trim()) { + report(`${config.keyEnv} is set in the environment and takes precedence over a stored key. Unset it before using /typesafe login.`, "warning"); return; } - const login = await loginWithPrompt(ctx); + const login = await loginWithPrompt(ctx, { backend: spec }); if (login === undefined) { report("Login cancelled; nothing was saved."); return; } client = undefined; - report(`Key verified (${login.models} model${login.models === 1 ? "" : "s"} available) and saved to ${login.path} with owner-only permissions. Run /typesafe enable to allow agent tool calls.`); + const proof = login.models === undefined ? "verified with one request" : `verified (${login.models} model${login.models === 1 ? "" : "s"} available)`; + const hint = isActive + ? (enabled ? "" : " Run /typesafe enable to allow agent tool calls.") + : ` ${BACKEND_ENV} is ${backend}; set it to ${config.name} to send judgments there.`; + report(`${config.label} key ${proof} and saved to ${login.path} with owner-only permissions.${hint}`); return; } if (action === "setup") { diff --git a/src/key-prompt.ts b/src/key-prompt.ts index 17899b7..989e06a 100644 --- a/src/key-prompt.ts +++ b/src/key-prompt.ts @@ -15,10 +15,10 @@ class KeyPrompt extends Container implements Focusable { private readonly input = new SecretInput(); private isFocused = false; - constructor(theme: { fg(color: string, text: string): string; bold(text: string): string }, done: (value: string | undefined) => void) { + constructor(theme: { fg(color: string, text: string): string; bold(text: string): string }, label: string, done: (value: string | undefined) => void) { super(); - this.addChild(new Text(theme.fg("accent", theme.bold("TypeSafe API key")), 1, 0)); - this.addChild(new Text(theme.fg("muted", "Paste the key from console.typesafe.ai › API Keys. Input is hidden. Enter saves, Esc cancels."), 1, 0)); + this.addChild(new Text(theme.fg("accent", theme.bold(`${label} API key`)), 1, 0)); + this.addChild(new Text(theme.fg("muted", "Paste the key. Input is hidden. Enter saves, Esc cancels."), 1, 0)); this.addChild(this.input); this.input.onSubmit = value => done(value); this.input.onEscape = () => done(undefined); @@ -43,9 +43,9 @@ class KeyPrompt extends Container implements Focusable { } /** Hidden input in the TUI; falls back to Pi's plain input dialog where custom components are unavailable. */ -export async function promptForApiKey(ctx: ExtensionCommandContext): Promise { +export async function promptForApiKey(ctx: ExtensionCommandContext, label = "TypeSafe"): Promise { if (typeof ctx.ui.custom === "function") { - return ctx.ui.custom((_tui, theme, _keybindings, done) => new KeyPrompt(theme, done)); + return ctx.ui.custom((_tui, theme, _keybindings, done) => new KeyPrompt(theme, label, done)); } - return ctx.ui.input("TypeSafe API key (visible while typing)", "Paste the key, then press Enter"); + return ctx.ui.input(`${label} API key (visible while typing)`, "Paste the key, then press Enter"); } diff --git a/src/login.ts b/src/login.ts index 92438fb..5806c2c 100644 --- a/src/login.ts +++ b/src/login.ts @@ -1,5 +1,6 @@ +import { noul } from "@typesafe-ai/sdk"; import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent"; -import { DEFAULT_BACKEND, resolveBackend, usesTypesafeKey } from "./backends.js"; +import { DEFAULT_BACKEND, resolveBackend } from "./backends.js"; import type { BackendSpec } from "./backends.js"; import { createTypeSafe } from "./client.js"; import { keySituation, normalizeApiKey, storeApiKey } from "./credentials.js"; @@ -10,29 +11,36 @@ import { promptForApiKey } from "./key-prompt.js"; export interface LoginResult { /** Where the verified key was saved. */ path: string; - /** Number of models the key can access; proves the key was accepted by the API. */ - models: number; + /** Models the key can access, when the backend's model list proves the key. Absent when one judge request proved it. */ + models?: number; } /** - * Prompt for a TypeSafe key (hidden input), verify it against api.typesafe.ai with a model listing, and store it for - * every pi-typesafe consumer. Resolves to undefined when the user cancels. Rejects with TypeSafeIntegrationError for an - * invalid or unverifiable key. Refuses when TYPESAFE_API_KEY is set, because the environment would shadow the stored - * key. Other backends have no login: their key is an environment variable only. + * Prompt for one backend's key (hidden input), prove it before saving, and store it for every pi-typesafe consumer. + * A backend whose model list is public is proved with one judge question instead, because that list accepts any key. + * Resolves to undefined when the user cancels. Rejects with TypeSafeIntegrationError for an invalid or unverifiable + * key. Refuses when the backend's environment variable is set, because the environment would shadow the stored key. */ -export async function loginWithPrompt(ctx: ExtensionCommandContext): Promise { - if (process.env.TYPESAFE_API_KEY?.trim()) { - throw new TypeSafeIntegrationError("configuration", "TYPESAFE_API_KEY is set in the environment and takes precedence over a stored key. Unset it before logging in interactively."); +export async function loginWithPrompt(ctx: ExtensionCommandContext, options: { backend?: BackendSpec } = {}): Promise { + const spec = options.backend ?? DEFAULT_BACKEND; + const config = resolveBackend(spec); + if (process.env[config.keyEnv]?.trim()) { + throw new TypeSafeIntegrationError("configuration", `${config.keyEnv} is set in the environment and takes precedence over a stored key. Unset it before logging in interactively.`); } if (!ctx.hasUI) { - throw new TypeSafeIntegrationError("configuration", "Logging in needs an interactive session. Set TYPESAFE_API_KEY in the environment instead."); + throw new TypeSafeIntegrationError("configuration", `Logging in needs an interactive session. Set ${config.keyEnv} in the environment instead.`); } - const entered = await promptForApiKey(ctx); + const entered = await promptForApiKey(ctx, config.label); if (entered === undefined) return undefined; const key = normalizeApiKey(entered); - // Verify before saving so a bad paste fails here, not on first use. - const models = await createTypeSafe({ apiKey: key }).listModels(); - return { path: storeApiKey(key), models: models.length }; + const client = createTypeSafe({ apiKey: key, backend: spec }); + if (config.modelsVerifyKey) { + const models = await client.listModels(); + return { path: storeApiKey(key, spec), models: models.length }; + } + // A public model list accepts any key; one judge question proves this one. + await client.evaluate({ state: "login", questions: { key: noul("Is this key accepted?") } }); + return { path: storeApiKey(key, spec) }; } export type EnsureApiKeyResult = @@ -42,19 +50,14 @@ export type EnsureApiKeyResult = /** * Use the configured key for the backend if there is one; otherwise run the login prompt. `undefined` means the user * cancelled. A store that must not be read throws `configuration` with the reason instead of prompting, so a - * permissions problem stays visible; the result shape is frozen for existing callers. A backend without a login store - * (every backend except TypeSafe) throws `configuration` naming its environment variable instead of prompting, because - * the prompt would verify the key against the wrong service and store it where the TypeSafe key lives. + * permissions problem stays visible; the result shape is frozen for existing callers. Every backend has a login store, + * so the prompt proves the key against the backend it was asked for and saves it under that backend. */ export async function ensureApiKey(ctx: ExtensionCommandContext, options: { backend?: BackendSpec } = {}): Promise { const backend = options.backend ?? DEFAULT_BACKEND; const situation = keySituation(backend); if (situation.kind === "environment" || situation.kind === "stored") return { source: situation.kind }; if (situation.kind === "unusable") throw new TypeSafeIntegrationError("configuration", situation.reason); - const config = resolveBackend(backend); - if (!usesTypesafeKey(config)) { - throw new TypeSafeIntegrationError("configuration", `No ${config.label} key. Set ${config.keyEnv} in the environment; /typesafe login stores a TypeSafe key only.`); - } - const login = await loginWithPrompt(ctx); + const login = await loginWithPrompt(ctx, { backend }); return login ? { source: "stored", login } : undefined; } diff --git a/tests/auth.test.ts b/tests/auth.test.ts index 6456360..03d1a66 100644 --- a/tests/auth.test.ts +++ b/tests/auth.test.ts @@ -43,8 +43,8 @@ test("another backend reports its own key and never the TypeSafe login hint", () const report = describeAuth(missing); assert.equal(report.level, "error"); assert.ok(report.text.startsWith("OpenRouter key: missing")); - assert.ok(report.text.includes("OPENROUTER_API_KEY is set")); - assert.ok(!report.text.includes("/typesafe login")); + assert.ok(report.text.includes("OPENROUTER_API_KEY")); + assert.ok(report.text.includes("/typesafe login openrouter")); process.env.OPENROUTER_API_KEY = "sk-or-0123456789abcdef"; const present = authState({ backend: "openrouter" }); diff --git a/tests/backends.test.ts b/tests/backends.test.ts index 9ab3e34..433169f 100644 --- a/tests/backends.test.ts +++ b/tests/backends.test.ts @@ -267,7 +267,7 @@ test("an endpoint never reads TYPESAFE_API_KEY or the login store", () => { assert.throws(() => createTypeSafe({ backend: { label: "Gateway", host: "https://gw.example.com", keyEnv: "GATEWAY_JEV_KEY", defaultModel: "jev-latest" }, fetch }), (error: unknown) => { assert.ok(error instanceof TypeSafeIntegrationError); assert.equal(error.code, "configuration"); - assert.equal(error.message, "No API key. Set GATEWAY_JEV_KEY in the environment."); + assert.equal(error.message, "No API key. Run /typesafe login in Pi, or set GATEWAY_JEV_KEY in the environment."); return true; }); assert.equal(fetchCalls, 0); @@ -283,7 +283,7 @@ test("commandcode with no COMMANDCODE_API_KEY is missing and unusable", () => { assert.throws(() => createTypeSafe({ backend: "commandcode", fetch }), (error: unknown) => { assert.ok(error instanceof TypeSafeIntegrationError); assert.equal(error.code, "configuration"); - assert.equal(error.message, "No API key. Set COMMANDCODE_API_KEY in the environment."); + assert.equal(error.message, "No API key. Run /typesafe login commandcode in Pi, or set COMMANDCODE_API_KEY in the environment."); return true; }); assert.equal(fetchCalls, 0); @@ -460,3 +460,11 @@ test("sentModelId is the id a request carries, with and without a model override assert.equal(sentModelId("openrouter"), "typesafe/jev-1.13"); assert.equal(sentModelId("openrouter", "jev-1.14"), "typesafe/jev-1.14"); }); + +test("every backend has a login store, so usesTypesafeKey is gone from the exported surface", async () => { + // A key can be stored for any backend now, so the question the removed export answered has one answer. + for (const module of [await import("../src/backends.js"), await import("../src/index.js")]) { + assert.equal("usesTypesafeKey" in module, false); + assert.equal("hasLoginStore" in module, false); + } +}); diff --git a/tests/client.test.ts b/tests/client.test.ts index 25bb867..f752776 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -463,7 +463,7 @@ test("a TypeSafe key is never sent to another backend", () => { assert.ok(error instanceof TypeSafeIntegrationError); assert.equal(error.code, "configuration"); assert.match(error.message, /OPENROUTER_API_KEY/); - assert.doesNotMatch(error.message, /typesafe login/); + assert.match(error.message, /\/typesafe login openrouter/); return true; }); } finally { diff --git a/tests/credentials.test.ts b/tests/credentials.test.ts index 2750a2d..6491116 100644 --- a/tests/credentials.test.ts +++ b/tests/credentials.test.ts @@ -36,7 +36,7 @@ test("store, read, and clear with owner-only permissions", () => { assert.equal(path, credentialsPath()); assert.equal(statSync(path).mode & 0o777, 0o600); assert.equal(statSync(join(agentDir, "pi-typesafe")).mode & 0o777, 0o700); - assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { apiKey: validKey }); + assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 1, keys: { typesafe: validKey } }); assert.deepEqual(resolveApiKey(), { key: validKey, source: "stored" }); assert.equal(clearStoredApiKey(), true); assert.equal(clearStoredApiKey(), false); @@ -142,3 +142,45 @@ test("keySourceLabel names each source", () => { assert.equal(keySourceLabel({ kind: "missing" }), "no key"); assert.equal(keySourceLabel({ kind: "unusable", path: "/tmp/auth.json", reason: "r" }), "unusable key"); }); + +test("each backend keeps its own stored key, and an endpoint is keyed by its keyEnv", () => { + const savedSubq = process.env.SUBQ_API_KEY; + delete process.env.SUBQ_API_KEY; + try { + const subqKey = "sq_test_key_0123456789abcdef"; + assert.equal(storeApiKey(validKey), credentialsPath()); + assert.equal(storeApiKey(subqKey, "subq"), credentialsPath()); + assert.deepEqual(JSON.parse(readFileSync(credentialsPath(), "utf8")), { version: 1, keys: { typesafe: validKey, subq: subqKey } }); + assert.equal(readStoredApiKey(), validKey); + assert.equal(readStoredApiKey("subq"), subqKey); + assert.equal(keySituation("subq").kind, "stored"); + // A key stored for one backend is never read as another backend's key. + assert.equal(readStoredApiKey("openrouter"), undefined); + assert.equal(keySituation("openrouter").kind, "missing"); + const gateway = { label: "Gateway", host: "https://gw.example.com", keyEnv: "GATEWAY_JEV_KEY", defaultModel: "jev-latest" } as const; + const gwKey = "gw_test_key_0123456789abcdefg"; + assert.equal(storeApiKey(gwKey, gateway), credentialsPath()); + assert.equal(readStoredApiKey(gateway), gwKey); + assert.equal(readStoredApiKey("typesafe"), validKey, "the endpoint's key does not replace the TypeSafe key"); + + assert.equal(clearStoredApiKey("subq"), true); + assert.equal(readStoredApiKey("subq"), undefined); + assert.equal(readStoredApiKey(), validKey, "clearing one backend leaves the others"); + assert.equal(clearStoredApiKey("subq"), false); + assert.equal(clearStoredApiKey(), true, "no argument clears every stored key"); + assert.equal(readStoredApiKey(gateway), undefined); + } finally { + if (savedSubq === undefined) delete process.env.SUBQ_API_KEY; else process.env.SUBQ_API_KEY = savedSubq; + } +}); + +test("a pre-0.12 store with a lone apiKey reads as the TypeSafe key", () => { + const path = credentialsPath(); + mkdirSync(join(agentDir, "pi-typesafe"), { recursive: true, mode: 0o700 }); + writeFileSync(path, `${JSON.stringify({ apiKey: validKey })}\n`, { mode: 0o600 }); + assert.equal(readStoredApiKey(), validKey); + assert.equal(readStoredApiKey("subq"), undefined); + // The next write migrates the file to the per-backend shape. + storeApiKey("sq_test_key_0123456789abcdef", "subq"); + assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 1, keys: { typesafe: validKey, subq: "sq_test_key_0123456789abcdef" } }); +}); diff --git a/tests/extension.test.ts b/tests/extension.test.ts index 661472f..55c91ab 100644 --- a/tests/extension.test.ts +++ b/tests/extension.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { existsSync, statSync } from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -196,7 +196,7 @@ test("login verifies, stores with owner-only permissions, and never echoes the k customResult = "ts_live_key_0123456789abcdef"; await runCommand("login"); assert.equal(modelListCalls, 1); - assert.ok(notices.at(-1)?.includes("Key verified (1 model available)")); + assert.ok(notices.at(-1)?.includes("TypeSafe key verified (1 model available)")); assert.equal(notices.some(text => text.includes("ts_live_key")), false); assert.equal(statSync(storedPath).mode & 0o777, 0o600); await runCommand("status"); @@ -462,21 +462,75 @@ test("an inherited property name is refused, and the extension still loads its t } }); -test("login and logout refuse on a backend that does not use the TypeSafe key store", async () => { +test("on a named backend the environment variable wins over the store, and logout still removes the stored key", async () => { process.env.PI_TYPESAFE_BACKEND = "openrouter"; process.env.OPENROUTER_API_KEY = "or_fake_key_0123456789abcdef"; + storeApiKey("sk-or-0123456789abcdef", "openrouter"); try { const scoped = await reloadExtension(); - for (const action of ["login", "logout"]) { - const before = confirmations; - await Reflect.apply(scoped.command.handler, scoped.command, [action, ctx]); - const line = notices.at(-1) ?? ""; - assert.ok(line.includes("OPENROUTER_API_KEY"), `${action}: ${line}`); - assert.equal(confirmations, before, `${action} must not prompt for a key it cannot use`); - } + const run = (args: string) => Reflect.apply(scoped.command.handler, scoped.command, [args, ctx]); + // The stub throws, so reaching the network here would fail the test instead of only changing a count. + await withFetch(async () => { throw new Error("no request may be made while the variable wins"); }, async () => { + await run("login openrouter"); + assert.ok(notices.at(-1)?.includes("OPENROUTER_API_KEY is set in the environment and takes precedence"), `login: ${notices.at(-1)}`); + await run("status"); + assert.ok(notices.at(-1)?.includes("OPENROUTER_API_KEY"), `status names the winning source: ${notices.at(-1)}`); + await run("logout"); + assert.ok(notices.at(-1)?.includes("OPENROUTER_API_KEY is still set in the environment"), `logout: ${notices.at(-1)}`); + }); + assert.equal(existsSync(join(temporary, "pi-typesafe", "auth.json")), false, "logout removes the stored key the variable shadows"); } finally { delete process.env.PI_TYPESAFE_BACKEND; delete process.env.OPENROUTER_API_KEY; + clearStoredApiKey(); + } +}); + +test("login and logout act on the session's own backend and leave another backend's key in place", async () => { + const savedEnvKey = process.env.TYPESAFE_API_KEY; + delete process.env.TYPESAFE_API_KEY; + delete process.env.OPENROUTER_API_KEY; + process.env.PI_TYPESAFE_BACKEND = "openrouter"; + const storedPath = join(temporary, "pi-typesafe", "auth.json"); + const storedKeys = () => (existsSync(storedPath) ? Object.keys(JSON.parse(readFileSync(storedPath, "utf8")).keys).sort() : []); + try { + const scoped = await reloadExtension(); + const run = (args: string) => Reflect.apply(scoped.command.handler, scoped.command, [args, ctx]); + await withFetch( + async input => String(input).endsWith("/v1/models") + ? Response.json({ models: [{ name: "jev-latest", description: "", release_date: "2026-01-01" }], data: [{ id: "typesafe/jev-latest" }] }) + : Response.json({ model: "jev-test", answers: { key: { type: "noul", noul: 0.9 } }, usage: { input_tokens: 12, output_tokens: 0 } }), + async (requests, bodies) => { + customResult = "ts_live_key_0123456789abcdef"; + await run("login typesafe"); + assert.deepEqual(storedKeys(), ["typesafe"]); + assert.ok(requests.at(-1)?.endsWith("/v1/models"), `TypeSafe is proved with its model list: ${requests.at(-1)}`); + + // No backend named: the session's own backend, proved with one judge request on its own host. + customResult = "sk-or-0123456789abcdef"; + const before = requests.length; + await run("login"); + const sent = requests.slice(before); + assert.equal(sent.length, 1, `a public model list cannot prove a key, so one request does: ${sent.join(", ")}`); + assert.ok(sent[0]?.startsWith("https://openrouter.ai/"), `the request goes to OpenRouter: ${sent[0]}`); + assert.ok(bodies.at(-1)?.includes('"typesafe/jev'), `the request names the mapped model: ${bodies.at(-1)}`); + assert.ok(notices.at(-1)?.includes("OpenRouter key verified with one request"), `login: ${notices.at(-1)}`); + assert.deepEqual(storedKeys(), ["openrouter", "typesafe"]); + + await run("logout"); + assert.deepEqual(storedKeys(), ["typesafe"], "logging out one backend leaves the other's key"); + customResult = "sk-or-fedcba9876543210"; + await run("login"); + assert.deepEqual(storedKeys(), ["openrouter", "typesafe"], "login works again after a logout"); + await run("status"); + assert.ok(notices.at(-1)?.includes("/typesafe login"), `status reads the stored OpenRouter key: ${notices.at(-1)}`); + }, + ); + } finally { + if (savedEnvKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedEnvKey; + delete process.env.PI_TYPESAFE_BACKEND; + delete process.env.OPENROUTER_API_KEY; + clearStoredApiKey(); } }); diff --git a/tests/login.test.ts b/tests/login.test.ts index 329a535..57d897e 100644 --- a/tests/login.test.ts +++ b/tests/login.test.ts @@ -14,6 +14,7 @@ const savedAgentDir = process.env.PI_CODING_AGENT_DIR; const originalFetch = globalThis.fetch; let customResult: string | undefined; let modelListCalls = 0; +let decisionCalls = 0; const storedPath = () => join(temporary, "pi-typesafe", "auth.json"); const ctx = (hasUI = true) => ({ hasUI, ui: { custom: async () => customResult, input: async () => { throw new Error("plain input must not be used"); } } }) as unknown as ExtensionCommandContext; @@ -25,12 +26,17 @@ before(async () => { modelListCalls++; return Response.json({ models: [{ name: "jev-latest", description: "", release_date: "2026-01-01" }, { name: "jev-preview", description: "", release_date: "2026-01-01" }] }); } + if (String(input).includes("openrouter.ai/api/alpha/decisions")) { + decisionCalls++; + return Response.json({ model: "typesafe/jev-1.13", answers: { key: { type: "noul", noul: 0.99 } }, usage: { input_tokens: 10, output_tokens: 0 } }); + } throw new Error("unexpected request"); }; }); beforeEach(async () => { delete process.env.TYPESAFE_API_KEY; modelListCalls = 0; + decisionCalls = 0; customResult = undefined; await rm(storedPath(), { force: true }); }); @@ -82,15 +88,20 @@ test("ensureApiKey reports an existing key without prompting, otherwise logs in" assert.equal(modelListCalls, 1); }); -test("ensureApiKey for another backend uses its environment variable and never opens the TypeSafe login", async () => { +test("another backend keeps its own key: its environment variable wins, otherwise its own login stores it", async () => { const savedOpenRouter = process.env.OPENROUTER_API_KEY; try { delete process.env.OPENROUTER_API_KEY; - // A stored TypeSafe key does not satisfy OpenRouter, and the prompt must not run: it would verify against api.typesafe.ai. + // A stored TypeSafe key does not satisfy OpenRouter: the store is keyed by backend. customResult = "ts_live_key_0123456789abcdef"; assert.deepEqual(await ensureApiKey(ctx()), { source: "stored", login: { path: storedPath(), models: 2 } }); - await assert.rejects(ensureApiKey(ctx(), { backend: "openrouter" }), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "configuration" && /OPENROUTER_API_KEY/.test(error.message)); - assert.equal(modelListCalls, 1, "no second verification"); + + // OpenRouter's model list is public, so the login proves the key with one request and stores it under its own name. + customResult = "sk-or-0123456789abcdef"; + const login = await ensureApiKey(ctx(), { backend: "openrouter" }); + assert.equal(login?.source, "stored"); + assert.equal(decisionCalls, 1, "one request proves the key"); + assert.equal(modelListCalls, 1, "no model-list call for a public list"); process.env.OPENROUTER_API_KEY = "sk-or-0123456789abcdef"; assert.deepEqual(await ensureApiKey(ctx(), { backend: "openrouter" }), { source: "environment" }); From e9d1039690bc9b9175897d3eafa62c044074dbf7 Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Sat, 24 Oct 2026 10:10:43 +0800 Subject: [PATCH 2/4] Fire ask's deadline on an idle event loop AbortSignal.timeout's timer is unref'd, so a call whose only pending work was the deadline never settled: node:test on Node 22.19.0 reported the ask promise as 'Promise resolution is still pending but the event loop has already resolved' (CI run 39945731438), while Node 24 passed on timing alone. The deadline is now a ref'd timer, cleared when the ask settles, so timeoutMs bounds the call on every supported Node version and the assertion stands unchanged. --- src/ask.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ask.ts b/src/ask.ts index b7569a7..0594bfa 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -30,13 +30,19 @@ export type AskAnswer = * share `prepareEvaluationRequest`, so what one accepts the other accepts. */ export async function ask(judge: Judge, request: SystemOneRequest, options: AskOptions = {}): Promise> { - const timeout = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_ASK_TIMEOUT_MS); - const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout; + // AbortSignal.timeout's timer is unref'd, so a deadline that is the only pending work never fires: node:test on + // Node 22.19 reported this promise as never settling (CI run 39945731438). A ref'd timer, cleared when the ask + // settles, makes the documented deadline real and leaves no handle behind. + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), options.timeoutMs ?? DEFAULT_ASK_TIMEOUT_MS); + const signal = options.signal ? AbortSignal.any([options.signal, deadline.signal]) : deadline.signal; try { const result = await judge.evaluate(request, { signal }); return { ok: true, answers: result.answers, model: result.model, usage: result.usage, elapsedMs: result.elapsedMs }; } catch (error) { if (error instanceof TypeSafeIntegrationError) return { ok: false, error: error.message, errorCode: error.code }; return { ok: false, error: FALLBACK_MESSAGE }; + } finally { + clearTimeout(timer); } } From c1145478e042c083e4416450186faff7da41723a Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Sat, 24 Oct 2026 10:10:43 +0800 Subject: [PATCH 3/4] Release 0.12.0 One stored key per judgment backend, /typesafe login and logout taking a backend name, and the removal of the usesTypesafeKey export: a minor bump, per CONTRIBUTING. Also records the store's downgrade behaviour. --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a70e181..54ee30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## Unreleased +## 0.12.0 + +### Added + +- The login store keeps one key per judgment backend: a registry name, or an endpoint's `keyEnv`. `keySituation`, `authState`, `ensureApiKey`, and `createTypeSafe` read the key stored for the backend in play, so a key stored for one backend is never sent to another. A backend's own environment variable still wins over its stored key. +- `/typesafe login [backend]` prompts with that backend's name, proves the key before saving, and stores it under that backend. TypeSafe and Liquid AI are proved with their model list, which checks the key; SubQ, OpenRouter, and Command Code are proved with one judge request, because their public model list accepts any key. A failed proof stores nothing. +- `/typesafe logout [backend]` removes one backend's key and leaves the others. With no backend named, both commands act on the session's own backend (`PI_TYPESAFE_BACKEND`), and a logout of that backend turns judgments off for the session. + +### Changed + +- `/typesafe login` and `/typesafe logout` no longer decline on a backend other than TypeSafe. One rule now covers every backend: the stored key is used unless the backend's environment variable is set, and then the variable wins. On a backend whose variable is set, `login` says so instead of prompting, and `logout` removes the stored key while saying the variable still wins. +- The README and `.env.example` no longer say that `/typesafe login` does not apply to another backend. + +### Removed + +- `usesTypesafeKey` is no longer exported: every backend has a login store now, so no caller needs to ask. This is a minor release, per CONTRIBUTING. + +### Fixed + +- `ask`'s deadline fired only while some other handle kept the Node event loop alive. `AbortSignal.timeout`'s timer is unref'd, so a call whose only pending work was the deadline never settled and CI reported it as `Promise resolution is still pending but the event loop has already resolved` on Node 22.19.0. The deadline is now a ref'd timer cleared when the call settles, so the documented `timeoutMs` bounds the call on every supported Node version. + +### Storage and downgrade + +- The store file is now `{ "version": 1, "keys": { "": "" } }`, written under a temporary name and renamed whole. A pre-0.12 file `{ "apiKey": "..." }` reads as the TypeSafe key and is rewritten in the new shape on the next save. +- A 0.11.0 release run against a 0.12.0 store reads no TypeSafe key (it sees `missing`, not corruption) and does not touch another backend's key. If that older release saves a key, it rewrites the file in the old shape, and the other backends' stored keys are dropped; those backends fall back to their environment variables. + ## 0.11.0 ### Added diff --git a/package-lock.json b/package-lock.json index 7422bc9..4b816e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.11.0", + "version": "0.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.11.0", + "version": "0.12.0", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0" diff --git a/package.json b/package.json index fe5d276..33ccc2c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.11.0", + "version": "0.12.0", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT", From 0bcfa254ceac5d3b68eaaf0271d6cddce52f71fb Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Sat, 24 Oct 2026 10:11:15 +0800 Subject: [PATCH 4/4] Cover the per-backend key prompt label /typesafe login asks for the named backend's key, so the prompt must name that backend; only the TypeSafe default was asserted. --- tests/key-prompt.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/key-prompt.test.ts b/tests/key-prompt.test.ts index b224aca..12fd471 100644 --- a/tests/key-prompt.test.ts +++ b/tests/key-prompt.test.ts @@ -45,3 +45,13 @@ test("falls back to Pi's plain input when custom UI is unavailable", async () => const ctx = { hasUI: true, ui: { input: async () => " ts_plain_key_0123456789 " } } as unknown as ExtensionCommandContext; assert.equal(await promptForApiKey(ctx), " ts_plain_key_0123456789 "); }); + +test("the prompt names the backend whose key it asks for", async () => { + let screen = ""; + const key = await promptForApiKey(fakeContext(component => { + screen = component.render(60).join("\n"); + component.handleInput("\r"); + }), "OpenRouter"); + assert.ok(screen.includes("OpenRouter API key"), screen); + assert.equal(key, ""); +});