From d1eae711138756b742524bfd4cb8ab689f31f391 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 01:41:34 +0800 Subject: [PATCH 01/12] Add the generation-based proxy token module Tenant proxy tokens gain a v2 form, `v2..`, signed with a key derived from JWT_SECRET by HKDF. The legacy token stays generation 0 and keeps its exact bytes. A per-project accept floor in ROUTING (`proxy-token:`) decides which generations are accepted; a missing key is floor 0, so every current token keeps working. Checks run the project id and syntax checks first, then the MAC, and read the floor only for a valid MAC. A malformed or unreadable floor fails closed. The module also has an isolate cache for floor reads and a route-bound credential the proxy can use for the API metadata routes. Nothing uses the module yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/package.json | 1 + platform/packages/api/src/proxy-token.ts | 372 +++++++++++++++ .../api/test/unit/proxy-token.test.ts | 427 ++++++++++++++++++ 3 files changed, 800 insertions(+) create mode 100644 platform/packages/api/src/proxy-token.ts create mode 100644 platform/packages/api/test/unit/proxy-token.test.ts diff --git a/platform/packages/api/package.json b/platform/packages/api/package.json index aa76e332e..af915c8a8 100644 --- a/platform/packages/api/package.json +++ b/platform/packages/api/package.json @@ -9,6 +9,7 @@ "./reserved-slugs": "./src/reserved-slugs.ts", "./routing": "./src/routing.ts", "./usage-auth": "./src/usage-auth.ts", + "./proxy-token": "./src/proxy-token.ts", "./admin-jsx/jsx-runtime": "./src/admin/react-jsx/jsx-runtime.ts", "./admin-jsx/jsx-dev-runtime": "./src/admin/react-jsx/jsx-dev-runtime.ts" }, diff --git a/platform/packages/api/src/proxy-token.ts b/platform/packages/api/src/proxy-token.ts new file mode 100644 index 000000000..3cd367ace --- /dev/null +++ b/platform/packages/api/src/proxy-token.ts @@ -0,0 +1,372 @@ +import { isProjectId } from './id.ts'; + +/** + * Tenant proxy tokens. + * + * A tenant Worker authenticates to the platform with `__VOID_PROXY_TOKEN`. + * Two forms exist: + * + * - legacy (generation 0): base64url(HMAC-SHA256(JWT_SECRET, projectId)), + * 43 characters, the bytes `hmacSignState` produces; + * - `v2..`: the MAC is HMAC-SHA256 under a key derived from + * JWT_SECRET with HKDF, over the project id and the generation. + * + * ROUTING `proxy-token:` holds the accept floor `{"gen": F}`. A + * token is accepted when its MAC is valid and its generation is at least F. + * A missing key is floor 0, which accepts the legacy token. + * + * Every check runs `isProjectId` and the syntax check before any crypto, and + * the MAC before any KV read, so a forged token never costs a KV read. + */ + +export const PROXY_TOKEN_STATE_KEY_PREFIX = 'proxy-token:'; + +export function proxyTokenStateKey(projectId: string): string { + return `${PROXY_TOKEN_STATE_KEY_PREFIX}${projectId}`; +} + +/** Accept floor. Generation 0 accepts the legacy token. */ +export type ProxyTokenState = Readonly<{ gen: number }>; + +export const LEGACY_PROXY_TOKEN_STATE: ProxyTokenState = Object.freeze({ gen: 0 }); + +/** KV `cacheTtl` for floor reads on request paths (the KV minimum). */ +export const PROXY_TOKEN_KV_CACHE_TTL_SECONDS = 30; +/** How long a proxy isolate serves a floor it has read. */ +export const PROXY_TOKEN_ISOLATE_TTL_MS = 15_000; + +/** The floor could not be read or is malformed. Callers fail closed. */ +export class ProxyTokenStateError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'ProxyTokenStateError'; + } +} + +function isStateGeneration(gen: unknown): gen is number { + return ( + typeof gen === 'number' && + Number.isSafeInteger(gen) && + gen >= 0 && + gen < Number.MAX_SAFE_INTEGER + ); +} + +/** Missing state is floor 0. Anything other than `{"gen": = 0>}` throws. */ +export function parseProxyTokenState(raw: string | null | undefined): ProxyTokenState { + if (raw === null || raw === undefined) { + return LEGACY_PROXY_TOKEN_STATE; + } + let value: unknown; + try { + value = JSON.parse(raw); + } catch (error) { + throw new ProxyTokenStateError('proxy token state is not JSON', { cause: error }); + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ProxyTokenStateError('proxy token state is not an object'); + } + const gen = (value as { gen?: unknown }).gen; + if (!isStateGeneration(gen)) { + throw new ProxyTokenStateError('proxy token state has an invalid generation'); + } + return { gen }; +} + +export function serializeProxyTokenState(state: ProxyTokenState): string { + if (!isStateGeneration(state.gen)) { + throw new ProxyTokenStateError('proxy token state has an invalid generation'); + } + return JSON.stringify({ gen: state.gen }); +} + +export function isAcceptedGeneration(state: ProxyTokenState, gen: number): boolean { + return gen >= state.gen; +} + +/** + * Generations a sender tries, in order, for the issued generation: the + * current one, then the one before it while Workers may still hold it. + */ +export function sendGenerations(issueGen: number): ReadonlyArray { + if (!Number.isSafeInteger(issueGen) || issueGen < 0) { + throw new RangeError(`invalid proxy token generation: ${issueGen}`); + } + return issueGen >= 1 ? [issueGen, issueGen - 1] : [0]; +} + +// --- Keys --- + +const encoder = new TextEncoder(); + +const legacyKeys = new Map>(); +const proxyTokenKeys = new Map>(); +const proxyInternalKeys = new Map>(); + +function cachedKey( + cache: Map>, + secret: string, + create: () => Promise, +): Promise { + let key = cache.get(secret); + if (!key) { + key = create(); + cache.set(secret, key); + key.catch(() => cache.delete(secret)); + } + return key; +} + +function legacyKey(secret: string): Promise { + return cachedKey(legacyKeys, secret, () => + crypto.subtle.importKey( + 'raw', + encoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign', 'verify'], + ), + ); +} + +async function hkdfHmacKey(secret: string, salt: string, info: string): Promise { + const base = await crypto.subtle.importKey('raw', encoder.encode(secret), 'HKDF', false, [ + 'deriveKey', + ]); + return crypto.subtle.deriveKey( + { name: 'HKDF', hash: 'SHA-256', salt: encoder.encode(salt), info: encoder.encode(info) }, + base, + // Without `length`, WebCrypto derives a 64-byte (block size) HMAC key. + { name: 'HMAC', hash: 'SHA-256', length: 256 }, + false, + ['sign', 'verify'], + ); +} + +function proxyTokenKey(secret: string): Promise { + return cachedKey(proxyTokenKeys, secret, () => hkdfHmacKey(secret, 'void-proxy-token', 'v2')); +} + +function proxyInternalKey(secret: string): Promise { + return cachedKey(proxyInternalKeys, secret, () => + hkdfHmacKey(secret, 'void-proxy-internal', 'v1'), + ); +} + +// --- Encoding --- + +const MAC = /^[A-Za-z0-9_-]{43}$/; +const V2_TOKEN = /^v2\.([1-9][0-9]{0,15})\.([A-Za-z0-9_-]{43})$/; + +function encodeBase64url(bytes: ArrayBuffer): string { + let binary = ''; + for (const byte of new Uint8Array(bytes)) { + binary += String.fromCharCode(byte); + } + return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +/** Only for values that already matched `MAC`. */ +function decodeMac(value: string): Uint8Array { + const binary = atob(`${value.replace(/-/g, '+').replace(/_/g, '/')}=`); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); +} + +async function sign(key: Promise, message: string): Promise { + return encodeBase64url(await crypto.subtle.sign('HMAC', await key, encoder.encode(message))); +} + +async function verify(key: Promise, message: string, mac: string): Promise { + return crypto.subtle.verify('HMAC', await key, decodeMac(mac), encoder.encode(message)); +} + +function proxyTokenMessage(projectId: string, gen: number): string { + return `void-proxy-token:v2:${projectId}:${gen}`; +} + +// --- Tenant tokens --- + +/** Generation 0 is the legacy token. */ +export async function deriveProxyToken( + jwtSecret: string, + projectId: string, + gen: number, +): Promise { + if (!isProjectId(projectId)) { + throw new Error('invalid project id'); + } + if (!Number.isSafeInteger(gen) || gen < 0) { + throw new RangeError(`invalid proxy token generation: ${gen}`); + } + if (gen === 0) { + return sign(legacyKey(jwtSecret), projectId); + } + return `v2.${gen}.${await sign(proxyTokenKey(jwtSecret), proxyTokenMessage(projectId, gen))}`; +} + +/** The token's generation when its MAC is valid for `projectId`, else null. No I/O. */ +export async function proxyTokenGeneration( + jwtSecret: string, + projectId: string, + token: string, +): Promise { + if (!isProjectId(projectId)) { + return null; + } + if (MAC.test(token)) { + return (await verify(legacyKey(jwtSecret), projectId, token)) ? 0 : null; + } + const match = V2_TOKEN.exec(token); + if (!match) { + return null; + } + const gen = Number(match[1]); + if (!Number.isSafeInteger(gen)) { + return null; + } + const valid = await verify(proxyTokenKey(jwtSecret), proxyTokenMessage(projectId, gen), match[2]); + return valid ? gen : null; +} + +/** + * Reads the accept floor. A KV error or a malformed value throws + * `ProxyTokenStateError`. + */ +export async function readProxyTokenState( + routing: Pick, + projectId: string, + options?: Readonly<{ cacheTtl?: number }>, +): Promise { + let raw: string | null; + try { + raw = + options?.cacheTtl === undefined + ? await routing.get(proxyTokenStateKey(projectId)) + : await routing.get(proxyTokenStateKey(projectId), { cacheTtl: options.cacheTtl }); + } catch (error) { + throw new ProxyTokenStateError('proxy token state unavailable', { cause: error }); + } + return parseProxyTokenState(raw); +} + +/** + * MAC first, then the floor. `readState` runs only for a token with a valid + * MAC; a `ProxyTokenStateError` from it propagates. + */ +export async function verifyProxyToken( + input: Readonly<{ + jwtSecret: string; + projectId: string; + token: string; + readState: () => Promise; + }>, +): Promise { + const gen = await proxyTokenGeneration(input.jwtSecret, input.projectId, input.token); + if (gen === null) { + return false; + } + const state = await input.readState(); + if (!isAcceptedGeneration(state, gen)) { + console.warn('[proxy-token] rejected generation', { + projectId: input.projectId, + gen, + floor: state.gen, + }); + return false; + } + return true; +} + +/** + * Returns the token for the current floor (floor 0 is the legacy token). + * Stopgap until the issued generation is stored with the project. + */ +export async function issueProxyToken( + routing: Pick, + jwtSecret: string, + projectId: string, +): Promise { + const state = await readProxyTokenState(routing, projectId); + return deriveProxyToken(jwtSecret, projectId, state.gen); +} + +// --- Isolate cache --- + +export type ProxyTokenStateCache = Readonly<{ + read(routing: Pick, projectId: string): Promise; + clear(): void; +}>; + +/** + * Floors read with `cacheTtl`, kept for `ttlMs` (missing keys too). Errors are + * never cached, and an expired entry is never served when KV fails. + */ +export function createProxyTokenStateCache( + options?: Readonly<{ maxEntries?: number; ttlMs?: number; now?: () => number }>, +): ProxyTokenStateCache { + const maxEntries = options?.maxEntries ?? 1000; + const ttlMs = options?.ttlMs ?? PROXY_TOKEN_ISOLATE_TTL_MS; + const now = options?.now ?? Date.now; + const entries = new Map(); + + return { + async read(routing, projectId) { + const entry = entries.get(projectId); + if (entry && now() - entry.fetchedAt < ttlMs) { + return entry.state; + } + entries.delete(projectId); + const state = await readProxyTokenState(routing, projectId, { + cacheTtl: PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + }); + entries.set(projectId, { state, fetchedAt: now() }); + while (entries.size > maxEntries) { + const oldest = entries.keys().next().value; + if (oldest === undefined) { + break; + } + entries.delete(oldest); + } + return state; + }, + clear() { + entries.clear(); + }, + }; +} + +// --- Proxy-to-API metadata auth --- + +/** Header the proxy uses to authenticate itself to the API metadata routes. */ +export const PROXY_INTERNAL_AUTH_HEADER = 'x-void-proxy-internal'; + +export type ProxyInternalRoute = 'resolve-worker' | 'resolve-resource' | 'verify-prerender-build'; + +function proxyInternalMessage(route: ProxyInternalRoute, projectId: string): string { + return `void-proxy-internal:v1\n${route}\n${projectId}`; +} + +/** + * Proxy credential for one metadata route and project. Platform only: never + * put it in a Worker binding, KV, a response body or a log. + */ +export function deriveProxyInternalAuth( + jwtSecret: string, + route: ProxyInternalRoute, + projectId: string, +): Promise { + return sign(proxyInternalKey(jwtSecret), proxyInternalMessage(route, projectId)); +} + +/** Never throws on bad input. */ +export async function verifyProxyInternalAuth( + jwtSecret: string, + route: ProxyInternalRoute, + projectId: string, + value: string, +): Promise { + if (!MAC.test(value)) { + return false; + } + return verify(proxyInternalKey(jwtSecret), proxyInternalMessage(route, projectId), value); +} diff --git a/platform/packages/api/test/unit/proxy-token.test.ts b/platform/packages/api/test/unit/proxy-token.test.ts new file mode 100644 index 000000000..fbba55351 --- /dev/null +++ b/platform/packages/api/test/unit/proxy-token.test.ts @@ -0,0 +1,427 @@ +import { createHmac, hkdfSync } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vite-plus/test'; +import { hmacSignState } from '../../src/auth/hmac.ts'; +import { signJwt } from '../../src/auth/jwt.ts'; +import { + createProxyTokenStateCache, + deriveProxyInternalAuth, + deriveProxyToken, + isAcceptedGeneration, + issueProxyToken, + LEGACY_PROXY_TOKEN_STATE, + parseProxyTokenState, + PROXY_INTERNAL_AUTH_HEADER, + PROXY_TOKEN_ISOLATE_TTL_MS, + PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + type ProxyInternalRoute, + proxyTokenGeneration, + proxyTokenStateKey, + type ProxyTokenState, + ProxyTokenStateError, + readProxyTokenState, + sendGenerations, + serializeProxyTokenState, + verifyProxyInternalAuth, + verifyProxyToken, +} from '../../src/proxy-token.ts'; + +const SECRET = 'test-secret-key'; +const PROJECT = 'proj_000000000001'; +const OTHER_PROJECT = 'proj_000000000002'; + +const TOK_0 = 'ZMRzpVRsaOJSG_unVlA0oqSaz2kApOohvao_XGR8Q0c'; +const TOK_1 = 'v2.1.3W5_i2g79pUCXyzxWPQWmTqscSNtC_2vuHC84z775Y8'; +const TOK_2 = 'v2.2.Vx_qycCHPx2BoYvQrbQ7JrhbONKU7f0jNK23B0YUhOs'; +const INTERNAL: Record = { + 'resolve-worker': 'fSljhl8vCcBSR-jWXklS-KDL4K8lFb_wPaRAu-aKPv0', + 'resolve-resource': 'gNOx85Zh_di7yHFjG8AlV0QQ8PbHYs1ipC2ukAl7i5Y', + 'verify-prerender-build': 'A7F1IjLUbEWq0atQH3W5dsR1DidJ6ZgqU6QQ3lehBbc', +}; +const ROUTES = Object.keys(INTERNAL) as Array; + +function base64url(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('base64url'); +} + +function nodeTok(gen: number, projectId = PROJECT): string { + if (gen === 0) { + return base64url(createHmac('sha256', SECRET).update(projectId).digest()); + } + const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-token', 'v2', 32)); + const mac = createHmac('sha256', key).update(`void-proxy-token:v2:${projectId}:${gen}`).digest(); + return `v2.${gen}.${base64url(mac)}`; +} + +function nodeInternal(route: ProxyInternalRoute, projectId = PROJECT): string { + const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-internal', 'v1', 32)); + return base64url( + createHmac('sha256', key).update(`void-proxy-internal:v1\n${route}\n${projectId}`).digest(), + ); +} + +function kvStub(values: Record = {}) { + const get = vi.fn(async (key: string, _options?: unknown) => values[key] ?? null); + return { routing: { get } as unknown as Pick, get, values }; +} + +function throwingKv() { + const get = vi.fn(async () => { + throw new Error('kv down'); + }); + return { routing: { get } as unknown as Pick, get }; +} + +function verify(token: string, state: ProxyTokenState | Error = LEGACY_PROXY_TOKEN_STATE) { + const readState = vi.fn(async () => { + if (state instanceof Error) { + throw state; + } + return state; + }); + return { + readState, + result: verifyProxyToken({ jwtSecret: SECRET, projectId: PROJECT, token, readState }), + }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('vectors', () => { + it.each([ + [0, TOK_0], + [1, TOK_1], + [2, TOK_2], + ])('derives tok(%i)', async (gen, expected) => { + const token = await deriveProxyToken(SECRET, PROJECT, gen); + expect(token).toBe(expected); + expect(token).toBe(nodeTok(gen)); + }); + + it('keeps the legacy token byte-identical to hmacSignState', async () => { + expect(await deriveProxyToken(SECRET, PROJECT, 0)).toBe(await hmacSignState(PROJECT, SECRET)); + expect(TOK_0).toMatch(/^[A-Za-z0-9_-]{43}$/); + }); + + it.each(ROUTES)('derives the internal value for %s', async (route) => { + const value = await deriveProxyInternalAuth(SECRET, route, PROJECT); + expect(value).toBe(INTERNAL[route]); + expect(value).toBe(nodeInternal(route)); + }); + + it('keeps internal values distinct from each other and from tenant tokens', () => { + const values = [...Object.values(INTERNAL), TOK_0, TOK_1.slice('v2.1.'.length)]; + expect(new Set(values).size).toBe(values.length); + }); + + it('names the proxy internal header', () => { + expect(PROXY_INTERNAL_AUTH_HEADER).toBe('x-void-proxy-internal'); + }); +}); + +describe('parseProxyTokenState', () => { + it.each([ + [null, 0], + [undefined, 0], + ['{"gen":0}', 0], + ['{"gen":3}', 3], + ['{"gen":3,"next":4}', 3], + ])('parses %j', (raw, gen) => { + expect(parseProxyTokenState(raw)).toEqual({ gen }); + }); + + it.each([ + 'not json', + '[]', + 'null', + '{"gen":-1}', + '{"gen":1.5}', + '{"gen":"1"}', + '{"next":1}', + `{"gen":${Number.MAX_SAFE_INTEGER}}`, + ])('throws for %s', (raw) => { + expect(() => parseProxyTokenState(raw)).toThrow(ProxyTokenStateError); + }); + + it('serializes a state that parses back', () => { + expect(serializeProxyTokenState({ gen: 4 })).toBe('{"gen":4}'); + expect(parseProxyTokenState(serializeProxyTokenState({ gen: 4 }))).toEqual({ gen: 4 }); + expect(() => serializeProxyTokenState({ gen: -1 })).toThrow(ProxyTokenStateError); + }); + + it('keys the state by project id', () => { + expect(proxyTokenStateKey(PROJECT)).toBe('proxy-token:proj_000000000001'); + }); +}); + +describe('state table', () => { + it.each([ + { raw: null, accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, + { raw: '{"gen":0}', accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, + { raw: '{"gen":1}', accepted: [1, 2], rejected: [0], send: [1, 0], issue: 1 }, + { raw: '{"gen":3}', accepted: [3, 4], rejected: [0, 1, 2], send: [3, 2], issue: 3 }, + ])('floor $raw', async ({ raw, accepted, rejected, send, issue }) => { + const { routing } = kvStub(raw === null ? {} : { [proxyTokenStateKey(PROJECT)]: raw }); + const state = await readProxyTokenState(routing, PROJECT); + for (const gen of accepted) { + expect(isAcceptedGeneration(state, gen)).toBe(true); + expect(await verify(await deriveProxyToken(SECRET, PROJECT, gen), state).result).toBe(true); + } + for (const gen of rejected) { + expect(isAcceptedGeneration(state, gen)).toBe(false); + expect(await verify(await deriveProxyToken(SECRET, PROJECT, gen), state).result).toBe(false); + } + expect(sendGenerations(state.gen)).toEqual(send); + expect(await issueProxyToken(routing, SECRET, PROJECT)).toBe( + await deriveProxyToken(SECRET, PROJECT, issue), + ); + }); + + it('fails closed on a malformed floor', async () => { + const { routing } = kvStub({ [proxyTokenStateKey(PROJECT)]: '{"gen":"x"}' }); + await expect(readProxyTokenState(routing, PROJECT)).rejects.toThrow(ProxyTokenStateError); + await expect(issueProxyToken(routing, SECRET, PROJECT)).rejects.toThrow(ProxyTokenStateError); + }); + + it('sends the issued generation, then the one before it', () => { + expect(sendGenerations(0)).toEqual([0]); + expect(sendGenerations(1)).toEqual([1, 0]); + expect(sendGenerations(5)).toEqual([5, 4]); + }); +}); + +describe('verifyProxyToken', () => { + it('accepts legacy and v2 tokens when no floor is set', async () => { + expect(await verify(TOK_0).result).toBe(true); + expect(await verify(TOK_1).result).toBe(true); + }); + + it('accepts generations at or above the floor only', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + expect(await verify(TOK_2, { gen: 2 }).result).toBe(true); + expect(await verify(await deriveProxyToken(SECRET, PROJECT, 3), { gen: 2 }).result).toBe(true); + expect(await verify(TOK_1, { gen: 2 }).result).toBe(false); + expect(await verify(TOK_0, { gen: 2 }).result).toBe(false); + expect(warn).toHaveBeenCalledWith('[proxy-token] rejected generation', { + projectId: PROJECT, + gen: 1, + floor: 2, + }); + expect(JSON.stringify(warn.mock.calls)).not.toContain(TOK_1); + }); + + it("rejects another project's token", async () => { + expect(await verify(await deriveProxyToken(SECRET, OTHER_PROJECT, 1)).result).toBe(false); + expect(await verify(await deriveProxyToken(SECRET, OTHER_PROJECT, 0)).result).toBe(false); + }); + + it('rejects a MAC relabelled with another generation', async () => { + const relabelled = `v2.2.${TOK_1.slice('v2.1.'.length)}`; + const { result, readState } = verify(relabelled); + expect(await result).toBe(false); + expect(readState).not.toHaveBeenCalled(); + }); + + it('rejects a bad MAC without reading the floor', async () => { + const badMac = `v2.1.${'A'.repeat(43)}`; + for (const token of [badMac, 'A'.repeat(43)]) { + const { result, readState } = verify(token); + expect(await result).toBe(false); + expect(readState).not.toHaveBeenCalled(); + } + }); + + it.each([ + ['leading zero generation', `v2.01.${TOK_1.slice('v2.1.'.length)}`], + ['leading zero and short MAC', 'v2.01.x'], + ['42-char MAC', TOK_0.slice(0, 42)], + ['44-char MAC', `${TOK_0}A`], + ['trailing padding', `${TOK_0}=`], + ['standard base64 alphabet', `+${TOK_0.slice(1)}`], + ['empty token', ''], + ['unsafe generation', `v2.9999999999999999.${TOK_1.slice('v2.1.'.length)}`], + ])('rejects a malformed token (%s) without reading the floor', async (_name, token) => { + const { result, readState } = verify(token); + expect(await result).toBe(false); + expect(readState).not.toHaveBeenCalled(); + }); + + it('propagates a state error after a valid MAC', async () => { + await expect(verify(TOK_1, new ProxyTokenStateError('down')).result).rejects.toThrow( + ProxyTokenStateError, + ); + }); + + it('rejects a login JWT split into project id and token', async () => { + const jwt = await signJwt({ sub: 'user_1', login: 'octocat', aud: 'api' }, SECRET); + const [header, body, signature] = jwt.split('.'); + const projectId = `${header}.${body}`; + // The JWT signature is the legacy MAC over this string. + expect(await hmacSignState(projectId, SECRET)).toBe(signature); + expect(await proxyTokenGeneration(SECRET, projectId, signature)).toBeNull(); + const readState = vi.fn(async () => LEGACY_PROXY_TOKEN_STATE); + expect( + await verifyProxyToken({ jwtSecret: SECRET, projectId, token: signature, readState }), + ).toBe(false); + expect(readState).not.toHaveBeenCalled(); + }); +}); + +describe('proxyTokenGeneration', () => { + it('returns the generation of a valid token', async () => { + expect(await proxyTokenGeneration(SECRET, PROJECT, TOK_0)).toBe(0); + expect(await proxyTokenGeneration(SECRET, PROJECT, TOK_2)).toBe(2); + expect(await proxyTokenGeneration('other-secret', PROJECT, TOK_2)).toBeNull(); + }); +}); + +describe('deriveProxyToken', () => { + it.each([ + ['a non-project id', 'proj_1', 1], + ['a negative generation', PROJECT, -1], + ['a fractional generation', PROJECT, 1.5], + ])('throws for %s', async (_name, projectId, gen) => { + await expect(deriveProxyToken(SECRET, projectId, gen)).rejects.toThrow(); + }); +}); + +describe('readProxyTokenState', () => { + it('passes cacheTtl through only when given', async () => { + const { routing, get } = kvStub({ [proxyTokenStateKey(PROJECT)]: '{"gen":1}' }); + expect(await readProxyTokenState(routing, PROJECT, { cacheTtl: 30 })).toEqual({ gen: 1 }); + expect(get).toHaveBeenLastCalledWith('proxy-token:proj_000000000001', { cacheTtl: 30 }); + await readProxyTokenState(routing, PROJECT); + expect(get).toHaveBeenLastCalledWith('proxy-token:proj_000000000001'); + }); + + it('wraps a KV error', async () => { + const { routing } = throwingKv(); + const error = await readProxyTokenState(routing, PROJECT).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ProxyTokenStateError); + expect((error as Error).cause).toEqual(new Error('kv down')); + }); + + it('pins the timing constants', () => { + expect(PROXY_TOKEN_KV_CACHE_TTL_SECONDS).toBe(30); + expect(PROXY_TOKEN_ISOLATE_TTL_MS).toBe(15_000); + }); +}); + +describe('createProxyTokenStateCache', () => { + function clock() { + let time = 1_000_000; + return { + now: () => time, + advance(ms: number) { + time += ms; + }, + }; + } + + it('caches a missing key and re-reads after the isolate TTL', async () => { + const time = clock(); + const cache = createProxyTokenStateCache({ now: time.now }); + const { routing, get, values } = kvStub(); + + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 0 }); + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 0 }); + expect(get).toHaveBeenCalledTimes(1); + expect(get).toHaveBeenCalledWith('proxy-token:proj_000000000001', { + cacheTtl: PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + }); + + values[proxyTokenStateKey(PROJECT)] = '{"gen":2}'; + time.advance(PROXY_TOKEN_ISOLATE_TTL_MS - 1); + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 0 }); + time.advance(1); + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 2 }); + expect(get).toHaveBeenCalledTimes(2); + }); + + it('evicts the oldest entry past the size bound', async () => { + const cache = createProxyTokenStateCache({ now: clock().now }); + const { routing, get } = kvStub(); + const ids = Array.from({ length: 1001 }, (_, i) => `proj_${String(i).padStart(12, '0')}`); + for (const id of ids) { + await cache.read(routing, id); + } + expect(get).toHaveBeenCalledTimes(1001); + await cache.read(routing, ids[1000]); + await cache.read(routing, ids[1]); + expect(get).toHaveBeenCalledTimes(1001); + await cache.read(routing, ids[0]); + expect(get).toHaveBeenCalledTimes(1002); + }); + + it('does not serve an expired entry or cache an error when KV fails', async () => { + const time = clock(); + const cache = createProxyTokenStateCache({ now: time.now }); + const { routing, get, values } = kvStub({ [proxyTokenStateKey(PROJECT)]: '{"gen":1}' }); + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 1 }); + + time.advance(PROXY_TOKEN_ISOLATE_TTL_MS); + get.mockRejectedValueOnce(new Error('kv down')); + await expect(cache.read(routing, PROJECT)).rejects.toThrow(ProxyTokenStateError); + + values[proxyTokenStateKey(PROJECT)] = '{"gen":2}'; + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 2 }); + expect(get).toHaveBeenCalledTimes(3); + }); + + it('does not cache a malformed value', async () => { + const cache = createProxyTokenStateCache({ now: clock().now }); + const { routing, get, values } = kvStub({ [proxyTokenStateKey(PROJECT)]: 'broken' }); + await expect(cache.read(routing, PROJECT)).rejects.toThrow(ProxyTokenStateError); + values[proxyTokenStateKey(PROJECT)] = '{"gen":1}'; + expect(await cache.read(routing, PROJECT)).toEqual({ gen: 1 }); + expect(get).toHaveBeenCalledTimes(2); + }); + + it('clear() empties the cache', async () => { + const cache = createProxyTokenStateCache({ now: clock().now }); + const { routing, get } = kvStub(); + await cache.read(routing, PROJECT); + cache.clear(); + await cache.read(routing, PROJECT); + expect(get).toHaveBeenCalledTimes(2); + }); +}); + +describe('verifyProxyInternalAuth', () => { + it.each(ROUTES)('accepts the value for %s', async (route) => { + expect(await verifyProxyInternalAuth(SECRET, route, PROJECT, INTERNAL[route])).toBe(true); + }); + + it('rejects a value for another route or project', async () => { + expect( + await verifyProxyInternalAuth( + SECRET, + 'resolve-resource', + PROJECT, + INTERNAL['resolve-worker'], + ), + ).toBe(false); + expect( + await verifyProxyInternalAuth( + SECRET, + 'resolve-worker', + OTHER_PROJECT, + INTERNAL['resolve-worker'], + ), + ).toBe(false); + }); + + it.each([ + ['42 chars', INTERNAL['resolve-worker'].slice(0, 42)], + ['44 chars', `${INTERNAL['resolve-worker']}A`], + ['a v2 token shape', 'v2.1.x'], + ['a tenant token', TOK_1], + ['the legacy token', TOK_0], + ['empty', ''], + ])('rejects %s without throwing', async (_name, value) => { + await expect(verifyProxyInternalAuth(SECRET, 'resolve-worker', PROJECT, value)).resolves.toBe( + false, + ); + }); +}); From 520546a28dfd94e6562e3d0b9d7354ddba84eb0f Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 01:47:04 +0800 Subject: [PATCH 02/12] Accept v2 proxy tokens alongside the legacy token The proxy's service-binding auth and /email/verify-receipt now check the tenant project token through the shared proxy token module: the MAC first, then the project's accept floor in ROUTING. Both the legacy token and `v2..` tokens are accepted while their generation is at or above the floor. The proxy keeps one floor cache per isolate. A floor that cannot be read or is malformed returns 503 instead of accepting the token. /internal/revalidate checks the forwarded token against the same floor, so revalidation keeps working with v2 tokens and stops accepting a token the proxy would refuse. No floor is written yet, so every project keeps floor 0 and the legacy token works exactly as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../packages/api/src/internal-proxy-auth.ts | 37 +++++ .../api/src/routes/internal-revalidate.ts | 17 ++- .../api-internal-project-token.test.ts | 97 ++++++++++++ platform/packages/proxy/src/index.ts | 44 +++--- platform/packages/proxy/src/proxy-token.ts | 58 ++++++++ platform/packages/proxy/test/unit/d1.test.ts | 5 +- .../test/unit/email-custom-domain.test.ts | 2 + .../packages/proxy/test/unit/email.test.ts | 21 ++- platform/packages/proxy/test/unit/kv.test.ts | 5 +- .../packages/proxy/test/unit/proxy.test.ts | 140 +++++++++++++++++- platform/packages/proxy/test/unit/r2.test.ts | 5 +- 11 files changed, 391 insertions(+), 40 deletions(-) create mode 100644 platform/packages/api/src/internal-proxy-auth.ts create mode 100644 platform/packages/proxy/src/proxy-token.ts diff --git a/platform/packages/api/src/internal-proxy-auth.ts b/platform/packages/api/src/internal-proxy-auth.ts new file mode 100644 index 000000000..3151704d7 --- /dev/null +++ b/platform/packages/api/src/internal-proxy-auth.ts @@ -0,0 +1,37 @@ +import type { ApiEnv } from './env.ts'; +import { + PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + ProxyTokenStateError, + readProxyTokenState, + verifyProxyToken, +} from './proxy-token.ts'; + +/** + * Checks a tenant project token forwarded by the proxy against the project's + * accept floor. A floor that cannot be read or is malformed is + * `'unavailable'`, never accepted. + */ +export async function checkTenantProxyToken( + env: Pick, + projectId: string, + token: string, +): Promise<'ok' | 'invalid' | 'unavailable'> { + try { + const valid = await verifyProxyToken({ + jwtSecret: env.JWT_SECRET, + projectId, + token, + readState: () => + readProxyTokenState(env.ROUTING, projectId, { + cacheTtl: PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + }), + }); + return valid ? 'ok' : 'invalid'; + } catch (error) { + if (error instanceof ProxyTokenStateError) { + console.error('[proxy-token] state unavailable', { projectId, error }); + return 'unavailable'; + } + throw error; + } +} diff --git a/platform/packages/api/src/routes/internal-revalidate.ts b/platform/packages/api/src/routes/internal-revalidate.ts index c36de1a78..2983143f3 100644 --- a/platform/packages/api/src/routes/internal-revalidate.ts +++ b/platform/packages/api/src/routes/internal-revalidate.ts @@ -1,15 +1,16 @@ import { Hono } from 'hono'; -import { hmacVerifyState } from '../auth/hmac.ts'; import { getProject } from '../db.ts'; import type { ApiEnv } from '../env.ts'; import { isProjectId } from '../id.ts'; +import { checkTenantProxyToken } from '../internal-proxy-auth.ts'; import { runRevalidate } from '../revalidate-core.ts'; const app = new Hono(); /** * Internal revalidation endpoint — called by the proxy via service binding. - * Authenticated via HMAC token in X-Void-Internal header (no JWT/userId). + * Authenticated by the tenant project token the proxy forwards in the + * X-Void-Internal header (no JWT/userId). */ app.post('/:id', async (context) => { const projectId = context.req.param('id'); @@ -18,10 +19,14 @@ app.post('/:id', async (context) => { if (!token) { return context.json({ error: 'missing internal auth' }, 401); } - if ( - !isProjectId(projectId) || - !(await hmacVerifyState(projectId, token, context.env.JWT_SECRET)) - ) { + if (!isProjectId(projectId)) { + return context.json({ error: 'invalid internal auth' }, 403); + } + const tokenCheck = await checkTenantProxyToken(context.env, projectId, token); + if (tokenCheck === 'unavailable') { + return context.json({ error: 'project token state unavailable' }, 503); + } + if (tokenCheck === 'invalid') { return context.json({ error: 'invalid internal auth' }, 403); } diff --git a/platform/packages/api/test/integration/api-internal-project-token.test.ts b/platform/packages/api/test/integration/api-internal-project-token.test.ts index 4ff2e96dd..5879bb547 100644 --- a/platform/packages/api/test/integration/api-internal-project-token.test.ts +++ b/platform/packages/api/test/integration/api-internal-project-token.test.ts @@ -12,6 +12,7 @@ import { hmacSignState } from '../../src/auth/hmac.ts'; import { signJwt } from '../../src/auth/jwt.ts'; import { createResource } from '../../src/db.ts'; import app from '../../src/index.ts'; +import { deriveProxyToken } from '../../src/proxy-token.ts'; import { clearTables, createTestDeployment, @@ -70,6 +71,14 @@ async function internalRequest( ); } +async function clearProxyTokenStateCache(): Promise { + const moduleUrl = new URL('../../../proxy/src/proxy-token.ts', import.meta.url).href; + const proxyToken = (await import(/* @vite-ignore */ moduleUrl)) as { + clearProxyTokenStateCache(): void; + }; + proxyToken.clearProxyTokenStateCache(); +} + async function proxyRequest(projectId: string, aiToken: string) { const proxyModuleUrl = new URL('../../../proxy/src/index.ts', import.meta.url).href; const proxyWorker = ( @@ -162,3 +171,91 @@ describe('per-project token checks', () => { expect(await prerender.json()).toEqual({ ok: true }); }); }); + +describe('project token generations', () => { + async function liveProject() { + const user = await createTestUser(env.db); + const project = await createTestProject(env.db, user.id); + const deployment = await createTestDeployment(env.db, project.id, 'live'); + await env.ROUTING.put(project.slug, JSON.stringify({ version: deployment.id })); + mockCfApi(); + return project; + } + + async function setFloor(projectId: string, value: string) { + await env.ROUTING.put(`proxy-token:${projectId}`, value); + } + + beforeEach(async () => { + await clearProxyTokenStateCache(); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + it('accepts tokens at or above the floor at the proxy and rejects the legacy token', async () => { + const project = await liveProject(); + await setFloor(project.id, '{"gen":1}'); + for (const gen of [1, 2]) { + const { response } = await proxyRequest( + project.id, + await deriveProxyToken(JWT_SECRET, project.id, gen), + ); + expect(response.status).toBe(200); + } + const { response, run } = await proxyRequest( + project.id, + await hmacSignState(project.id, JWT_SECRET), + ); + expect(response.status).toBe(403); + expect(await response.json()).toEqual({ error: 'invalid project token' }); + expect(run).not.toHaveBeenCalled(); + }); + + it('accepts the legacy and v2 tokens at /internal/revalidate while no floor is set', async () => { + const project = await liveProject(); + for (const gen of [0, 1]) { + const response = await internalRequest( + 'revalidate', + project.id, + await deriveProxyToken(JWT_SECRET, project.id, gen), + { paths: ['/about'] }, + ); + expect(response.status).toBe(200); + } + }); + + it('applies the floor at /internal/revalidate', async () => { + const project = await liveProject(); + await setFloor(project.id, '{"gen":1}'); + for (const gen of [1, 2]) { + const response = await internalRequest( + 'revalidate', + project.id, + await deriveProxyToken(JWT_SECRET, project.id, gen), + { paths: ['/about'] }, + ); + expect(response.status).toBe(200); + } + const legacy = await internalRequest( + 'revalidate', + project.id, + await hmacSignState(project.id, JWT_SECRET), + { paths: ['/about'] }, + ); + expect(legacy.status).toBe(403); + expect(await legacy.json()).toEqual({ error: 'invalid internal auth' }); + }); + + it('fails closed at /internal/revalidate when the floor is malformed', async () => { + const project = await liveProject(); + await setFloor(project.id, 'not json'); + const response = await internalRequest( + 'revalidate', + project.id, + await hmacSignState(project.id, JWT_SECRET), + { paths: ['/about'] }, + ); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: 'project token state unavailable' }); + }); +}); diff --git a/platform/packages/proxy/src/index.ts b/platform/packages/proxy/src/index.ts index 0b713dbf6..f67c8e90c 100644 --- a/platform/packages/proxy/src/index.ts +++ b/platform/packages/proxy/src/index.ts @@ -9,10 +9,10 @@ import { asString } from './body.ts'; import { proxyCapabilityEnabled } from './capabilities.ts'; import { d1 } from './d1.ts'; import { email } from './email.ts'; -import { hmacSign } from './hmac.ts'; import { guardHumanStream } from './human-stream.ts'; import { kv } from './kv.ts'; import { PLATFORM_DISABLED_KEY, PLATFORM_DISABLED_VALUE } from './platform-lifecycle.ts'; +import { verifyTenantProxyToken } from './proxy-token.ts'; import { r2 } from './r2.ts'; import { readRequestJson, requestBodyMaxBytes, RequestBodyError } from './request-body.ts'; import { verifyPrerenderAuthority, verifyProjectDeployAccess } from './resolve.ts'; @@ -98,13 +98,17 @@ app.post('/email/verify-receipt', async (context) => { ) { return context.json({ valid: false }, 400); } - if ( - !isProjectId(input.projectId) || - context.req.header('x-void-internal') !== - (await hmacSign(input.projectId, context.env.JWT_SECRET)) - ) { + if (!isProjectId(input.projectId)) { return context.json({ valid: false }, 403); } + const tokenCheck = await verifyTenantProxyToken( + context.env, + input.projectId, + context.req.header('x-void-internal') ?? '', + ); + if (tokenCheck !== 'ok') { + return context.json({ valid: false }, tokenCheck === 'unavailable' ? 503 : 403); + } if (!context.env.EMAIL_SIGNING_SECRET) { return context.json({ valid: false }, 503); } @@ -229,16 +233,20 @@ app.post('/*', async (context, next) => { } context.set('projectId', projectId); } else { - // Service binding path (production) — verify HMAC-signed project token + // Service binding path (production) — verify the project token const projectId = asString(body.projectId); const aiToken = asString(body.aiToken); if (!projectId || !aiToken) { return context.json({ error: 'projectId and aiToken required' }, 400); } - if ( - !isProjectId(projectId) || - !(await verifyHmac(projectId, aiToken, context.env.JWT_SECRET)) - ) { + if (!isProjectId(projectId)) { + return context.json({ error: 'invalid project token' }, 403); + } + const tokenCheck = await verifyTenantProxyToken(context.env, projectId, aiToken); + if (tokenCheck === 'unavailable') { + return context.json({ error: 'project token state unavailable' }, 503); + } + if (tokenCheck === 'invalid') { return context.json({ error: 'invalid project token' }, 403); } context.set('projectId', projectId); @@ -301,7 +309,7 @@ app.route('/email', email); export default { fetch: app.fetch }; -// --- HMAC verification --- +// --- JWT verification --- const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -324,18 +332,6 @@ async function getVerifyKey(secret: string): Promise { return cachedVerifyKey; } -async function verifyHmac(data: string, signature: string, secret: string): Promise { - try { - const key = await getVerifyKey(secret); - const sigBytes = base64urlDecode(signature); - return crypto.subtle.verify('HMAC', key, sigBytes.buffer as ArrayBuffer, encoder.encode(data)); - } catch { - return false; - } -} - -// --- JWT verification --- - function base64urlDecode(str: string): Uint8Array { const padded = str.replace(/-/g, '+').replace(/_/g, '/'); const binary = atob(padded); diff --git a/platform/packages/proxy/src/proxy-token.ts b/platform/packages/proxy/src/proxy-token.ts new file mode 100644 index 000000000..874d31fd0 --- /dev/null +++ b/platform/packages/proxy/src/proxy-token.ts @@ -0,0 +1,58 @@ +import { + createProxyTokenStateCache, + deriveProxyToken, + ProxyTokenStateError, + sendGenerations, + verifyProxyToken, +} from '@voidcloud/api/proxy-token'; +import type { Env } from './types.ts'; + +// One floor cache per isolate, shared by every request it serves. +const stateCache = createProxyTokenStateCache(); + +/** @internal Exposed for testing only. */ +export function clearProxyTokenStateCache() { + stateCache.clear(); +} + +/** + * Checks a tenant's project token against its accept floor. A floor that + * cannot be read or is malformed is `'unavailable'`, never accepted. + */ +export async function verifyTenantProxyToken( + env: Pick, + projectId: string, + token: string, +): Promise<'ok' | 'invalid' | 'unavailable'> { + try { + const valid = await verifyProxyToken({ + jwtSecret: env.JWT_SECRET, + projectId, + token, + readState: () => stateCache.read(env.ROUTING, projectId), + }); + return valid ? 'ok' : 'invalid'; + } catch (error) { + if (error instanceof ProxyTokenStateError) { + console.error('[proxy-token] state unavailable', { projectId, error }); + return 'unavailable'; + } + throw error; + } +} + +/** + * Tokens to present to a tenant Worker, in order. Without the issued + * generation, sends for the cached floor, which throws `ProxyTokenStateError` + * when the floor cannot be read. + */ +export async function tenantSendTokens( + env: Pick, + projectId: string, + issueGen: number | undefined, +): Promise> { + const gen = issueGen ?? (await stateCache.read(env.ROUTING, projectId)).gen; + return Promise.all( + sendGenerations(gen).map((sendGen) => deriveProxyToken(env.JWT_SECRET, projectId, sendGen)), + ); +} diff --git a/platform/packages/proxy/test/unit/d1.test.ts b/platform/packages/proxy/test/unit/d1.test.ts index 70915d747..62b66686b 100644 --- a/platform/packages/proxy/test/unit/d1.test.ts +++ b/platform/packages/proxy/test/unit/d1.test.ts @@ -1,5 +1,6 @@ -import { describe, it, expect, vi, beforeAll, afterEach } from 'vite-plus/test'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import worker from '../../src/index.ts'; +import { clearProxyTokenStateCache } from '../../src/proxy-token.ts'; const JWT_SECRET = 'test-secret-key'; @@ -100,6 +101,8 @@ describe('D1 proxy endpoints', () => { token = await signAiToken('proj_000000000001', JWT_SECRET); }); + beforeEach(clearProxyTokenStateCache); + afterEach(() => { globalThis.fetch = originalFetch; vi.restoreAllMocks(); diff --git a/platform/packages/proxy/test/unit/email-custom-domain.test.ts b/platform/packages/proxy/test/unit/email-custom-domain.test.ts index 024bb10c5..6d3c26b09 100644 --- a/platform/packages/proxy/test/unit/email-custom-domain.test.ts +++ b/platform/packages/proxy/test/unit/email-custom-domain.test.ts @@ -16,6 +16,7 @@ vi.mock('cloudflare:email', () => ({ const worker = (await import('../../src/index.ts')).default; const { requestSendRaw } = await import('../../src/email.ts'); +const { clearProxyTokenStateCache } = await import('../../src/proxy-token.ts'); const JWT_SECRET = 'test-secret-key'; const EMAIL_SIGNING_SECRET = 'test-email-signing-key-independent'; @@ -207,6 +208,7 @@ function stubCloudflare(handler?: (call: { url: string; init: RequestInit }) => } beforeEach(() => { + clearProxyTokenStateCache(); vi.spyOn(console, 'error').mockImplementation(() => {}); vi.spyOn(console, 'warn').mockImplementation(() => {}); vi.spyOn(console, 'log').mockImplementation(() => {}); diff --git a/platform/packages/proxy/test/unit/email.test.ts b/platform/packages/proxy/test/unit/email.test.ts index 772338052..e7656b0be 100644 --- a/platform/packages/proxy/test/unit/email.test.ts +++ b/platform/packages/proxy/test/unit/email.test.ts @@ -18,6 +18,7 @@ vi.mock('cloudflare:email', () => ({ const worker = (await import('../../src/index.ts')).default; const { readMimeFromAddrSpec } = await import('../../src/email.ts'); +const { clearProxyTokenStateCache } = await import('../../src/proxy-token.ts'); // The framework's own MIME builder — the ONLY legitimate producer of the // `mimeBase64` a tenant posts — so the proxy's From pin is checked against // every `From:` shape it really renders, not against hand-typed lookalikes. @@ -210,6 +211,14 @@ function happyPathKv(extra: Record = {}) { }); } +/** KV keys read outside the project token check. */ +function nonAuthKvReads(kv: MockKv): Array { + return kv.get.mock.calls.map((c) => c[0]).filter((key) => !key.startsWith('proxy-token:')); +} + +// The floor cache lives for the whole isolate; start every test cold. +beforeEach(clearProxyTokenStateCache); + // --- Inbound helpers --- function streamOf(bytes: Uint8Array): ReadableStream { @@ -462,7 +471,7 @@ describe('POST /email/send — platform capability', () => { message: 'This Void platform does not enable email.', }); expect((env.SEND_EMAIL as { send: ReturnType }).send).not.toHaveBeenCalled(); - expect(kv.get).not.toHaveBeenCalled(); + expect(nonAuthKvReads(kv)).toEqual([]); expect(kv.put).not.toHaveBeenCalled(); }); @@ -736,7 +745,7 @@ describe('POST /email/send — request validation', () => { // Refused above the normalization loop, so not one per-recipient // delivery object was built. expect(body.deliveries).toBeUndefined(); - expect(kv.get).not.toHaveBeenCalled(); + expect(nonAuthKvReads(kv)).toEqual([]); expect(kv.put).not.toHaveBeenCalled(); expect(sender.calls).toHaveLength(0); }); @@ -776,7 +785,7 @@ describe('POST /email/send — request validation', () => { const body = await readSend(res); expect(body.error?.code).toBe('INVALID_TO'); expect(body.deliveries).toBeUndefined(); - expect(kv.get).not.toHaveBeenCalled(); + expect(nonAuthKvReads(kv)).toEqual([]); expect(kv.put).not.toHaveBeenCalled(); expect(sender.calls).toHaveLength(0); }); @@ -806,7 +815,7 @@ describe('POST /email/send — request validation', () => { expect(body.error?.code).toBe('INVALID_FROM'); expect(body.error?.message).not.toContain(tag); expect(body.deliveries).toBeUndefined(); - expect(kv.get).not.toHaveBeenCalled(); + expect(nonAuthKvReads(kv)).toEqual([]); expect(kv.put).not.toHaveBeenCalled(); expect(sender.calls).toHaveLength(0); }); @@ -852,7 +861,7 @@ describe('POST /email/send — MIME From pin (what the recipient sees)', () => { expect(body.error?.code).toBe('INVALID_FROM'); expect(sender.calls).toHaveLength(0); // Refused before the suspension, allowlist and quota reads. - expect(kv.get.mock.calls.map((c) => c[0])).toEqual([]); + expect(nonAuthKvReads(kv)).toEqual([]); expect(kv.put).not.toHaveBeenCalled(); }); @@ -1115,7 +1124,7 @@ describe('POST /email/send — MIME From pin (what the recipient sees)', () => { expect(body.error?.code).toBe('INVALID_FROM'); expect(body.error?.message).toContain('does not match the envelope sender'); expect(sender.calls).toHaveLength(0); - expect(kv.get.mock.calls.map((c) => c[0])).toEqual([]); + expect(nonAuthKvReads(kv)).toEqual([]); }); it.each([ diff --git a/platform/packages/proxy/test/unit/kv.test.ts b/platform/packages/proxy/test/unit/kv.test.ts index b9be01f60..023f2efa9 100644 --- a/platform/packages/proxy/test/unit/kv.test.ts +++ b/platform/packages/proxy/test/unit/kv.test.ts @@ -1,5 +1,6 @@ -import { describe, it, expect, vi, beforeAll, afterEach } from 'vite-plus/test'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import worker from '../../src/index.ts'; +import { clearProxyTokenStateCache } from '../../src/proxy-token.ts'; const JWT_SECRET = 'test-secret-key'; @@ -89,6 +90,8 @@ describe('KV proxy endpoints', () => { token = await signAiToken('proj_000000000001', JWT_SECRET); }); + beforeEach(clearProxyTokenStateCache); + afterEach(() => { globalThis.fetch = originalFetch; vi.restoreAllMocks(); diff --git a/platform/packages/proxy/test/unit/proxy.test.ts b/platform/packages/proxy/test/unit/proxy.test.ts index 59cf0cc60..5af5b4699 100644 --- a/platform/packages/proxy/test/unit/proxy.test.ts +++ b/platform/packages/proxy/test/unit/proxy.test.ts @@ -1,7 +1,9 @@ import { signEmailReceipt } from '@void/platform/email-protocol'; -import { describe, it, expect, vi, beforeAll, afterEach } from 'vite-plus/test'; +import { deriveProxyToken } from '@voidcloud/api/proxy-token'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import worker from '../../src/index.ts'; import { PLATFORM_DISABLED_KEY } from '../../src/platform-lifecycle.ts'; +import { clearProxyTokenStateCache } from '../../src/proxy-token.ts'; import { readRequestJson } from '../../src/request-body.ts'; const JWT_SECRET = 'test-secret-key'; @@ -177,6 +179,8 @@ describe('proxy worker', () => { let validToken: string; const aiTokens: Record = {}; + beforeEach(clearProxyTokenStateCache); + afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); @@ -1124,6 +1128,140 @@ describe('proxy worker', () => { }); }); + describe('project token generations', () => { + const projectId = 'proj_000000000001'; + const floorKey = `proxy-token:${projectId}`; + + function floorReads(env: ReturnType): number { + return env.ROUTING.get.mock.calls.filter((call) => String(call[0]).startsWith('proxy-token:')) + .length; + } + + function aiRun(env: ReturnType, aiToken: string) { + return worker.fetch( + post('/ai/run', { model: 'test-model', inputs: {}, projectId, aiToken }), + env, + ); + } + + beforeEach(() => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('accepts the legacy and v2 tokens while no floor is set', async () => { + for (const gen of [0, 1]) { + clearProxyTokenStateCache(); + const env = createEnv(); + const res = await aiRun(env, await deriveProxyToken(JWT_SECRET, projectId, gen)); + expect(res.status).toBe(200); + expect(env.AI.run).toHaveBeenCalledOnce(); + } + }); + + it('accepts generations at or above the floor and rejects the legacy token', async () => { + for (const gen of [1, 2]) { + clearProxyTokenStateCache(); + const res = await aiRun( + createEnv({}, { [floorKey]: '{"gen":1}' }), + await deriveProxyToken(JWT_SECRET, projectId, gen), + ); + expect(res.status).toBe(200); + } + + clearProxyTokenStateCache(); + const env = createEnv({}, { [floorKey]: '{"gen":1}' }); + const res = await aiRun(env, aiTokens[projectId]); + expect(res.status).toBe(403); + expect(await res.json()).toEqual({ error: 'invalid project token' }); + expect(env.AI.run).not.toHaveBeenCalled(); + }); + + it('rejects a wrong MAC without reading the floor', async () => { + const other = await deriveProxyToken(JWT_SECRET, 'proj_000000000042', 1); + const env = createEnv(); + const res = await aiRun(env, other); + expect(res.status).toBe(403); + expect(await res.json()).toEqual({ error: 'invalid project token' }); + expect(floorReads(env)).toBe(0); + }); + + it('fails closed with 503 when the floor is malformed', async () => { + const env = createEnv({}, { [floorKey]: 'not json' }); + const res = await aiRun(env, aiTokens[projectId]); + expect(res.status).toBe(503); + expect(await res.json()).toEqual({ error: 'project token state unavailable' }); + expect(env.AI.run).not.toHaveBeenCalled(); + }); + + it('reads the floor once for repeated calls in the same isolate', async () => { + const env = createEnv(); + expect((await aiRun(env, aiTokens[projectId])).status).toBe(200); + expect((await aiRun(env, aiTokens[projectId])).status).toBe(200); + expect(floorReads(env)).toBe(1); + }); + + describe('/email/verify-receipt', () => { + const claims = { + version: 1 as const, + receiptId: 'a'.repeat(64), + projectId, + deploymentId: 'deploy_1', + recipient: 'support@example.com', + expiresAt: Date.now() + 60_000, + }; + + async function verifyReceipt( + env: ReturnType, + headers: Record, + ) { + return worker.fetch( + post( + '/email/verify-receipt', + { ...claims, receipt: await signEmailReceipt(claims, EMAIL_SIGNING_SECRET) }, + headers, + ), + env, + ); + } + + it('accepts a token at the floor and rejects the legacy token below it', async () => { + const accepted = await verifyReceipt(createEnv(undefined, { [floorKey]: '{"gen":1}' }), { + 'x-void-internal': await deriveProxyToken(JWT_SECRET, projectId, 1), + }); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toEqual({ valid: true, receiptId: claims.receiptId }); + + clearProxyTokenStateCache(); + const rejected = await verifyReceipt(createEnv(undefined, { [floorKey]: '{"gen":1}' }), { + 'x-void-internal': aiTokens[projectId], + }); + expect(rejected.status).toBe(403); + expect(await rejected.json()).toEqual({ valid: false }); + }); + + it('fails closed with 503 when the floor cannot be read', async () => { + const env = createEnv(); + env.ROUTING.get.mockRejectedValue(new Error('kv unavailable')); + const res = await verifyReceipt(env, { 'x-void-internal': aiTokens[projectId] }); + expect(res.status).toBe(503); + expect(await res.json()).toEqual({ valid: false }); + }); + + it('rejects a missing token without reading the floor', async () => { + const env = createEnv(); + const res = await verifyReceipt(env, {}); + expect(res.status).toBe(403); + expect(await res.json()).toEqual({ valid: false }); + expect(floorReads(env)).toBe(0); + }); + }); + }); + describe('AI usage limit (KV block)', () => { it('returns 429 when ai-block key exists (service binding path)', async () => { const env = createEnv({}, { 'ai-block:proj_000000000001': '1' }); diff --git a/platform/packages/proxy/test/unit/r2.test.ts b/platform/packages/proxy/test/unit/r2.test.ts index 004314067..c2962aea4 100644 --- a/platform/packages/proxy/test/unit/r2.test.ts +++ b/platform/packages/proxy/test/unit/r2.test.ts @@ -1,5 +1,6 @@ -import { describe, it, expect, vi, beforeAll, afterEach } from 'vite-plus/test'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import worker from '../../src/index.ts'; +import { clearProxyTokenStateCache } from '../../src/proxy-token.ts'; const JWT_SECRET = 'test-secret-key'; @@ -89,6 +90,8 @@ describe('R2 proxy endpoints', () => { token = await signAiToken('proj_000000000001', JWT_SECRET); }); + beforeEach(clearProxyTokenStateCache); + afterEach(() => { globalThis.fetch = originalFetch; vi.restoreAllMocks(); From 8f0d25c9993e086ed6d0f40b8379f8396652b8e6 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 01:50:54 +0800 Subject: [PATCH 03/12] Send the issued proxy token generation to tenant Workers The proxy now picks the token it presents to a tenant Worker from the project's token generation. When /internal/resolve-worker returns `proxyTokenGen`, the proxy sends that generation's token. Without it, the proxy sends the token for the project's cached accept floor, so tenant Workers keep working when the API does not report a generation. If the Worker rejects the token with its pre-handler 403 `unauthorized` response, the proxy retries once with the previous generation's token and the same body. It never retries after a thrown fetch or any other response. If the floor cannot be read, a write fails with 502 and a read takes its REST fallback, without dispatching. With no floor written, every project still receives the legacy token in a single request. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/proxy/src/dispatch.ts | 70 ++++- .../packages/proxy/test/unit/dispatch.test.ts | 261 +++++++++++++++++- 2 files changed, 320 insertions(+), 11 deletions(-) diff --git a/platform/packages/proxy/src/dispatch.ts b/platform/packages/proxy/src/dispatch.ts index 71b44653a..e92c61034 100644 --- a/platform/packages/proxy/src/dispatch.ts +++ b/platform/packages/proxy/src/dispatch.ts @@ -1,7 +1,14 @@ +import { ProxyTokenStateError } from '@voidcloud/api/proxy-token'; import { hmacSign } from './hmac.ts'; +import { tenantSendTokens } from './proxy-token.ts'; import type { Env } from './types.ts'; -type WorkerResolution = Readonly<{ workerName: string; deploymentId: string }>; +type WorkerResolution = Readonly<{ + workerName: string; + deploymentId: string; + /** Generation of the token the platform issues to this project's Workers. */ + proxyTokenGen?: number; +}>; type DispatchOptions = Readonly<{ /** Read-only operations may retry through their REST fallback after a transport failure. */ @@ -35,11 +42,35 @@ async function resolveWorker( throw new Error(`worker resolution failed with status ${res.status}`); } - const data = await res.json>(); + const data = await res.json>>(); if (typeof data.workerName !== 'string' || typeof data.deploymentId !== 'string') { throw new Error('worker resolution returned an invalid response'); } - return { workerName: data.workerName, deploymentId: data.deploymentId }; + const resolution = { workerName: data.workerName, deploymentId: data.deploymentId }; + return isGeneration(data.proxyTokenGen) + ? { ...resolution, proxyTokenGen: data.proxyTokenGen } + : resolution; +} + +function isGeneration(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} + +/** The binding handler's reject for a token it does not hold, sent before any handler runs. */ +async function isTokenRejection(res: Response): Promise { + if (res.status !== 403) { + return false; + } + try { + const data = await res.clone().json(); + return ( + typeof data === 'object' && + data !== null && + (data as { error?: unknown }).error === 'unauthorized' + ); + } catch { + return false; + } } /** @@ -68,7 +99,19 @@ export async function dispatchToWorker( return null; } - const proxyToken = await hmacSign(projectId, env.JWT_SECRET); + let tokens: ReadonlyArray; + try { + tokens = await tenantSendTokens(env, projectId, resolution.proxyTokenGen); + } catch (error) { + if (!(error instanceof ProxyTokenStateError)) { + throw error; + } + console.error('[proxy-token] state unavailable', { projectId, error }); + return options.fallbackOnDispatchError + ? null + : Response.json({ error: 'project token state unavailable' }, { status: 502 }); + } + let worker: Fetcher; try { worker = env.DISPATCH.get(resolution.workerName); @@ -76,16 +119,27 @@ export async function dispatchToWorker( return null; } - try { - const res = await worker.fetch(`https://worker/__void${path}`, { + const payload = JSON.stringify(body); + const send = (token: string) => + worker.fetch(`https://worker/__void${path}`, { method: 'POST', headers: { 'content-type': 'application/json', - 'x-void-internal': proxyToken, + 'x-void-internal': token, }, - body: JSON.stringify(body), + body: payload, }); + try { + let res = await send(tokens[0]); + // While a token rotation is in progress the Worker may still hold the + // previous token. Its reject comes before any handler runs, so a second + // attempt cannot repeat a write. + if (tokens.length > 1 && (await isTokenRejection(res))) { + await res.body?.cancel(); + res = await send(tokens[1]); + } + // 404 means old deploy without /__void handler → fall back if (res.status === 404) { return null; diff --git a/platform/packages/proxy/test/unit/dispatch.test.ts b/platform/packages/proxy/test/unit/dispatch.test.ts index ee8e08073..be502a8ff 100644 --- a/platform/packages/proxy/test/unit/dispatch.test.ts +++ b/platform/packages/proxy/test/unit/dispatch.test.ts @@ -1,6 +1,10 @@ -import { describe, it, expect, vi, beforeAll, afterEach } from 'vite-plus/test'; +import { deriveProxyToken } from '@voidcloud/api/proxy-token'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import { createBindingHandler } from '../../../../../packages/void/src/runtime/remote/binding-handler.ts'; +import { dispatchToWorker } from '../../src/dispatch.ts'; import worker from '../../src/index.ts'; +import { clearProxyTokenStateCache } from '../../src/proxy-token.ts'; +import type { Env } from '../../src/types.ts'; const JWT_SECRET = 'test-secret-key'; @@ -159,6 +163,8 @@ describe('WfP dispatch path', () => { token = await signAiToken('proj_000000000001', JWT_SECRET); }); + beforeEach(clearProxyTokenStateCache); + afterEach(() => { vi.restoreAllMocks(); }); @@ -609,8 +615,257 @@ describe('WfP dispatch path', () => { env, ); - expect(capturedHeaders['x-void-internal']).toBeDefined(); - expect(capturedHeaders['x-void-internal'].length).toBeGreaterThan(0); + expect(capturedHeaders['x-void-internal']).toBe(token); + }); + }); +}); + +describe('proxy token generations', () => { + const PROJECT_ID = 'proj_000000000001'; + const KV_BODY = { binding: 'CACHE', key: 'greeting', type: 'text' }; + + afterEach(() => { + vi.restoreAllMocks(); + }); + beforeEach(clearProxyTokenStateCache); + + function tok(gen: number): Promise { + return deriveProxyToken(JWT_SECRET, PROJECT_ID, gen); + } + + type Sent = { token: string | null; body: unknown }; + + /** + * `resolution` is the resolve-worker response body. `floor` is the raw + * ROUTING value of the project's accept floor. `respond` answers each + * dispatched request; by default a real binding handler holding `held`. + */ + function setup( + options: Readonly<{ + resolution?: Record; + floor?: string | null; + held?: string; + respond?: (token: string | null) => Response | Promise; + }>, + ) { + const sent: Array = []; + const handler = createBindingHandler({ + CACHE: { get: vi.fn(async () => 'hello') }, + __VOID_PROXY_TOKEN: options.held, + }); + const routingGet = vi.fn(async (key: string) => + key === `proxy-token:${PROJECT_ID}` ? (options.floor ?? null) : null, + ); + const env = { + JWT_SECRET, + ROUTING: { get: routingGet }, + API: { + fetch: vi.fn(async () => + Response.json( + options.resolution ?? { workerName: 'user-my-app', deploymentId: 'dpl_active' }, + ), + ), + }, + DISPATCH: { + get: vi.fn(() => ({ + fetch: vi.fn(async (url: string, init: RequestInit) => { + const sentToken = new Headers(init.headers).get('x-void-internal'); + sent.push({ token: sentToken, body: JSON.parse(init.body as string) }); + if (options.respond) { + return options.respond(sentToken); + } + const target = new URL(url); + target.pathname = target.pathname.replace(/^\/__void/, ''); + return handler.fetch(new Request(target, init)); + }), + })), + }, + } as unknown as Env; + const floorReads = () => + routingGet.mock.calls.filter(([key]) => key.startsWith('proxy-token:')).length; + return { env, sent, floorReads }; + } + + it('sends the legacy token once when there is no issued generation and no floor', async () => { + const legacy = await tok(0); + const { env, sent } = setup({ held: legacy }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(await res?.json()).toBe('hello'); + expect(sent.map((s) => s.token)).toEqual([legacy]); + }); + + it('sends the floor token, then the previous one, when there is no issued generation', async () => { + const { env, sent } = setup({ floor: '{"gen":2}', held: await tok(1) }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(sent.map((s) => s.token)).toEqual([await tok(2), await tok(1)]); + }); + + it('sends one request when the Worker holds the floor token', async () => { + const { env, sent } = setup({ floor: '{"gen":1}', held: await tok(1) }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(sent.map((s) => s.token)).toEqual([await tok(1)]); + }); + + it('retries generation 1 with the legacy token and the same body', async () => { + const legacy = await tok(0); + const { env, sent, floorReads } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + held: legacy, + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(await res?.json()).toBe('hello'); + expect(sent.map((s) => s.token)).toEqual([await tok(1), legacy]); + expect(sent[1].body).toEqual(sent[0].body); + expect(sent[0].body).toEqual(KV_BODY); + expect(floorReads()).toBe(0); + }); + + it.each([ + { held: 1, tokens: [2, 1] }, + { held: 2, tokens: [2] }, + ])('sends issued generation 2 to a Worker holding $held', async ({ held, tokens }) => { + const { env, sent, floorReads } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, + held: await tok(held), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(sent.map((s) => s.token)).toEqual(await Promise.all(tokens.map(tok))); + expect(floorReads()).toBe(0); + }); + + it('does not retry a 403 with another error', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => Response.json({ error: 'forbidden' }, { status: 403 }), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(403); + expect(await res?.json()).toEqual({ error: 'forbidden' }); + expect(sent).toHaveLength(1); + }); + + it('does not retry a 403 whose body is not JSON', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => new Response('denied', { status: 403 }), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(403); + expect(await res?.text()).toBe('denied'); + expect(sent).toHaveLength(1); + }); + + it('returns the second reject when the Worker holds neither token', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, + held: await tok(0), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(403); + expect(await res?.json()).toEqual({ error: 'unauthorized' }); + expect(sent.map((s) => s.token)).toEqual([await tok(2), await tok(1)]); + }); + + it('does not retry after a thrown fetch', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => { + throw new Error('response lost after write'); + }, + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/put', KV_BODY); + + expect(res?.status).toBe(502); + expect(sent).toHaveLength(1); + }); + + it('returns null for a 404', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => new Response('Not Found', { status: 404 }), + }); + + expect(await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY)).toBeNull(); + expect(sent).toHaveLength(1); + }); + + it.each([-1, '1', 1.5])('reads the floor when the issued generation is %j', async (gen) => { + const { env, sent, floorReads } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: gen }, + floor: '{"gen":2}', + held: await tok(2), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(sent.map((s) => s.token)).toEqual([await tok(2)]); + expect(floorReads()).toBe(1); + }); + + describe('when the floor cannot be read', () => { + function failingSetup() { + const ctx = setup({ held: 'unused' }); + vi.mocked(ctx.env.ROUTING.get).mockRejectedValue(new Error('KV unavailable')); + vi.spyOn(console, 'error').mockImplementation(() => {}); + return ctx; + } + + it('fails a write with 502 without dispatching', async () => { + const ctx = failingSetup(); + + const res = await dispatchToWorker(ctx.env, PROJECT_ID, '/kv/put', KV_BODY); + + expect(res?.status).toBe(502); + expect(await res?.json()).toEqual({ error: 'project token state unavailable' }); + expect(ctx.env.DISPATCH.get).not.toHaveBeenCalled(); + expect(ctx.sent).toHaveLength(0); + }); + + it('lets a read fall back to the REST path', async () => { + const ctx = failingSetup(); + + const res = await dispatchToWorker(ctx.env, PROJECT_ID, '/kv/get', KV_BODY, { + fallbackOnDispatchError: true, + }); + + expect(res).toBeNull(); + expect(ctx.sent).toHaveLength(0); + }); + + it('does not need the floor when the issued generation is known', async () => { + const ctx = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + held: await tok(1), + }); + vi.mocked(ctx.env.ROUTING.get).mockRejectedValue(new Error('KV unavailable')); + + const res = await dispatchToWorker(ctx.env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(ctx.sent.map((s) => s.token)).toEqual([await tok(1)]); }); }); }); From ecff301a2431b3649b366d7ba645f5a62e72b112 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 01:55:35 +0800 Subject: [PATCH 04/12] Sign proxy-to-API metadata calls with a proxy-only key /internal/resolve-worker, /internal/resolve-resource and /internal/verify-prerender-build accepted the same HMAC that tenant Workers hold as their project token, so a tenant could call them directly on the public API host. The proxy now also sends `x-void-proxy-internal`, an HMAC bound to the route and project under a key derived from JWT_SECRET with HKDF. It never leaves the platform. The API accepts that header first and never falls through when it is present but wrong. A request without it is checked like a tenant project token: the MAC, then the project's accept floor. A legacy token therefore stops opening these routes as soon as the floor rises past it, the same moment it stops working at the proxy. The proxy keeps sending the legacy `x-void-internal` header so an API that predates the new credential keeps working while the two deploy. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../packages/api/src/internal-proxy-auth.ts | 42 ++++++ .../src/routes/internal-resolve-resource.ts | 17 +-- .../api/src/routes/internal-resolve-worker.ts | 17 +-- .../routes/internal-verify-prerender-build.ts | 15 +- .../api-internal-project-token.test.ts | 141 +++++++++++++++++- .../api/test/integration/api-projects.test.ts | 39 ++++- .../integration/api-resolve-worker.test.ts | 6 +- platform/packages/proxy/src/dispatch.ts | 15 +- platform/packages/proxy/src/resolve.ts | 19 ++- .../packages/proxy/test/unit/dispatch.test.ts | 19 ++- .../packages/proxy/test/unit/resolve.test.ts | 51 +++++-- 11 files changed, 317 insertions(+), 64 deletions(-) diff --git a/platform/packages/api/src/internal-proxy-auth.ts b/platform/packages/api/src/internal-proxy-auth.ts index 3151704d7..6b14b1acb 100644 --- a/platform/packages/api/src/internal-proxy-auth.ts +++ b/platform/packages/api/src/internal-proxy-auth.ts @@ -1,8 +1,13 @@ +import type { Context } from 'hono'; import type { ApiEnv } from './env.ts'; +import { isProjectId } from './id.ts'; import { + PROXY_INTERNAL_AUTH_HEADER, PROXY_TOKEN_KV_CACHE_TTL_SECONDS, + type ProxyInternalRoute, ProxyTokenStateError, readProxyTokenState, + verifyProxyInternalAuth, verifyProxyToken, } from './proxy-token.ts'; @@ -35,3 +40,40 @@ export async function checkTenantProxyToken( throw error; } } + +/** + * Authenticates the proxy on an internal metadata route. The proxy signs + * `x-void-proxy-internal` with its own key, bound to the route and project. + * A request without that header is checked like a tenant project token in + * `x-void-internal`, so a token below the project's accept floor is refused + * here as it is at the proxy. + * + * Returns null when the call may proceed, else the error response. + */ +export async function requireProxyInternalAuth( + context: Context, + route: ProxyInternalRoute, + projectId: string, +): Promise { + const proxyAuth = context.req.header(PROXY_INTERNAL_AUTH_HEADER); + const token = context.req.header('x-void-internal'); + if (!proxyAuth && !token) { + return context.json({ error: 'missing internal auth' }, 401); + } + if (!isProjectId(projectId)) { + return context.json({ error: 'invalid internal auth' }, 403); + } + if (proxyAuth) { + return (await verifyProxyInternalAuth(context.env.JWT_SECRET, route, projectId, proxyAuth)) + ? null + : context.json({ error: 'invalid internal auth' }, 403); + } + const tokenCheck = await checkTenantProxyToken(context.env, projectId, token ?? ''); + if (tokenCheck === 'unavailable') { + return context.json({ error: 'project token state unavailable' }, 503); + } + if (tokenCheck === 'invalid') { + return context.json({ error: 'invalid internal auth' }, 403); + } + return null; +} diff --git a/platform/packages/api/src/routes/internal-resolve-resource.ts b/platform/packages/api/src/routes/internal-resolve-resource.ts index 8bc09d910..02c9a1437 100644 --- a/platform/packages/api/src/routes/internal-resolve-resource.ts +++ b/platform/packages/api/src/routes/internal-resolve-resource.ts @@ -1,14 +1,13 @@ import { Hono } from 'hono'; -import { hmacVerifyState } from '../auth/hmac.ts'; import { getResourceByBinding } from '../db.ts'; import type { ApiEnv } from '../env.ts'; -import { isProjectId } from '../id.ts'; +import { requireProxyInternalAuth } from '../internal-proxy-auth.ts'; const app = new Hono(); /** * Internal resource resolution endpoint — called by the proxy via service binding. - * Authenticated via HMAC token in X-Void-Internal header (no JWT/userId). + * Authenticated by the proxy's own credential (see requireProxyInternalAuth), no JWT/userId. * * POST /internal/resolve-resource/:projectId * Body: { "type": "d1" | "kv" | "r2" | "queue", "binding": "DB" } @@ -17,15 +16,9 @@ const app = new Hono(); app.post('/:projectId', async (context) => { const projectId = context.req.param('projectId'); - const token = context.req.header('x-void-internal'); - if (!token) { - return context.json({ error: 'missing internal auth' }, 401); - } - if ( - !isProjectId(projectId) || - !(await hmacVerifyState(projectId, token, context.env.JWT_SECRET)) - ) { - return context.json({ error: 'invalid internal auth' }, 403); + const denied = await requireProxyInternalAuth(context, 'resolve-resource', projectId); + if (denied) { + return denied; } const { type, binding } = await context.req.json<{ diff --git a/platform/packages/api/src/routes/internal-resolve-worker.ts b/platform/packages/api/src/routes/internal-resolve-worker.ts index fca338a80..5e00cbaa5 100644 --- a/platform/packages/api/src/routes/internal-resolve-worker.ts +++ b/platform/packages/api/src/routes/internal-resolve-worker.ts @@ -1,8 +1,7 @@ import { Hono } from 'hono'; -import { hmacVerifyState } from '../auth/hmac.ts'; import { getDeployment, getProject } from '../db.ts'; import type { ApiEnv } from '../env.ts'; -import { isProjectId } from '../id.ts'; +import { requireProxyInternalAuth } from '../internal-proxy-auth.ts'; import { getProjectDeploymentWorkerName } from '../naming.ts'; const app = new Hono(); @@ -10,7 +9,7 @@ const app = new Hono(); /** * Internal worker name resolution — called by the proxy via service binding. * Returns the active versioned WfP worker name for a given projectId. - * Authenticated via HMAC token in X-Void-Internal header (same as resolve-resource). + * Authenticated by the proxy's own credential (see requireProxyInternalAuth). * * POST /internal/resolve-worker/:projectId * Returns: { "workerName": "user--", "deploymentId": "..." } @@ -19,15 +18,9 @@ const app = new Hono(); app.post('/:projectId', async (context) => { const projectId = context.req.param('projectId'); - const token = context.req.header('x-void-internal'); - if (!token) { - return context.json({ error: 'missing internal auth' }, 401); - } - if ( - !isProjectId(projectId) || - !(await hmacVerifyState(projectId, token, context.env.JWT_SECRET)) - ) { - return context.json({ error: 'invalid internal auth' }, 403); + const denied = await requireProxyInternalAuth(context, 'resolve-worker', projectId); + if (denied) { + return denied; } const project = await getProject(context.get('db'), projectId); diff --git a/platform/packages/api/src/routes/internal-verify-prerender-build.ts b/platform/packages/api/src/routes/internal-verify-prerender-build.ts index c4aac34a2..9047f37a3 100644 --- a/platform/packages/api/src/routes/internal-verify-prerender-build.ts +++ b/platform/packages/api/src/routes/internal-verify-prerender-build.ts @@ -1,9 +1,8 @@ import { and, eq, gt, isNull } from 'drizzle-orm'; import { Hono } from 'hono'; -import { hmacVerifyState } from '../auth/hmac.ts'; import { ACTIVE_BUILD_STATUSES, getBuild, getUser } from '../db.ts'; import type { ApiEnv } from '../env.ts'; -import { isProjectId } from '../id.ts'; +import { requireProxyInternalAuth } from '../internal-proxy-auth.ts'; import { ProjectAccessDeniedError, ProjectNotFoundError, @@ -20,15 +19,9 @@ const app = new Hono(); */ app.post('/:projectId', async (context) => { const projectId = context.req.param('projectId'); - const token = context.req.header('x-void-internal'); - if (!token) { - return context.json({ error: 'missing internal auth' }, 401); - } - if ( - !isProjectId(projectId) || - !(await hmacVerifyState(projectId, token, context.env.JWT_SECRET)) - ) { - return context.json({ error: 'invalid internal auth' }, 403); + const denied = await requireProxyInternalAuth(context, 'verify-prerender-build', projectId); + if (denied) { + return denied; } const { userId, buildId, credentialId } = await context.req.json<{ diff --git a/platform/packages/api/test/integration/api-internal-project-token.test.ts b/platform/packages/api/test/integration/api-internal-project-token.test.ts index 5879bb547..6d7c92259 100644 --- a/platform/packages/api/test/integration/api-internal-project-token.test.ts +++ b/platform/packages/api/test/integration/api-internal-project-token.test.ts @@ -12,7 +12,11 @@ import { hmacSignState } from '../../src/auth/hmac.ts'; import { signJwt } from '../../src/auth/jwt.ts'; import { createResource } from '../../src/db.ts'; import app from '../../src/index.ts'; -import { deriveProxyToken } from '../../src/proxy-token.ts'; +import { + deriveProxyInternalAuth, + deriveProxyToken, + type ProxyInternalRoute, +} from '../../src/proxy-token.ts'; import { clearTables, createTestDeployment, @@ -58,13 +62,14 @@ async function loginJwtAsProjectToken(): Promise<{ projectId: string; token: str async function internalRequest( route: string, projectId: string, - token: string, + token: string | Record, body: unknown = {}, ): Promise { + const auth = typeof token === 'string' ? { 'x-void-internal': token } : token; return app.fetch( new Request(`http://localhost/internal/${route}/${projectId}`, { method: 'POST', - headers: { 'content-type': 'application/json', 'x-void-internal': token }, + headers: { 'content-type': 'application/json', ...auth }, body: JSON.stringify(body), }), env, @@ -259,3 +264,133 @@ describe('project token generations', () => { expect(await response.json()).toEqual({ error: 'project token state unavailable' }); }); }); + +describe('proxy credential on the metadata routes', () => { + const routes: ReadonlyArray = [ + 'resolve-worker', + 'resolve-resource', + 'verify-prerender-build', + ]; + + async function liveProject() { + const user = await createTestUser(env.db); + const project = await createTestProject(env.db, user.id); + const deployment = await createTestDeployment(env.db, project.id, 'live'); + await createResource(env.db, project.id, 'kv', 'cf-kv-id', 'CACHE'); + const bodies: Record = { + 'resolve-worker': {}, + 'resolve-resource': { type: 'kv', binding: 'CACHE' }, + 'verify-prerender-build': { userId: user.id }, + }; + const expected: Record = { + 'resolve-worker': { + workerName: `user-${project.slug}-${deployment.id}`, + deploymentId: deployment.id, + }, + 'resolve-resource': { cf_id: 'cf-kv-id' }, + 'verify-prerender-build': { ok: true }, + }; + return { project, bodies, expected }; + } + + function floorReads(get: { mock: { calls: Array> } }) { + return get.mock.calls.filter(([key]) => String(key).startsWith('proxy-token:')); + } + + beforeEach(() => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + it.each(routes)( + 'accepts the proxy credential at /internal/%s without a floor read', + async (route) => { + const { project, bodies, expected } = await liveProject(); + const get = vi.spyOn(env.ROUTING, 'get'); + const response = await internalRequest( + route, + project.id, + { 'x-void-proxy-internal': await deriveProxyInternalAuth(JWT_SECRET, route, project.id) }, + bodies[route], + ); + expect(response.status).toBe(200); + expect(await response.json()).toEqual(expected[route]); + expect(floorReads(get)).toEqual([]); + }, + ); + + it.each(routes)( + 'rejects a proxy credential made for another route at /internal/%s', + async (route) => { + const { project, bodies } = await liveProject(); + const other = routes.find((candidate) => candidate !== route)!; + const response = await internalRequest( + route, + project.id, + { 'x-void-proxy-internal': await deriveProxyInternalAuth(JWT_SECRET, other, project.id) }, + bodies[route], + ); + expect(response.status).toBe(403); + expect(await response.json()).toEqual({ error: 'invalid internal auth' }); + }, + ); + + it.each(routes)( + 'does not fall back to a valid project token after a bad proxy credential at /internal/%s', + async (route) => { + const { project, bodies } = await liveProject(); + const response = await internalRequest( + route, + project.id, + { + 'x-void-proxy-internal': await deriveProxyInternalAuth( + JWT_SECRET, + route, + 'proj_000000000999', + ), + 'x-void-internal': await hmacSignState(project.id, JWT_SECRET), + }, + bodies[route], + ); + expect(response.status).toBe(403); + expect(await response.json()).toEqual({ error: 'invalid internal auth' }); + }, + ); + + it.each(routes)('applies the floor to a project token at /internal/%s', async (route) => { + const { project, bodies, expected } = await liveProject(); + const legacy = await hmacSignState(project.id, JWT_SECRET); + + const accepted = await internalRequest(route, project.id, legacy, bodies[route]); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toEqual(expected[route]); + + await env.ROUTING.put(`proxy-token:${project.id}`, '{"gen":1}'); + const rejected = await internalRequest(route, project.id, legacy, bodies[route]); + expect(rejected.status).toBe(403); + expect(await rejected.json()).toEqual({ error: 'invalid internal auth' }); + + await env.ROUTING.put(`proxy-token:${project.id}`, 'not json'); + const unavailable = await internalRequest(route, project.id, legacy, bodies[route]); + expect(unavailable.status).toBe(503); + expect(await unavailable.json()).toEqual({ error: 'project token state unavailable' }); + }); + + it.each(routes)('requires internal auth at /internal/%s', async (route) => { + const { project, bodies } = await liveProject(); + const missing = await internalRequest(route, project.id, {}, bodies[route]); + expect(missing.status).toBe(401); + expect(await missing.json()).toEqual({ error: 'missing internal auth' }); + + const get = vi.spyOn(env.ROUTING, 'get'); + const forged = await internalRequest( + route, + project.id, + await hmacSignState('proj_000000000999', JWT_SECRET), + bodies[route], + ); + expect(forged.status).toBe(403); + expect(await forged.json()).toEqual({ error: 'invalid internal auth' }); + expect(floorReads(get)).toEqual([]); + }); +}); diff --git a/platform/packages/api/test/integration/api-projects.test.ts b/platform/packages/api/test/integration/api-projects.test.ts index 888d86ed0..0f34016b5 100644 --- a/platform/packages/api/test/integration/api-projects.test.ts +++ b/platform/packages/api/test/integration/api-projects.test.ts @@ -7,7 +7,6 @@ import { verifyDeployToken, verifyPrerenderToken, } from '../../src/auth/deploy-token.ts'; -import { hmacSignState } from '../../src/auth/hmac.ts'; import { revokeConnectionSessions } from '../../src/auth/sessions.ts'; import { createDeployment, @@ -19,6 +18,7 @@ import { import * as db from '../../src/db.ts'; import app from '../../src/index.ts'; import * as projectAccess from '../../src/project-access.ts'; +import { deriveProxyInternalAuth } from '../../src/proxy-token.ts'; import { users } from '../../src/schema.ts'; import { deployments, @@ -728,7 +728,11 @@ describe('project CI credential ownership', () => { }); describe('POST /projects/:id/prerender-token', () => { - async function proxyRequest(token: string, projectId: string) { + async function proxyRequest( + token: string, + projectId: string, + options: { dropLegacyInternalHeader?: boolean } = {}, + ) { const proxyModuleUrl = new URL('../../../proxy/src/index.ts', import.meta.url).href; const proxyWorker = ( (await import(/* @vite-ignore */ proxyModuleUrl)) as { @@ -737,7 +741,11 @@ describe('POST /projects/:id/prerender-token', () => { ).default; const apiBinding = { fetch(input: string | Request, init?: RequestInit) { - return app.fetch(input instanceof Request ? input : new Request(input, init), env); + const request = input instanceof Request ? input : new Request(input, init); + if (options.dropLegacyInternalHeader) { + request.headers.delete('x-void-internal'); + } + return app.fetch(request, env); }, }; return proxyWorker.fetch( @@ -820,6 +828,23 @@ describe('POST /projects/:id/prerender-token', () => { expect(payload!.exp - payload!.iat).toBe(65 * 60); }); + it('checks prerender authority with the proxy credential alone', async () => { + const user = await createTestUser(env.db, 'prerender-proxy-auth'); + const project = await createTestProject(env.db, user.id, 'prerender-proxy-auth-app'); + + const res = await app.fetch( + new Request(`http://localhost/projects/${project.id}/prerender-token`, { + method: 'POST', + headers: { Authorization: `Bearer ${user.jwt}` }, + }), + env, + ); + expect(res.status).toBe(200); + const { token } = await res.json<{ token: string }>(); + const proxied = await proxyRequest(token, project.id, { dropLegacyInternalHeader: true }); + expect(proxied.status).toBe(200); + }); + it('revokes a collaborator manual prerender token when the member is removed', async () => { const owner = await createTestUser(env.db, 'prerender-removal-owner'); const collaborator = await createTestUser(env.db, 'prerender-removal-collaborator'); @@ -915,14 +940,18 @@ describe('POST /projects/:id/prerender-token', () => { expect(payload!.exp).toBeLessThanOrEqual(parent!.exp); expect(payload!.exp).toBeGreaterThanOrEqual(parent!.exp - 1); - const internalAuth = await hmacSignState(project.id, env.JWT_SECRET); + const internalAuth = await deriveProxyInternalAuth( + env.JWT_SECRET, + 'verify-prerender-build', + project.id, + ); const verifyBuild = () => app.fetch( new Request(`http://localhost/internal/verify-prerender-build/${project.id}`, { method: 'POST', headers: { 'content-type': 'application/json', - 'x-void-internal': internalAuth, + 'x-void-proxy-internal': internalAuth, }, body: JSON.stringify({ userId: user.id, buildId: build.id }), }), diff --git a/platform/packages/api/test/integration/api-resolve-worker.test.ts b/platform/packages/api/test/integration/api-resolve-worker.test.ts index 1cd5f58ee..004a5caa2 100644 --- a/platform/packages/api/test/integration/api-resolve-worker.test.ts +++ b/platform/packages/api/test/integration/api-resolve-worker.test.ts @@ -1,8 +1,8 @@ import { eq } from 'drizzle-orm'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vite-plus/test'; -import { hmacSignState } from '../../src/auth/hmac.ts'; import { setActiveDeployment } from '../../src/db.ts'; import app from '../../src/index.ts'; +import { deriveProxyInternalAuth } from '../../src/proxy-token.ts'; import { projects } from '../../src/schema.ts'; import { clearTables, @@ -29,11 +29,11 @@ beforeEach(async () => { }); async function resolveWorker(projectId: string): Promise { - const token = await hmacSignState(projectId, env.JWT_SECRET); + const auth = await deriveProxyInternalAuth(env.JWT_SECRET, 'resolve-worker', projectId); return app.fetch( new Request(`http://localhost/internal/resolve-worker/${projectId}`, { method: 'POST', - headers: { 'x-void-internal': token }, + headers: { 'x-void-proxy-internal': auth }, }), env, ); diff --git a/platform/packages/proxy/src/dispatch.ts b/platform/packages/proxy/src/dispatch.ts index e92c61034..487b3c9e1 100644 --- a/platform/packages/proxy/src/dispatch.ts +++ b/platform/packages/proxy/src/dispatch.ts @@ -1,4 +1,8 @@ -import { ProxyTokenStateError } from '@voidcloud/api/proxy-token'; +import { + deriveProxyInternalAuth, + PROXY_INTERNAL_AUTH_HEADER, + ProxyTokenStateError, +} from '@voidcloud/api/proxy-token'; import { hmacSign } from './hmac.ts'; import { tenantSendTokens } from './proxy-token.ts'; import type { Env } from './types.ts'; @@ -26,12 +30,17 @@ async function resolveWorker( jwtSecret: string, projectId: string, ): Promise { - const token = await hmacSign(projectId, jwtSecret); const res = await api.fetch(`https://api/internal/resolve-worker/${projectId}`, { method: 'POST', headers: { 'content-type': 'application/json', - 'x-void-internal': token, + // Legacy header for an API that predates the proxy credential. + 'x-void-internal': await hmacSign(projectId, jwtSecret), + [PROXY_INTERNAL_AUTH_HEADER]: await deriveProxyInternalAuth( + jwtSecret, + 'resolve-worker', + projectId, + ), }, }); diff --git a/platform/packages/proxy/src/resolve.ts b/platform/packages/proxy/src/resolve.ts index ab760aad2..65978f06e 100644 --- a/platform/packages/proxy/src/resolve.ts +++ b/platform/packages/proxy/src/resolve.ts @@ -1,3 +1,4 @@ +import { deriveProxyInternalAuth, PROXY_INTERNAL_AUTH_HEADER } from '@voidcloud/api/proxy-token'; import { hmacSign } from './hmac.ts'; // In-memory cache: resource mappings are stable (only change on deploy). @@ -27,12 +28,17 @@ export async function resolveResource( return cached; } - const token = await hmacSign(projectId, jwtSecret); const res = await api.fetch(`https://api/internal/resolve-resource/${projectId}`, { method: 'POST', headers: { 'content-type': 'application/json', - 'x-void-internal': token, + // Legacy header for an API that predates the proxy credential. + 'x-void-internal': await hmacSign(projectId, jwtSecret), + [PROXY_INTERNAL_AUTH_HEADER]: await deriveProxyInternalAuth( + jwtSecret, + 'resolve-resource', + projectId, + ), }, body: JSON.stringify({ type, binding: bindingName }), }); @@ -97,12 +103,17 @@ export async function verifyPrerenderAuthority( provenance: { buildId?: string; credentialId?: string }, ): Promise { try { - const token = await hmacSign(projectId, jwtSecret); const res = await api.fetch(`https://api/internal/verify-prerender-build/${projectId}`, { method: 'POST', headers: { 'content-type': 'application/json', - 'x-void-internal': token, + // Legacy header for an API that predates the proxy credential. + 'x-void-internal': await hmacSign(projectId, jwtSecret), + [PROXY_INTERNAL_AUTH_HEADER]: await deriveProxyInternalAuth( + jwtSecret, + 'verify-prerender-build', + projectId, + ), }, body: JSON.stringify({ userId, ...provenance }), }); diff --git a/platform/packages/proxy/test/unit/dispatch.test.ts b/platform/packages/proxy/test/unit/dispatch.test.ts index be502a8ff..33d77d6b0 100644 --- a/platform/packages/proxy/test/unit/dispatch.test.ts +++ b/platform/packages/proxy/test/unit/dispatch.test.ts @@ -1,4 +1,4 @@ -import { deriveProxyToken } from '@voidcloud/api/proxy-token'; +import { deriveProxyInternalAuth, deriveProxyToken } from '@voidcloud/api/proxy-token'; import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vite-plus/test'; import { createBindingHandler } from '../../../../../packages/void/src/runtime/remote/binding-handler.ts'; import { dispatchToWorker } from '../../src/dispatch.ts'; @@ -616,6 +616,23 @@ describe('WfP dispatch path', () => { ); expect(capturedHeaders['x-void-internal']).toBe(token); + expect(capturedHeaders['x-void-proxy-internal']).toBeUndefined(); + }); + + it('authenticates worker resolution with the proxy credential and the legacy header', async () => { + const env = createEnv({ proj_000000000001: 'user-my-app' }, async () => + Response.json({ results: [], success: true, meta: {} }), + ); + + await dispatchToWorker(env as unknown as Env, 'proj_000000000001', '/d1/query', {}); + + const [url, init] = env.API.fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe('https://api/internal/resolve-worker/proj_000000000001'); + const headers = init.headers as Record; + expect(headers['x-void-proxy-internal']).toBe( + await deriveProxyInternalAuth(JWT_SECRET, 'resolve-worker', 'proj_000000000001'), + ); + expect(headers['x-void-internal']).toBe(token); }); }); }); diff --git a/platform/packages/proxy/test/unit/resolve.test.ts b/platform/packages/proxy/test/unit/resolve.test.ts index 96f8541e8..0d310bc12 100644 --- a/platform/packages/proxy/test/unit/resolve.test.ts +++ b/platform/packages/proxy/test/unit/resolve.test.ts @@ -1,7 +1,10 @@ +import { deriveProxyInternalAuth, deriveProxyToken } from '@voidcloud/api/proxy-token'; import { describe, it, expect, vi, beforeEach } from 'vite-plus/test'; +import { hmacSign } from '../../src/hmac.ts'; import { resolveResource, clearResolveCache, + verifyPrerenderAuthority, verifyProjectDeployAccess, } from '../../src/resolve.ts'; @@ -62,16 +65,39 @@ describe('resolveResource', () => { expect((api.fetch as ReturnType).mock.calls).toHaveLength(1); }); - it('includes HMAC token in x-void-internal header', async () => { + it('authenticates with the proxy credential and the legacy header', async () => { const api = createMockApi({ 'proj_000000000001:d1:DB': 'd1-uuid-123' }); await resolveResource(api, JWT_SECRET, 'proj_000000000001', 'd1', 'DB'); - const call = (api.fetch as ReturnType).mock.calls[0]; - const init = call[1] as RequestInit; - const headers = init.headers as Record; - expect(headers['x-void-internal']).toBeDefined(); - expect(typeof headers['x-void-internal']).toBe('string'); - expect(headers['x-void-internal'].length).toBeGreaterThan(0); + const [, init] = (api.fetch as ReturnType).mock.calls[0]!; + const headers = (init as RequestInit).headers as Record; + expect(headers['x-void-proxy-internal']).toBe( + await deriveProxyInternalAuth(JWT_SECRET, 'resolve-resource', 'proj_000000000001'), + ); + expect(headers['x-void-internal']).toBe( + await deriveProxyToken(JWT_SECRET, 'proj_000000000001', 0), + ); + }); +}); + +describe('verifyPrerenderAuthority', () => { + it('authenticates with the proxy credential and the legacy header', async () => { + const api = { fetch: vi.fn(async () => Response.json({ ok: true })) } as unknown as Fetcher; + await expect( + verifyPrerenderAuthority(api, JWT_SECRET, 'proj_000000000001', 'user_1', { + buildId: 'bld_1', + }), + ).resolves.toBe(true); + + const [url, init] = (api.fetch as ReturnType).mock.calls[0]!; + expect(url).toBe('https://api/internal/verify-prerender-build/proj_000000000001'); + const headers = (init as RequestInit).headers as Record; + expect(headers['x-void-proxy-internal']).toBe( + await deriveProxyInternalAuth(JWT_SECRET, 'verify-prerender-build', 'proj_000000000001'), + ); + expect(headers['x-void-internal']).toBe( + await deriveProxyToken(JWT_SECRET, 'proj_000000000001', 0), + ); }); }); @@ -94,9 +120,14 @@ describe('verifyProjectDeployAccess', () => { userId: 'user_1', proxyOrigin: 'https://proxy.example.com', }); - expect( - ((init as RequestInit).headers as Record)['x-void-internal'], - ).toBeTruthy(); + const headers = (init as RequestInit).headers as Record; + expect(headers['x-void-internal']).toBe( + await hmacSign( + 'void-proxy-owner-v2\nproj_000000000001\nhttps://proxy.example.com', + JWT_SECRET, + ), + ); + expect(headers['x-void-proxy-internal']).toBeUndefined(); const missing = { fetch: vi.fn() } as unknown as Fetcher; await expect( From fedb65d03512dcf11814065a2f26213e7a82e01e Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 01:59:01 +0800 Subject: [PATCH 05/12] Issue proxy tokens from the accept floor provisionBindings now takes the Worker's proxy token in its platform options instead of deriving it from JWT_SECRET. The deploy upload issues it with issueProxyToken, which returns the token for the project's accept floor (the legacy token while no floor is set), so existing projects get the same bytes as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../packages/api/src/deploy/upload-worker.ts | 3 +- .../api/src/provisioning/provision.ts | 7 +- .../api/test/integration/api-deploy.test.ts | 48 ++++++- .../packages/api/test/unit/provision.test.ts | 117 +++++++----------- .../api/test/unit/reserved-bindings.test.ts | 29 ++--- 5 files changed, 109 insertions(+), 95 deletions(-) diff --git a/platform/packages/api/src/deploy/upload-worker.ts b/platform/packages/api/src/deploy/upload-worker.ts index a62bd3a26..084d66e91 100644 --- a/platform/packages/api/src/deploy/upload-worker.ts +++ b/platform/packages/api/src/deploy/upload-worker.ts @@ -24,6 +24,7 @@ import { provisionSandboxController, sandboxControllerWorkerName, } from '../provisioning/sandbox-controller.ts'; +import { issueProxyToken } from '../proxy-token.ts'; import { RESERVED_BINDING_NAMES } from '../reserved-bindings.ts'; import { deployments } from '../schema.ts'; import { type DeployManifest, type DeployEvent, type Deployment, type Project } from '../types.ts'; @@ -113,10 +114,10 @@ export async function uploadDeploymentWorker(input: { context.env.CF_API_TOKEN, projectId, manifest, - context.env.JWT_SECRET, context.env.VOID_PROXY_SERVICE ?? 'void-proxy', { projectSlug: project.slug, + proxyToken: await issueProxyToken(context.env.ROUTING, context.env.JWT_SECRET, projectId), emailSenderDomain: context.env.EMAIL_SENDER_DOMAIN, deploymentId: deployment.id, }, diff --git a/platform/packages/api/src/provisioning/provision.ts b/platform/packages/api/src/provisioning/provision.ts index b65a16aa9..a81a68d06 100644 --- a/platform/packages/api/src/provisioning/provision.ts +++ b/platform/packages/api/src/provisioning/provision.ts @@ -1,5 +1,4 @@ import { createDeployIntent } from '@void/deploy-core'; -import { hmacSignState } from '../auth/hmac.ts'; import { getProjectResources, createResource, type Db } from '../db.ts'; import { getProjectQueueName } from '../naming.ts'; import type { DeployManifest, Resource } from '../types.ts'; @@ -36,7 +35,6 @@ export async function provisionBindings( apiToken: string, projectId: string, manifest: DeployManifest, - jwtSecret: string, aiProxyService: string, /** * Per-project platform identity injected into every worker. Grouped into @@ -47,6 +45,8 @@ export async function provisionBindings( platform: { /** Project slug. Framework reads it as `__VOID_PROJECT_SLUG`. */ projectSlug: string; + /** Token the Worker presents to the proxy. Framework reads it as `__VOID_PROXY_TOKEN`. */ + proxyToken: string; /** Immutable deployment identity for signed inbound email receipts. */ deploymentId?: string; /** @@ -165,10 +165,9 @@ export async function provisionBindings( // Always inject proxy bindings — used by AI, revalidation, email, and future // platform features. The AI-specific usage limit check stays in the proxy // (/ai/* middleware). - const proxyToken = await hmacSignState(projectId, jwtSecret); bindings.push({ type: 'service', name: '__VOID_PROXY', service: aiProxyService }); bindings.push({ type: 'plain_text', name: '__PROJECT_ID', text: projectId }); - bindings.push({ type: 'plain_text', name: '__VOID_PROXY_TOKEN', text: proxyToken }); + bindings.push({ type: 'plain_text', name: '__VOID_PROXY_TOKEN', text: platform.proxyToken }); // Project slug — the SINGLE producer of `__VOID_PROJECT_SLUG`. `void/email` // needs it on every worker to default an outbound `from` to // `+noreply@`, and the sandbox runtime reads the same name. diff --git a/platform/packages/api/test/integration/api-deploy.test.ts b/platform/packages/api/test/integration/api-deploy.test.ts index 112fb0cab..8b8b0966c 100644 --- a/platform/packages/api/test/integration/api-deploy.test.ts +++ b/platform/packages/api/test/integration/api-deploy.test.ts @@ -12,6 +12,7 @@ import { vi, } from 'vite-plus/test'; import { computeAssetHash } from '../../src/assets.ts'; +import { hmacSignState } from '../../src/auth/hmac.ts'; import { beginDeploymentCutover, cancelDeployment, @@ -27,6 +28,7 @@ import app from '../../src/index.ts'; import { writePlanCatalog } from '../../src/plans.ts'; import * as projectAccess from '../../src/project-access.ts'; import * as projectIngress from '../../src/project-ingress.ts'; +import { deriveProxyToken, proxyTokenStateKey } from '../../src/proxy-token.ts'; import { validateManifestShape } from '../../src/routes/deploy.ts'; import { builds, @@ -54,6 +56,7 @@ import { cfSuccess, readAllEvents, CF_ACCOUNT_ID, + JWT_SECRET, type TestEnv, } from './helpers.ts'; import { setTestDb } from './setup.ts'; @@ -1999,9 +2002,50 @@ describe('POST /projects/:id/deploy', () => { workerName: expect.stringMatching(/^user-queue-kv-app-dpl_/), projectId: project.id, queueName: 'emails', - proxyToken: expect.any(String), + proxyToken: await hmacSignState(project.id, JWT_SECRET), }); - expect(queueRouting?.proxyToken.length ?? 0).toBeGreaterThan(0); + }); + + it('issues the proxy token for the project accept floor', async () => { + const user = await createTestUser(env.db); + const project = await createTestProject(env.db, user.id, 'queue-kv-floor-app'); + await env.ROUTING.put(proxyTokenStateKey(project.id), JSON.stringify({ gen: 1 })); + const fetchMock = mockCfApi(); + + const res = await app.fetch( + await buildDeployRequest(env, project.id, user.jwt, { + manifest: { + version: 3, + bindings: {}, + queues: [{ name: 'emails', bindingName: 'QUEUE_EMAILS' }], + }, + }), + env, + ); + expect(res.status).toBe(200); + await res.text(); + + const expected = await deriveProxyToken(JWT_SECRET, project.id, 1); + const uploadCall = fetchMock.mock.calls.find((call: Array) => { + const url = typeof call[0] === 'string' ? call[0] : String(call[0]); + return url.includes('/workers/dispatch/') && (call[1] as RequestInit)?.method === 'PUT'; + }); + expect(uploadCall).toBeDefined(); + const metadataBlob = ((uploadCall![1] as RequestInit).body as FormData).get('metadata'); + const metadata = JSON.parse(await (metadataBlob as Blob).text()) as { + bindings?: Array<{ type: string; name: string; text?: string }>; + }; + expect(metadata.bindings).toContainEqual({ + type: 'plain_text', + name: '__VOID_PROXY_TOKEN', + text: expected, + }); + + const expectedQueueName = `${project.id}-emails`.toLowerCase().replaceAll('_', '-'); + const queueRouting = (await env.ROUTING.get(`queue:${expectedQueueName}`, 'json')) as { + proxyToken: string; + } | null; + expect(queueRouting?.proxyToken).toBe(expected); }); it('removes queue routing entries omitted by the next deployment', async () => { diff --git a/platform/packages/api/test/unit/provision.test.ts b/platform/packages/api/test/unit/provision.test.ts index d3d1772ae..11f336716 100644 --- a/platform/packages/api/test/unit/provision.test.ts +++ b/platform/packages/api/test/unit/provision.test.ts @@ -52,7 +52,7 @@ const ACCOUNT_ID = 'test-account'; const API_TOKEN = 'test-token'; const PROJECT_ID = 'proj_abc123'; const PROJECT_SLUG = 'test-slug'; -const JWT_SECRET = 'test-jwt-secret'; +const PROXY_TOKEN = 'test-proxy-token'; const mockDb = {} as Db; beforeEach(() => { @@ -70,9 +70,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).toHaveBeenCalledWith(ACCOUNT_ID, API_TOKEN, 'proj-abc123-db'); @@ -90,16 +89,10 @@ describe('provisionBindings', () => { vi.mocked(createResource).mockRejectedValueOnce(new Error('insert failed (FK violation)')); await expect( - provisionBindings( - mockDb, - ACCOUNT_ID, - API_TOKEN, - PROJECT_ID, - manifest, - JWT_SECRET, - 'void-proxy', - { projectSlug: PROJECT_SLUG }, - ), + provisionBindings(mockDb, ACCOUNT_ID, API_TOKEN, PROJECT_ID, manifest, 'void-proxy', { + projectSlug: PROJECT_SLUG, + proxyToken: PROXY_TOKEN, + }), ).rejects.toThrow('insert failed'); // CF created it... @@ -117,16 +110,10 @@ describe('provisionBindings', () => { vi.mocked(createResource).mockRejectedValueOnce(new Error('insert failed (FK violation)')); await expect( - provisionBindings( - mockDb, - ACCOUNT_ID, - API_TOKEN, - PROJECT_ID, - manifest, - JWT_SECRET, - 'void-proxy', - { projectSlug: PROJECT_SLUG }, - ), + provisionBindings(mockDb, ACCOUNT_ID, API_TOKEN, PROJECT_ID, manifest, 'void-proxy', { + projectSlug: PROJECT_SLUG, + proxyToken: PROXY_TOKEN, + }), ).rejects.toThrow('insert failed'); expect(deleteD1Database).not.toHaveBeenCalled(); @@ -141,9 +128,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createKVNamespace).toHaveBeenCalledWith(ACCOUNT_ID, API_TOKEN, 'proj-abc123-kv'); @@ -161,9 +147,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createR2Bucket).toHaveBeenCalledWith(ACCOUNT_ID, API_TOKEN, 'proj-abc123-storage'); @@ -195,9 +180,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createQueue).toHaveBeenCalledWith(ACCOUNT_ID, API_TOKEN, 'proj-abc123-emails'); @@ -238,9 +222,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).not.toHaveBeenCalled(); @@ -270,9 +253,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, { version: 3, bindings: {}, queues: [{ name: 'emails', bindingName: 'QUEUE_EMAILS' }] }, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(bindings[0]).toMatchObject({ type: 'queue', queue_id: id }); expect(getQueueByName).not.toHaveBeenCalled(); @@ -293,9 +275,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).toHaveBeenCalledOnce(); @@ -321,9 +302,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).not.toHaveBeenCalled(); @@ -335,7 +315,7 @@ describe('provisionBindings', () => { expect(bindings[0]).toEqual({ type: 'service', name: '__VOID_PROXY', service: 'void-proxy' }); }); - it('always injects proxy bindings with signed token', async () => { + it('always injects proxy bindings with the issued token', async () => { const manifest: DeployManifest = { version: 1, bindings: {} }; const bindings = await provisionBindings( @@ -344,19 +324,18 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(bindings).toHaveLength(4); expect(bindings[0]).toEqual({ type: 'service', name: '__VOID_PROXY', service: 'void-proxy' }); expect(bindings[1]).toEqual({ type: 'plain_text', name: '__PROJECT_ID', text: PROJECT_ID }); - expect(bindings[2].type).toBe('plain_text'); - expect(bindings[2].name).toBe('__VOID_PROXY_TOKEN'); - // Token should be a non-empty string (HMAC signature) - expect(typeof bindings[2].text).toBe('string'); - expect(bindings[2].text!.length).toBeGreaterThan(0); + expect(bindings[2]).toEqual({ + type: 'plain_text', + name: '__VOID_PROXY_TOKEN', + text: PROXY_TOKEN, + }); }); it('always injects __VOID_PROJECT_SLUG, for sandbox and non-sandbox projects alike', async () => { @@ -371,9 +350,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(bindings).toContainEqual({ @@ -395,9 +373,12 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG, emailSenderDomain: 'mail.voidcloud.dev' }, + { + projectSlug: PROJECT_SLUG, + proxyToken: PROXY_TOKEN, + emailSenderDomain: 'mail.voidcloud.dev', + }, ); expect(bindings).toContainEqual({ @@ -419,9 +400,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG, emailSenderDomain: '' }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN, emailSenderDomain: '' }, ); expect(bindings.some((b) => b.name === '__VOID_EMAIL_DOMAIN')).toBe(false); @@ -436,9 +416,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); // D1 + 4 platform bindings @@ -446,7 +425,11 @@ describe('provisionBindings', () => { expect(bindings[0]).toEqual({ type: 'd1', name: 'DB', id: 'd1-uuid-123' }); expect(bindings[1]).toEqual({ type: 'service', name: '__VOID_PROXY', service: 'void-proxy' }); expect(bindings[2]).toEqual({ type: 'plain_text', name: '__PROJECT_ID', text: PROJECT_ID }); - expect(bindings[3].name).toBe('__VOID_PROXY_TOKEN'); + expect(bindings[3]).toEqual({ + type: 'plain_text', + name: '__VOID_PROXY_TOKEN', + text: PROXY_TOKEN, + }); }); it('provisions only missing resources when some already exist', async () => { @@ -474,9 +457,8 @@ describe('provisionBindings', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).not.toHaveBeenCalled(); @@ -501,9 +483,8 @@ describe('ensureResource — already exists fallback', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createD1Database).toHaveBeenCalledOnce(); @@ -525,9 +506,8 @@ describe('ensureResource — already exists fallback', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createKVNamespace).toHaveBeenCalledOnce(); @@ -551,9 +531,8 @@ describe('ensureResource — already exists fallback', () => { API_TOKEN, PROJECT_ID, manifest, - JWT_SECRET, 'void-proxy', - { projectSlug: PROJECT_SLUG }, + { projectSlug: PROJECT_SLUG, proxyToken: PROXY_TOKEN }, ); expect(createQueue).toHaveBeenCalledOnce(); @@ -573,16 +552,10 @@ describe('ensureResource — already exists fallback', () => { const manifest: DeployManifest = { version: 1, bindings: { d1: ['DB'] } }; await expect( - provisionBindings( - mockDb, - ACCOUNT_ID, - API_TOKEN, - PROJECT_ID, - manifest, - JWT_SECRET, - 'void-proxy', - { projectSlug: PROJECT_SLUG }, - ), + provisionBindings(mockDb, ACCOUNT_ID, API_TOKEN, PROJECT_ID, manifest, 'void-proxy', { + projectSlug: PROJECT_SLUG, + proxyToken: PROXY_TOKEN, + }), ).rejects.toThrow('rate limit exceeded'); expect(getD1DatabaseByName).not.toHaveBeenCalled(); diff --git a/platform/packages/api/test/unit/reserved-bindings.test.ts b/platform/packages/api/test/unit/reserved-bindings.test.ts index 8a1902204..f3655a134 100644 --- a/platform/packages/api/test/unit/reserved-bindings.test.ts +++ b/platform/packages/api/test/unit/reserved-bindings.test.ts @@ -30,9 +30,12 @@ async function platformBindingNames(): Promise> { 'test-token', 'proj_abc123', EMPTY_MANIFEST, - 'test-jwt-secret', 'void-proxy', - { projectSlug: 'my-app', emailSenderDomain: 'mail.voidcloud.dev' }, + { + projectSlug: 'my-app', + proxyToken: 'test-proxy-token', + emailSenderDomain: 'mail.voidcloud.dev', + }, ); return provisioned.map((binding) => binding.name); } @@ -45,9 +48,12 @@ describe('reserved binding names cover every platform-injected binding', () => { 'test-token', 'proj_abc123', EMPTY_MANIFEST, - 'test-jwt-secret', 'void-proxy', - { projectSlug: 'my-app', emailSenderDomain: 'mail.voidcloud.dev' }, + { + projectSlug: 'my-app', + proxyToken: 'test-proxy-token', + emailSenderDomain: 'mail.voidcloud.dev', + }, ); expect(provisioned.length).toBeGreaterThan(0); for (const binding of provisioned) { @@ -115,9 +121,8 @@ it('pins managed email to its deployment identity and reserves that binding', as 'test-token', 'proj_abc123', { ...EMPTY_MANIFEST, emailProtocolVersion: 1 }, - 'test-jwt-secret', 'void-proxy', - { projectSlug: 'my-app', deploymentId: 'deploy_a' }, + { projectSlug: 'my-app', proxyToken: 'test-proxy-token', deploymentId: 'deploy_a' }, ] as const; const bindings = await provisionBindings(...args); expect(bindings).toContainEqual({ @@ -128,16 +133,8 @@ it('pins managed email to its deployment identity and reserves that binding', as expect(isReservedBindingName('__VOID_EMAIL_DEPLOYMENT_ID')).toBe(true); await expect( provisionBindings( - ...(args.slice(0, 7) as [ - (typeof args)[0], - string, - string, - string, - (typeof args)[4], - string, - string, - ]), - { projectSlug: 'my-app' }, + ...(args.slice(0, 6) as [(typeof args)[0], string, string, string, (typeof args)[4], string]), + { projectSlug: 'my-app', proxyToken: 'test-proxy-token' }, ), ).rejects.toThrow('deployment identity'); }); From 4757cecbcef1b84b28dd90e25e1e1926bed6bc90 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 02:03:20 +0800 Subject: [PATCH 06/12] Retry cron fires and queue relays with a fallback token Cron fire messages, the CronScheduler state and queue routing entries can now carry an optional `proxyTokenFallback`. When a tenant Worker rejects the first token with its generated `unauthorized` response, the cron-fires consumer and the dispatch queue relay try once more with the fallback token, using the same request body and the same /__void then legacy path order. Both rejects come before any cron or queue handler runs, so the second attempt cannot run a handler twice. There is no second attempt after a thrown fetch, another status or another body. If the fallback is rejected too, the existing behavior applies: cron fires ack with the AUTH_DRIFT log, and queue relays retry the batch. The CronScheduler copies the fallback into each fire message and leaves it out of /status, like `proxyToken`. Nothing writes the fallback yet, so every relay and cron fire still sends one request with one token. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/src/cron-fires-queue.ts | 45 +++++- .../packages/api/src/cron-scheduler-client.ts | 2 + platform/packages/api/src/cron-scheduler.ts | 5 +- .../api/test/unit/cron-fires-queue.test.ts | 153 ++++++++++++++++++ .../api/test/unit/cron-scheduler.test.ts | 67 ++++++++ platform/packages/dispatch/src/index.ts | 67 ++++++-- .../packages/dispatch/test/unit/index.test.ts | 106 ++++++++++++ 7 files changed, 418 insertions(+), 27 deletions(-) diff --git a/platform/packages/api/src/cron-fires-queue.ts b/platform/packages/api/src/cron-fires-queue.ts index 00a91b460..c78d088ba 100644 --- a/platform/packages/api/src/cron-fires-queue.ts +++ b/platform/packages/api/src/cron-fires-queue.ts @@ -15,6 +15,8 @@ export type CronFireMessage = Readonly<{ cron: string; scheduledTime: number; proxyToken?: string; + /** Second token to try when the Worker rejects `proxyToken`. */ + proxyTokenFallback?: string; }>; type InvokeOutcome = Readonly<{ @@ -24,8 +26,27 @@ type InvokeOutcome = Readonly<{ bodySnippet?: string; /** True only when the worker script no longer exists. */ unreachable?: boolean; + /** True when the worker refused the token before running the cron handler. */ + tokenRejected?: boolean; }>; +/** The generated cron route's reject for a token it does not hold. */ +function isTokenRejection(status: number, text: string): boolean { + if (status !== 401 && status !== 403) { + return false; + } + try { + const data: unknown = JSON.parse(text); + return ( + typeof data === 'object' && + data !== null && + (data as { error?: unknown }).error === 'unauthorized' + ); + } catch { + return false; + } +} + /** * Cloudflare's WfP dispatch surfaces a missing-worker error as a thrown * Error whose message starts with "Worker not found". This predicate is @@ -40,10 +61,11 @@ function isWorkerNotFoundError(err: unknown): boolean { async function invokeWorker( env: ApiEnv['Bindings'], message: CronFireMessage, + token: string | undefined, ): Promise { const headers: Record = { 'Content-Type': 'application/json' }; - if (message.proxyToken) { - headers['x-void-internal'] = message.proxyToken; + if (token) { + headers['x-void-internal'] = token; } const body = JSON.stringify({ cron: message.cron, scheduledTime: message.scheduledTime }); @@ -80,6 +102,7 @@ async function invokeWorker( responseReceived: true, status: res.status, bodySnippet: text.slice(0, 200), + tokenRejected: isTokenRejection(res.status, text), }; } // Fully consume the response body so the dispatch subrequest finalizes cleanly, @@ -103,7 +126,13 @@ export async function processCronFiresBatch( ): Promise { await Promise.allSettled( batch.messages.map(async (msg) => { - const outcome = await invokeWorker(env, msg.body); + let outcome = await invokeWorker(env, msg.body, msg.body.proxyToken); + // While a token rotation is in progress the worker may hold either + // token. Its reject comes before the cron handler runs, so a second + // attempt with the other token cannot run the handler twice. + if (outcome.tokenRejected && msg.body.proxyTokenFallback) { + outcome = await invokeWorker(env, msg.body, msg.body.proxyTokenFallback); + } // A Response came back — the user worker's handler ran (or the runtime // made a routing decision, e.g. 404 on missing handler). Either way the @@ -113,11 +142,11 @@ export async function processCronFiresBatch( if (outcome.responseReceived) { const status = outcome.status; if (status === 401 || status === 403) { - // Auth drift between CronScheduler's stored proxyToken and the - // user worker's baked __VOID_PROXY_TOKEN. Retrying would yield - // the same 401 (same token in, same status out), so we ack; the - // next deploy's /sync resyncs the token. Logged with a stable - // greppable marker so ops can alert on this. + // Auth drift: the user worker's baked __VOID_PROXY_TOKEN matches + // neither the stored proxyToken nor its fallback, if any. Retrying + // would yield the same 401 (same token in, same status out), so we + // ack; the next deploy's /sync resyncs the token. Logged with a + // stable greppable marker so ops can alert on this. console.error( `[cron-fires] AUTH_DRIFT non-delivery for ${msg.body.workerName} (cron ${msg.body.cron}): status=${status} body=${outcome.bodySnippet ?? ''}; acking — next deploy will resync token`, ); diff --git a/platform/packages/api/src/cron-scheduler-client.ts b/platform/packages/api/src/cron-scheduler-client.ts index 8a393aa45..6acd2c4e1 100644 --- a/platform/packages/api/src/cron-scheduler-client.ts +++ b/platform/packages/api/src/cron-scheduler-client.ts @@ -4,6 +4,8 @@ export type CronSyncState = Readonly<{ workerName: string; schedules: ReadonlyArray<{ name: string; cron: string }>; proxyToken?: string; + /** Second token to try when the Worker rejects `proxyToken`. */ + proxyTokenFallback?: string; }>; /** diff --git a/platform/packages/api/src/cron-scheduler.ts b/platform/packages/api/src/cron-scheduler.ts index 24845fb94..01e6802b7 100644 --- a/platform/packages/api/src/cron-scheduler.ts +++ b/platform/packages/api/src/cron-scheduler.ts @@ -183,8 +183,8 @@ export class CronScheduler extends DurableObject { private async handleStatus(): Promise { const state = await this.getState(); const alarm = await this.ctx.storage.getAlarm(); - // Omit proxyToken from status response to avoid leaking secrets. - const { proxyToken: _, ...safeState } = state ?? {}; + // Omit the proxy tokens from the status response to avoid leaking secrets. + const { proxyToken: _, proxyTokenFallback: __, ...safeState } = state ?? {}; return Response.json({ state: state ? safeState : null, nextAlarm: alarm }); } @@ -240,6 +240,7 @@ export class CronScheduler extends DurableObject { cron, scheduledTime: now, proxyToken: state.proxyToken, + ...(state.proxyTokenFallback ? { proxyTokenFallback: state.proxyTokenFallback } : {}), }, })); const chunks = chunkMessages(messages, SEND_BATCH_CHUNK_SIZE); diff --git a/platform/packages/api/test/unit/cron-fires-queue.test.ts b/platform/packages/api/test/unit/cron-fires-queue.test.ts index d27407891..f9926ed24 100644 --- a/platform/packages/api/test/unit/cron-fires-queue.test.ts +++ b/platform/packages/api/test/unit/cron-fires-queue.test.ts @@ -108,6 +108,159 @@ describe('processCronFiresBatch', () => { } }); + describe('fallback token', () => { + const unauthorized = () => Response.json({ error: 'unauthorized' }, { status: 401 }); + + function rotatingMessage() { + return makeMessage({ + workerName: 'user-foo-dpl_a', + cron: '* * * * *', + scheduledTime: 1_700_000_000_000, + proxyToken: 'old-tok', + proxyTokenFallback: 'new-tok', + }); + } + + function authDriftLogs(errSpy: { mock: { calls: Array> } }) { + return errSpy.mock.calls.filter((args) => + typeof args[0] === 'string' ? args[0].includes('AUTH_DRIFT') : false, + ); + } + + it('invokes once more with the fallback when the worker rejects the token', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const seen: Array<{ token: string | null; body: string }> = []; + const env = makeEnv(async (req) => { + const token = req.headers.get('x-void-internal'); + seen.push({ token, body: await req.text() }); + return token === 'new-tok' ? Response.json({ ok: true }) : unauthorized(); + }); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), env); + + expect(seen.map((call) => call.token)).toEqual(['old-tok', 'new-tok']); + expect(seen[1].body).toBe(seen[0].body); + expect(msg.ack).toHaveBeenCalledOnce(); + expect(msg.retry).not.toHaveBeenCalled(); + expect(authDriftLogs(errSpy)).toHaveLength(0); + } finally { + errSpy.mockRestore(); + } + }); + + it('invokes once more after a 403 unauthorized reject', async () => { + const fetchSpy = vi.fn(async (req: Request) => + req.headers.get('x-void-internal') === 'new-tok' + ? Response.json({ ok: true }) + : Response.json({ error: 'unauthorized' }, { status: 403 }), + ); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); + + expect(fetchSpy).toHaveBeenCalledTimes(2); + expect(msg.ack).toHaveBeenCalledOnce(); + }); + + it('acks with AUTH_DRIFT when the fallback is rejected too', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const fetchSpy = vi.fn(async () => unauthorized()); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); + + expect(fetchSpy).toHaveBeenCalledTimes(2); + expect(msg.ack).toHaveBeenCalledOnce(); + expect(msg.retry).not.toHaveBeenCalled(); + expect(authDriftLogs(errSpy)).toHaveLength(1); + } finally { + errSpy.mockRestore(); + } + }); + + it('does not invoke again without a fallback', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const fetchSpy = vi.fn(async () => unauthorized()); + const msg = makeMessage({ + workerName: 'user-foo-dpl_a', + cron: '* * * * *', + scheduledTime: 1_700_000_000_000, + proxyToken: 'tok', + }); + + await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); + + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(msg.ack).toHaveBeenCalledOnce(); + expect(authDriftLogs(errSpy)).toHaveLength(1); + } finally { + errSpy.mockRestore(); + } + }); + + it('does not invoke again for a 401 with another body', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const fetchSpy = vi.fn(async () => Response.json({ error: 'expired' }, { status: 401 })); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); + + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(msg.ack).toHaveBeenCalledOnce(); + } finally { + errSpy.mockRestore(); + } + }); + + it('does not invoke again after a thrown fetch', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const fetchSpy = vi.fn(async () => { + throw new Error('connection reset'); + }); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); + + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(msg.ack).toHaveBeenCalledOnce(); + expect(msg.retry).not.toHaveBeenCalled(); + } finally { + errSpy.mockRestore(); + } + }); + + it('uses the legacy /__scheduled path for the fallback too', async () => { + const seen: Array<{ path: string; token: string | null }> = []; + const env = makeEnv(async (req) => { + const path = new URL(req.url).pathname; + const token = req.headers.get('x-void-internal'); + seen.push({ path, token }); + if (path === '/__void/scheduled') { + return new Response('not found', { status: 404 }); + } + return token === 'new-tok' ? Response.json({ ok: true }) : unauthorized(); + }); + const msg = rotatingMessage(); + + await processCronFiresBatch(makeBatch([msg]), env); + + expect(seen).toEqual([ + { path: '/__void/scheduled', token: 'old-tok' }, + { path: '/__scheduled', token: 'old-tok' }, + { path: '/__void/scheduled', token: 'new-tok' }, + { path: '/__scheduled', token: 'new-tok' }, + ]); + expect(msg.ack).toHaveBeenCalledOnce(); + expect(msg.retry).not.toHaveBeenCalled(); + }); + }); + it('acks 5xx responses and does not retry the user worker', async () => { // A Response (any status) means the user worker received the request. // The user's handler ran or made a routing decision — the queue is not diff --git a/platform/packages/api/test/unit/cron-scheduler.test.ts b/platform/packages/api/test/unit/cron-scheduler.test.ts index e7b3519f2..f81474e88 100644 --- a/platform/packages/api/test/unit/cron-scheduler.test.ts +++ b/platform/packages/api/test/unit/cron-scheduler.test.ts @@ -246,6 +246,73 @@ describe('CronScheduler.alarm', () => { ); } + it('copies proxyTokenFallback into each message when present', async () => { + const ctx = makeStubCtx({ + state: { + workerName: 'user-foo', + schedules: [ + { name: 'every', cron: '* * * * *' }, + { name: 'fives', cron: '*/5 * * * *' }, + ], + proxyToken: 'old-tok', + proxyTokenFallback: 'new-tok', + }, + }); + const sendBatch = vi.fn(async (_batch?: unknown) => {}); + const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch } }); + + await scheduler.alarm({ + scheduledTime: Date.UTC(2026, 0, 1), + isRetry: false, + retryCount: 0, + }); + + const messages = sendBatch.mock.calls[0][0] as Array<{ body: Record }>; + expect(messages).toHaveLength(2); + for (const message of messages) { + expect(message.body.proxyToken).toBe('old-tok'); + expect(message.body.proxyTokenFallback).toBe('new-tok'); + } + }); + + it('leaves proxyTokenFallback out of messages when the state has none', async () => { + const ctx = makeStubCtx({ + state: { + workerName: 'user-foo', + schedules: [{ name: 'every', cron: '* * * * *' }], + proxyToken: 'tok', + }, + }); + const sendBatch = vi.fn(async (_batch?: unknown) => {}); + const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch } }); + + await scheduler.alarm(); + + const messages = sendBatch.mock.calls[0][0] as Array<{ body: Record }>; + expect(messages[0].body.proxyToken).toBe('tok'); + expect(messages[0].body).not.toHaveProperty('proxyTokenFallback'); + }); + + it('omits both proxy tokens from /status', async () => { + const ctx = makeStubCtx({ + state: { + workerName: 'user-foo', + schedules: [{ name: 'every', cron: '* * * * *' }], + proxyToken: 'old-tok', + proxyTokenFallback: 'new-tok', + }, + }); + const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch: async () => {} } }); + + const res = await scheduler.fetch(new Request('https://do/status')); + const body = await res.json<{ state: Record }>(); + + expect(body.state).toEqual({ + workerName: 'user-foo', + schedules: [{ name: 'every', cron: '* * * * *' }], + }); + }); + it('throws when ALL sendBatch chunks fail (DO alarm framework will retry)', async () => { // At-most-once contract: when no fires were enqueued, throwing surfaces // the failure to the DO runtime so it retries with backoff. Safe because diff --git a/platform/packages/dispatch/src/index.ts b/platform/packages/dispatch/src/index.ts index 5beef1314..18623bbeb 100644 --- a/platform/packages/dispatch/src/index.ts +++ b/platform/packages/dispatch/src/index.ts @@ -98,6 +98,8 @@ type QueueRoutingEntry = Readonly<{ // without a matching `x-void-internal` header with 401. Absent on entries // written by older API versions. proxyToken?: string; + // Second token to try when the Worker rejects `proxyToken`. + proxyTokenFallback?: string; }>; const CLIENT_REVALIDATE_CACHE_CONTROL = 'public, max-age=0, must-revalidate'; @@ -1919,7 +1921,9 @@ export default { batch.retryAll(); return; } - const { workerName, queueName, proxyToken } = JSON.parse(routingRaw) as QueueRoutingEntry; + const { workerName, queueName, proxyToken, proxyTokenFallback } = JSON.parse( + routingRaw, + ) as QueueRoutingEntry; // Serialize batch for relay const messages = Array.from(batch.messages, (message) => ({ @@ -1932,27 +1936,39 @@ export default { const worker = env.DISPATCH.get(workerName); try { const body = JSON.stringify({ queue: queueName, messages }); - const queueHeaders: Record = { 'Content-Type': 'application/json' }; - if (proxyToken) { - queueHeaders['x-void-internal'] = proxyToken; - } - let res = await worker.fetch( - new Request('https://internal/__void/queue', { - method: 'POST', - headers: queueHeaders, - body, - }), - ); - // Legacy: fall back for workers deployed before the /__void/ migration. - // TODO: remove once all deployed workers use /__void/* paths. - if (res.status === 404) { - res = await worker.fetch( - new Request('https://internal/__queue', { + const relay = async (token: string | undefined): Promise => { + const queueHeaders: Record = { 'Content-Type': 'application/json' }; + if (token) { + queueHeaders['x-void-internal'] = token; + } + const res = await worker.fetch( + new Request('https://internal/__void/queue', { method: 'POST', headers: queueHeaders, body, }), ); + // Legacy: fall back for workers deployed before the /__void/ migration. + // TODO: remove once all deployed workers use /__void/* paths. + if (res.status === 404) { + return worker.fetch( + new Request('https://internal/__queue', { + method: 'POST', + headers: queueHeaders, + body, + }), + ); + } + return res; + }; + + let res = await relay(proxyToken); + // While a token rotation is in progress the Worker may hold either + // token. Its reject comes before any queue handler runs, so a second + // attempt with the other token cannot deliver the batch twice. + if (proxyTokenFallback && (await isQueueTokenRejection(res))) { + await res.body?.cancel(); + res = await relay(proxyTokenFallback); } if (!res.ok) { @@ -2025,6 +2041,23 @@ export default { }, } satisfies ExportedHandler; +/** The generated queue route's reject for a token it does not hold. */ +async function isQueueTokenRejection(res: Response): Promise { + if (res.status !== 401) { + return false; + } + try { + const data: unknown = await res.clone().json(); + return ( + typeof data === 'object' && + data !== null && + (data as { error?: unknown }).error === 'unauthorized' + ); + } catch { + return false; + } +} + class StaticAssetReadError extends Error { constructor(cause: unknown) { super('Static asset storage read failed', { cause }); diff --git a/platform/packages/dispatch/test/unit/index.test.ts b/platform/packages/dispatch/test/unit/index.test.ts index 3354d3221..e0b13cb7f 100644 --- a/platform/packages/dispatch/test/unit/index.test.ts +++ b/platform/packages/dispatch/test/unit/index.test.ts @@ -4555,6 +4555,112 @@ describe('queue relay', () => { } }); + describe('fallback token', () => { + const rotatingRouting = JSON.stringify({ + workerName: 'user-app', + projectId: 'proj-1', + queueName: 'emails', + proxyToken: 'old', + proxyTokenFallback: 'new', + }); + const unauthorized = () => Response.json({ error: 'unauthorized' }, { status: 401 }); + + function setup(routingRaw: string, respond: (token: string | null) => Promise) { + const batch = createMockBatch('test-queue', [{ id: 'm1', body: 'hello' }]); + const { env, workerFetch } = createQueueEnv({ routingRaw }); + const seen: Array<{ token: string | null; body: string }> = []; + workerFetch.mockImplementation(async (req: Request) => { + const token = req.headers.get('x-void-internal'); + seen.push({ token, body: await req.clone().text() }); + return respond(token); + }); + return { batch, env, seen }; + } + + it('relays once more with the fallback when the Worker rejects the token', async () => { + const { batch, env, seen } = setup(rotatingRouting, async (token) => + token === 'new' ? Response.json({ ok: true }) : unauthorized(), + ); + + await app.queue(batch, env); + + expect(seen.map((call) => call.token)).toEqual(['old', 'new']); + expect(seen[1].body).toBe(seen[0].body); + expect(batch.ackAll).toHaveBeenCalledOnce(); + expect(batch.retryAll).not.toHaveBeenCalled(); + }); + + it('retries the batch without a second relay when there is no fallback', async () => { + const routing = JSON.stringify({ + workerName: 'user-app', + projectId: 'proj-1', + queueName: 'emails', + proxyToken: 'old', + }); + const { batch, env, seen } = setup(routing, async () => unauthorized()); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await app.queue(batch, env); + } finally { + errSpy.mockRestore(); + } + + expect(seen).toHaveLength(1); + expect(batch.retryAll).toHaveBeenCalledOnce(); + }); + + it.each([ + ['a 401 with another body', () => Response.json({ error: 'expired' }, { status: 401 })], + ['a 401 with a non-JSON body', () => new Response('unauthorized', { status: 401 })], + ['a 403 unauthorized', () => Response.json({ error: 'unauthorized' }, { status: 403 })], + ['a 500', () => new Response('boom', { status: 500 })], + ])('does not relay again after %s', async (_label, response) => { + const { batch, env, seen } = setup(rotatingRouting, async () => response()); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await app.queue(batch, env); + } finally { + errSpy.mockRestore(); + } + + expect(seen).toHaveLength(1); + expect(batch.retryAll).toHaveBeenCalledOnce(); + }); + + it('does not relay again after a thrown fetch', async () => { + const { batch, env, seen } = setup(rotatingRouting, async () => { + throw new Error('connection reset'); + }); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await app.queue(batch, env); + } finally { + errSpy.mockRestore(); + } + + expect(seen).toHaveLength(1); + expect(batch.retryAll).toHaveBeenCalledOnce(); + }); + + it('retries the batch after two relays when both tokens are rejected', async () => { + const { batch, env, seen } = setup(rotatingRouting, async () => unauthorized()); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await app.queue(batch, env); + } finally { + errSpy.mockRestore(); + } + + expect(seen.map((call) => call.token)).toEqual(['old', 'new']); + expect(batch.retryAll).toHaveBeenCalledOnce(); + expect(batch.ackAll).not.toHaveBeenCalled(); + }); + }); + it('forwards x-void-internal on legacy /__queue fallback too', async () => { const routing = JSON.stringify({ workerName: 'user-app', From 3af9d921d669ee90130c14d740711965d44a3963 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 02:05:29 +0800 Subject: [PATCH 07/12] Skip unknown cleanup jobs instead of deleting them The scheduled cleanup runner now claims only the job kinds it knows how to run. Rows of any other kind keep their attempts and payload instead of being claimed and deleted, so they cannot fill the retry batch and they survive a platform rollback. executeCleanupJob throws on an unknown kind as a guard. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../packages/api/src/provisioning/cleanup.ts | 22 +++++++++++++- .../api/test/integration/api-delete.test.ts | 29 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/platform/packages/api/src/provisioning/cleanup.ts b/platform/packages/api/src/provisioning/cleanup.ts index 03c49c63e..09efa809f 100644 --- a/platform/packages/api/src/provisioning/cleanup.ts +++ b/platform/packages/api/src/provisioning/cleanup.ts @@ -121,6 +121,19 @@ async function ignoreMissingProviderObject(operation: () => Promise): Prom } } +// Kinds the scheduled retry runner executes. Rows of any other kind stay in the +// table untouched, so a job kind added by a newer platform version survives a +// rollback and never fills the retry batch. +const RETRYABLE_CLEANUP_JOB_KINDS = [ + 'cf-resource', + 'worker', + 'worker-bundle', + 'project-assets', + 'routing-key', + 'cron', + 'custom-hostname', +] as const satisfies ReadonlyArray; + async function executeCleanupJob( env: ApiEnv['Bindings'], job: ProjectCleanupJobInput, @@ -172,6 +185,8 @@ async function executeCleanupJob( ), ); return; + default: + throw new Error(`unknown cleanup job kind: ${String((job as { kind: string }).kind)}`); } } @@ -750,7 +765,12 @@ export async function reconcileProjectCleanupJobs(db: Db, env: ApiEnv['Bindings' const due = await db .select() .from(projectCleanupJobs) - .where(lte(projectCleanupJobs.next_attempt_at, drizzleSql`now()`)) + .where( + and( + lte(projectCleanupJobs.next_attempt_at, drizzleSql`now()`), + inArray(projectCleanupJobs.kind, [...RETRYABLE_CLEANUP_JOB_KINDS]), + ), + ) .orderBy(asc(projectCleanupJobs.created_at), asc(projectCleanupJobs.target)) .limit(50); let next = 0; diff --git a/platform/packages/api/test/integration/api-delete.test.ts b/platform/packages/api/test/integration/api-delete.test.ts index 73bbe32c4..0f70d410d 100644 --- a/platform/packages/api/test/integration/api-delete.test.ts +++ b/platform/packages/api/test/integration/api-delete.test.ts @@ -12,6 +12,7 @@ import { import * as db from '../../src/db.ts'; import app, { FAST_CRON, scheduled } from '../../src/index.ts'; import * as projectAccess from '../../src/project-access.ts'; +import { reconcileProjectCleanupJobs } from '../../src/provisioning/cleanup.ts'; import { resources, projects, @@ -472,6 +473,34 @@ describe('DELETE /projects/:id', () => { expect(finished.rows).toEqual([]); }); + it('leaves cleanup jobs of an unknown kind unclaimed while running known jobs', async () => { + await env.db.execute(sql` + INSERT INTO project_cleanup_jobs (target, project_id, kind, payload, next_attempt_at) + VALUES ('unknown:1', 'proj_unknown', 'something-else', '{}'::jsonb, now() - interval '1 minute') + `); + await env.ROUTING.put('cleanup-filter-test', 'x'); + await env.db.execute(sql` + INSERT INTO project_cleanup_jobs (target, project_id, kind, payload, next_attempt_at) + VALUES ( + 'routing-key:proj_unknown:cleanup-filter-test', + 'proj_unknown', + 'routing-key', + '{"key":"cleanup-filter-test"}'::jsonb, + now() - interval '1 minute' + ) + `); + + await reconcileProjectCleanupJobs(env.db, env as unknown as Parameters[1]); + + expect(await env.ROUTING.get('cleanup-filter-test')).toBeNull(); + const remaining = await env.db.execute<{ + target: string; + attempts: number; + last_error: string | null; + }>(sql`SELECT target, attempts, last_error FROM project_cleanup_jobs ORDER BY target`); + expect(remaining.rows).toEqual([{ target: 'unknown:1', attempts: 0, last_error: null }]); + }); + it('treats an already-deleted CF resource as successfully cleaned up', async () => { const user = await createTestUser(env.db); const project = await createTestProject(env.db, user.id, 'warn-app'); From 4532d3e7c337300a48391e919a53aa5bff1ac34a Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 02:07:04 +0800 Subject: [PATCH 08/12] Document proxy token generations for maintainers Describe the legacy and v2 project token forms, the per-project accept floor in ROUTING, how the proxy picks tokens for tenant Workers, and the proxy-only key on the internal metadata routes. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/README.md | 6 ++++ platform/packages/proxy/README.md | 46 +++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/platform/packages/api/README.md b/platform/packages/api/README.md index 507ee81ab..26b3fa0c2 100644 --- a/platform/packages/api/README.md +++ b/platform/packages/api/README.md @@ -141,8 +141,14 @@ HMAC. See [Platform Development](../../../docs/guide/platform/development/runtim - `POST /internal/revalidate/:projectId` - `POST /internal/resolve-resource/:projectId` - `POST /internal/resolve-worker/:projectId` — resolve the active deployment's versioned worker; static active deployments report that no worker exists +- `POST /internal/verify-prerender-build/:projectId` — live-check that a prerender JWT's user, build, and deploy credential are still active - `POST /internal/verify-owner/:projectId` — legacy URL that live-checks the caller's project deploy capability +`resolve-resource`, `resolve-worker` and `verify-prerender-build` authenticate +the proxy with its own key (`x-void-proxy-internal`); a request without it is +checked like a project token. `revalidate` checks the project token forwarded +by the proxy against the project's accept floor (see the proxy README). + ### Operator API `/operator/v1` exposes administrative operations as JSON. `POST /session` diff --git a/platform/packages/proxy/README.md b/platform/packages/proxy/README.md index 42568d81d..834ae654f 100644 --- a/platform/packages/proxy/README.md +++ b/platform/packages/proxy/README.md @@ -24,11 +24,11 @@ Deployed user worker Proxy Downstream Project requests use these authentication paths: -| Path | When | Auth | -| --------------- | ------------------ | ------------------------------------------------------------------------------------- | -| Service binding | Deployed Void apps | HMAC-signed `projectId` token (`aiToken`); `projectId` must be a generated project id | -| HTTPS | Local development | `Authorization: Bearer ` | -| HTTPS | Managed prerender | Five-minute, project-scoped prerender JWT | +| Path | When | Auth | +| --------------- | ------------------ | ------------------------------------------------------------------------------------------------------------ | +| Service binding | Deployed Void apps | Project token (`aiToken`), checked against the project's accept floor; see [Project tokens](#project-tokens) | +| HTTPS | Local development | `Authorization: Bearer ` | +| HTTPS | Managed prerender | Five-minute, project-scoped prerender JWT | Project-scoped POST routes require `projectId` in the JSON body. The public `GET /health` route has no project body, and `/email/verify-receipt` checks a @@ -42,6 +42,40 @@ routes accept up to 8 MiB. Email sends retain their 8 MiB envelope limit and receipt verification accepts up to 2 KiB. A stalled body times out after five seconds; non-email routes return 413 for oversized bodies and 408 for timeouts. +### Project tokens + +Each deployed Worker holds its project token in the `__VOID_PROXY_TOKEN` +binding. Both forms are derived from `JWT_SECRET`: + +| Form | Generation | Value | +| ---------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| Legacy | 0 | base64url `HMAC-SHA256(JWT_SECRET, projectId)` | +| `v2..` | `` | `` = base64url `HMAC-SHA256(K, "void-proxy-token:v2:" + projectId + ":" + gen)`, `K` = HKDF of `JWT_SECRET` (salt `void-proxy-token`, info `v2`) | + +The ROUTING key `proxy-token:{projectId}` holds the project's accept floor, +`{"gen": N}`. A token is accepted when its MAC is valid and its generation is at +least `N`. A missing key means floor 0, so the legacy token is accepted. The +`projectId` must be a generated project id, and the MAC is checked before the +floor is read, so an invalid token never reads KV. A floor that cannot be read +or does not parse fails closed with 503 `project token state unavailable`. Each isolate caches a floor for 15 seconds +and reads KV with `cacheTtl: 30`, so with KV propagation a raised floor rejects +older tokens everywhere within about 105 seconds. + +Service-binding auth and `/email/verify-receipt` use this check. `/revalidate` +forwards the token, and the API checks it against the same floor. + +When the proxy calls a tenant Worker's `/__void/*` handler, it sends the token +for the generation `resolve-worker` reports (`proxyTokenGen`), or for the floor +when the API does not report one. If that Worker answers 403 +`{"error":"unauthorized"}`, the proxy retries once with the previous +generation. It never sends a generation above the issued one. + +The proxy authenticates its own `resolve-worker`, `resolve-resource`, and +`verify-prerender-build` calls with `x-void-proxy-internal`, an HMAC under a +separate HKDF key (salt `void-proxy-internal`, info `v1`) bound to the route and +project. That value never leaves the platform. The proxy also sends the legacy +`x-void-internal` header so an older API keeps working. + ## Endpoints ### AI @@ -80,7 +114,7 @@ The remote-binding path is: | ------------------ | --------------------------------------------------------- | | `POST /revalidate` | `{ paths?: string[], all?: boolean, projectId, aiToken }` | -The proxy forwards the request to `API.fetch("https://api/internal/revalidate/:projectId")` with `x-void-internal`. +The proxy forwards the request to `API.fetch("https://api/internal/revalidate/:projectId")` with the project token in `x-void-internal`. The API accepts it under the same floor as the proxy. ### Email From 2729af31792f7ae6e49dd5e35354445de9c2c4db Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 02:13:13 +0800 Subject: [PATCH 09/12] Cover floor 2 in proxy token state table and rewrap proxy README Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/test/unit/proxy-token.test.ts | 1 + platform/packages/proxy/README.md | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/platform/packages/api/test/unit/proxy-token.test.ts b/platform/packages/api/test/unit/proxy-token.test.ts index fbba55351..665bc56d9 100644 --- a/platform/packages/api/test/unit/proxy-token.test.ts +++ b/platform/packages/api/test/unit/proxy-token.test.ts @@ -160,6 +160,7 @@ describe('state table', () => { { raw: null, accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, { raw: '{"gen":0}', accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, { raw: '{"gen":1}', accepted: [1, 2], rejected: [0], send: [1, 0], issue: 1 }, + { raw: '{"gen":2}', accepted: [2, 3], rejected: [0, 1], send: [2, 1], issue: 2 }, { raw: '{"gen":3}', accepted: [3, 4], rejected: [0, 1, 2], send: [3, 2], issue: 3 }, ])('floor $raw', async ({ raw, accepted, rejected, send, issue }) => { const { routing } = kvStub(raw === null ? {} : { [proxyTokenStateKey(PROJECT)]: raw }); diff --git a/platform/packages/proxy/README.md b/platform/packages/proxy/README.md index 834ae654f..bd21ab382 100644 --- a/platform/packages/proxy/README.md +++ b/platform/packages/proxy/README.md @@ -57,9 +57,10 @@ The ROUTING key `proxy-token:{projectId}` holds the project's accept floor, least `N`. A missing key means floor 0, so the legacy token is accepted. The `projectId` must be a generated project id, and the MAC is checked before the floor is read, so an invalid token never reads KV. A floor that cannot be read -or does not parse fails closed with 503 `project token state unavailable`. Each isolate caches a floor for 15 seconds -and reads KV with `cacheTtl: 30`, so with KV propagation a raised floor rejects -older tokens everywhere within about 105 seconds. +or does not parse fails closed with 503 `project token state unavailable`. Each +isolate caches a floor for 15 seconds and reads KV with `cacheTtl: 30`, so with +KV propagation a raised floor rejects older tokens everywhere within about 105 +seconds. Service-binding auth and `/email/verify-receipt` use this check. `/revalidate` forwards the token, and the API checks it against the same floor. From bc360f05fe948573559d129d48b917526e078a02 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 02:21:02 +0800 Subject: [PATCH 10/12] Bound the proxy's read of a tenant 403 before retrying The proxy checks a 403 from the tenant Worker for the binding handler's token reject before retrying with the previous token. That check parsed the whole cloned body, so a Worker could make the shared proxy buffer an arbitrarily large response. Skip bodies that declare more than 256 bytes and stop reading a stream once it passes that size; the response returned to the caller still streams unbuffered. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/proxy/src/dispatch.ts | 32 +++++++++- .../packages/proxy/test/unit/dispatch.test.ts | 59 +++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/platform/packages/proxy/src/dispatch.ts b/platform/packages/proxy/src/dispatch.ts index 487b3c9e1..17659de47 100644 --- a/platform/packages/proxy/src/dispatch.ts +++ b/platform/packages/proxy/src/dispatch.ts @@ -65,13 +65,37 @@ function isGeneration(value: unknown): value is number { return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; } -/** The binding handler's reject for a token it does not hold, sent before any handler runs. */ +/** The binding handler's reject is `{"error":"unauthorized"}`; anything larger is not it. */ +const TOKEN_REJECTION_MAX_BYTES = 256; + +/** + * The binding handler's reject for a token it does not hold, sent before any handler runs. + * The tenant Worker controls this response, so only a small prefix is ever read and the + * returned response still streams unbuffered. + */ async function isTokenRejection(res: Response): Promise { - if (res.status !== 403) { + if (res.status !== 403 || !res.body) { return false; } + if (Number(res.headers.get('content-length')) > TOKEN_REJECTION_MAX_BYTES) { + return false; + } + const reader = res.clone().body!.getReader(); + const chunks: Array = []; + let size = 0; try { - const data = await res.clone().json(); + for (;;) { + const chunk = await reader.read(); + if (chunk.done) { + break; + } + size += chunk.value.byteLength; + if (size > TOKEN_REJECTION_MAX_BYTES) { + return false; + } + chunks.push(chunk.value); + } + const data: unknown = JSON.parse(await new Blob(chunks).text()); return ( typeof data === 'object' && data !== null && @@ -79,6 +103,8 @@ async function isTokenRejection(res: Response): Promise { ); } catch { return false; + } finally { + void reader.cancel().catch(() => {}); } } diff --git a/platform/packages/proxy/test/unit/dispatch.test.ts b/platform/packages/proxy/test/unit/dispatch.test.ts index 33d77d6b0..fdd9a2d7d 100644 --- a/platform/packages/proxy/test/unit/dispatch.test.ts +++ b/platform/packages/proxy/test/unit/dispatch.test.ts @@ -791,6 +791,65 @@ describe('proxy token generations', () => { expect(sent).toHaveLength(1); }); + it('retries a reject that is streamed without a declared length', async () => { + const legacy = await tok(0); + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: (token) => + token === legacy + ? Response.json('hello') + : new Response(new Blob(['{"error":"unauthorized"}']).stream(), { status: 403 }), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(200); + expect(sent.map((s) => s.token)).toEqual([await tok(1), legacy]); + }); + + it('does not read a 403 that declares a large body', async () => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => + new Response('{"error":"unauthorized"}', { + status: 403, + headers: { 'content-length': String(64 * 1024 * 1024) }, + }), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(403); + expect(sent).toHaveLength(1); + }); + + it('stops reading a streamed 403 once it outgrows a reject', async () => { + let pulls = 0; + const chunk = new TextEncoder().encode(' '.repeat(64)); + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, + respond: () => + new Response( + new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(chunk); + }, + }), + { status: 403 }, + ), + }); + + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + + expect(res?.status).toBe(403); + expect(sent).toHaveLength(1); + expect(pulls).toBeLessThan(16); + const reader = res!.body!.getReader(); + expect((await reader.read()).value).toEqual(chunk); + await reader.cancel(); + }); + it('returns the second reject when the Worker holds neither token', async () => { const { env, sent } = setup({ resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, From acb2624452ad7d459d2f9e411a6e27f50270ca47 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 16:15:07 +0800 Subject: [PATCH 11/12] Drop the version marker from proxy tokens and keys Tokens for generation 1 and above are now `.`. The dot still keeps them apart from the 43-character original token. The HKDF labels and signed messages carry the key's purpose instead of a version: the tenant key uses salt `void-proxy-token`, info `proxy-token`, and signs `void-proxy-token::`; the proxy internal key uses salt `void-proxy-internal`, info `proxy-internal`, and signs `void-proxy-internal\n\n`. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/src/proxy-token.ts | 25 ++++++++----- .../api-internal-project-token.test.ts | 2 +- .../api/test/unit/proxy-token.test.ts | 37 ++++++++++--------- platform/packages/proxy/README.md | 12 +++--- .../packages/proxy/test/unit/proxy.test.ts | 2 +- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/platform/packages/api/src/proxy-token.ts b/platform/packages/api/src/proxy-token.ts index 3cd367ace..794801168 100644 --- a/platform/packages/api/src/proxy-token.ts +++ b/platform/packages/api/src/proxy-token.ts @@ -8,8 +8,9 @@ import { isProjectId } from './id.ts'; * * - legacy (generation 0): base64url(HMAC-SHA256(JWT_SECRET, projectId)), * 43 characters, the bytes `hmacSignState` produces; - * - `v2..`: the MAC is HMAC-SHA256 under a key derived from - * JWT_SECRET with HKDF, over the project id and the generation. + * - `.` (generation 1 and above): the MAC is HMAC-SHA256 under a + * key derived from JWT_SECRET with HKDF, over the project id and the + * generation. The dot keeps it apart from the 43-character legacy token. * * ROUTING `proxy-token:` holds the accept floor `{"gen": F}`. A * token is accepted when its MAC is valid and its generation is at least F. @@ -143,20 +144,26 @@ async function hkdfHmacKey(secret: string, salt: string, info: string): Promise< ); } +// JWT_SECRET signs login JWTs, OAuth/link state and the legacy token as a raw +// HMAC key. The two keys below are HKDF outputs with their own salt and info, +// so they differ from that raw key and from each other: a tenant token never +// verifies as a proxy internal value, and neither verifies as any raw-key MAC. function proxyTokenKey(secret: string): Promise { - return cachedKey(proxyTokenKeys, secret, () => hkdfHmacKey(secret, 'void-proxy-token', 'v2')); + return cachedKey(proxyTokenKeys, secret, () => + hkdfHmacKey(secret, 'void-proxy-token', 'proxy-token'), + ); } function proxyInternalKey(secret: string): Promise { return cachedKey(proxyInternalKeys, secret, () => - hkdfHmacKey(secret, 'void-proxy-internal', 'v1'), + hkdfHmacKey(secret, 'void-proxy-internal', 'proxy-internal'), ); } // --- Encoding --- const MAC = /^[A-Za-z0-9_-]{43}$/; -const V2_TOKEN = /^v2\.([1-9][0-9]{0,15})\.([A-Za-z0-9_-]{43})$/; +const GENERATION_TOKEN = /^([1-9][0-9]{0,15})\.([A-Za-z0-9_-]{43})$/; function encodeBase64url(bytes: ArrayBuffer): string { let binary = ''; @@ -181,7 +188,7 @@ async function verify(key: Promise, message: string, mac: string): Pr } function proxyTokenMessage(projectId: string, gen: number): string { - return `void-proxy-token:v2:${projectId}:${gen}`; + return `void-proxy-token:${projectId}:${gen}`; } // --- Tenant tokens --- @@ -201,7 +208,7 @@ export async function deriveProxyToken( if (gen === 0) { return sign(legacyKey(jwtSecret), projectId); } - return `v2.${gen}.${await sign(proxyTokenKey(jwtSecret), proxyTokenMessage(projectId, gen))}`; + return `${gen}.${await sign(proxyTokenKey(jwtSecret), proxyTokenMessage(projectId, gen))}`; } /** The token's generation when its MAC is valid for `projectId`, else null. No I/O. */ @@ -216,7 +223,7 @@ export async function proxyTokenGeneration( if (MAC.test(token)) { return (await verify(legacyKey(jwtSecret), projectId, token)) ? 0 : null; } - const match = V2_TOKEN.exec(token); + const match = GENERATION_TOKEN.exec(token); if (!match) { return null; } @@ -343,7 +350,7 @@ export const PROXY_INTERNAL_AUTH_HEADER = 'x-void-proxy-internal'; export type ProxyInternalRoute = 'resolve-worker' | 'resolve-resource' | 'verify-prerender-build'; function proxyInternalMessage(route: ProxyInternalRoute, projectId: string): string { - return `void-proxy-internal:v1\n${route}\n${projectId}`; + return `void-proxy-internal\n${route}\n${projectId}`; } /** diff --git a/platform/packages/api/test/integration/api-internal-project-token.test.ts b/platform/packages/api/test/integration/api-internal-project-token.test.ts index 6d7c92259..4b0160fc5 100644 --- a/platform/packages/api/test/integration/api-internal-project-token.test.ts +++ b/platform/packages/api/test/integration/api-internal-project-token.test.ts @@ -216,7 +216,7 @@ describe('project token generations', () => { expect(run).not.toHaveBeenCalled(); }); - it('accepts the legacy and v2 tokens at /internal/revalidate while no floor is set', async () => { + it('accepts the legacy and generation 1 tokens at /internal/revalidate while no floor is set', async () => { const project = await liveProject(); for (const gen of [0, 1]) { const response = await internalRequest( diff --git a/platform/packages/api/test/unit/proxy-token.test.ts b/platform/packages/api/test/unit/proxy-token.test.ts index 665bc56d9..3244dfa70 100644 --- a/platform/packages/api/test/unit/proxy-token.test.ts +++ b/platform/packages/api/test/unit/proxy-token.test.ts @@ -30,12 +30,12 @@ const PROJECT = 'proj_000000000001'; const OTHER_PROJECT = 'proj_000000000002'; const TOK_0 = 'ZMRzpVRsaOJSG_unVlA0oqSaz2kApOohvao_XGR8Q0c'; -const TOK_1 = 'v2.1.3W5_i2g79pUCXyzxWPQWmTqscSNtC_2vuHC84z775Y8'; -const TOK_2 = 'v2.2.Vx_qycCHPx2BoYvQrbQ7JrhbONKU7f0jNK23B0YUhOs'; +const TOK_1 = '1.tSB1ya3fUJXjnL2nfUht0aCBRh9d1pFCrH_rJIJGGWo'; +const TOK_2 = '2.Ko4ZEz_VQ8IZH70koONArHkURyEHZUwLJsqFsGJgwyE'; const INTERNAL: Record = { - 'resolve-worker': 'fSljhl8vCcBSR-jWXklS-KDL4K8lFb_wPaRAu-aKPv0', - 'resolve-resource': 'gNOx85Zh_di7yHFjG8AlV0QQ8PbHYs1ipC2ukAl7i5Y', - 'verify-prerender-build': 'A7F1IjLUbEWq0atQH3W5dsR1DidJ6ZgqU6QQ3lehBbc', + 'resolve-worker': 'Phjba6joMrR31HPBsOamhNdtbT3__9Fg6zkd34dJiS0', + 'resolve-resource': 'qqnenNPCqY-C4RMAfPn0AUdXlvKS0tvkDcC8lyjTYck', + 'verify-prerender-build': 'L-_KXs8KdULytY5ibjHuH095R-1L3OM91NokMoj6828', }; const ROUTES = Object.keys(INTERNAL) as Array; @@ -47,15 +47,15 @@ function nodeTok(gen: number, projectId = PROJECT): string { if (gen === 0) { return base64url(createHmac('sha256', SECRET).update(projectId).digest()); } - const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-token', 'v2', 32)); - const mac = createHmac('sha256', key).update(`void-proxy-token:v2:${projectId}:${gen}`).digest(); - return `v2.${gen}.${base64url(mac)}`; + const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-token', 'proxy-token', 32)); + const mac = createHmac('sha256', key).update(`void-proxy-token:${projectId}:${gen}`).digest(); + return `${gen}.${base64url(mac)}`; } function nodeInternal(route: ProxyInternalRoute, projectId = PROJECT): string { - const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-internal', 'v1', 32)); + const key = Buffer.from(hkdfSync('sha256', SECRET, 'void-proxy-internal', 'proxy-internal', 32)); return base64url( - createHmac('sha256', key).update(`void-proxy-internal:v1\n${route}\n${projectId}`).digest(), + createHmac('sha256', key).update(`void-proxy-internal\n${route}\n${projectId}`).digest(), ); } @@ -111,7 +111,7 @@ describe('vectors', () => { }); it('keeps internal values distinct from each other and from tenant tokens', () => { - const values = [...Object.values(INTERNAL), TOK_0, TOK_1.slice('v2.1.'.length)]; + const values = [...Object.values(INTERNAL), TOK_0, TOK_1.slice('1.'.length)]; expect(new Set(values).size).toBe(values.length); }); @@ -193,7 +193,7 @@ describe('state table', () => { }); describe('verifyProxyToken', () => { - it('accepts legacy and v2 tokens when no floor is set', async () => { + it('accepts the legacy and generation tokens when no floor is set', async () => { expect(await verify(TOK_0).result).toBe(true); expect(await verify(TOK_1).result).toBe(true); }); @@ -218,14 +218,14 @@ describe('verifyProxyToken', () => { }); it('rejects a MAC relabelled with another generation', async () => { - const relabelled = `v2.2.${TOK_1.slice('v2.1.'.length)}`; + const relabelled = `2.${TOK_1.slice('1.'.length)}`; const { result, readState } = verify(relabelled); expect(await result).toBe(false); expect(readState).not.toHaveBeenCalled(); }); it('rejects a bad MAC without reading the floor', async () => { - const badMac = `v2.1.${'A'.repeat(43)}`; + const badMac = `1.${'A'.repeat(43)}`; for (const token of [badMac, 'A'.repeat(43)]) { const { result, readState } = verify(token); expect(await result).toBe(false); @@ -234,14 +234,15 @@ describe('verifyProxyToken', () => { }); it.each([ - ['leading zero generation', `v2.01.${TOK_1.slice('v2.1.'.length)}`], - ['leading zero and short MAC', 'v2.01.x'], + ['generation zero in the dotted form', `0.${TOK_1.slice('1.'.length)}`], + ['leading zero generation', `01.${TOK_1.slice('1.'.length)}`], + ['leading zero and short MAC', '01.x'], ['42-char MAC', TOK_0.slice(0, 42)], ['44-char MAC', `${TOK_0}A`], ['trailing padding', `${TOK_0}=`], ['standard base64 alphabet', `+${TOK_0.slice(1)}`], ['empty token', ''], - ['unsafe generation', `v2.9999999999999999.${TOK_1.slice('v2.1.'.length)}`], + ['unsafe generation', `9999999999999999.${TOK_1.slice('1.'.length)}`], ])('rejects a malformed token (%s) without reading the floor', async (_name, token) => { const { result, readState } = verify(token); expect(await result).toBe(false); @@ -416,7 +417,7 @@ describe('verifyProxyInternalAuth', () => { it.each([ ['42 chars', INTERNAL['resolve-worker'].slice(0, 42)], ['44 chars', `${INTERNAL['resolve-worker']}A`], - ['a v2 token shape', 'v2.1.x'], + ['a generation token shape', '1.x'], ['a tenant token', TOK_1], ['the legacy token', TOK_0], ['empty', ''], diff --git a/platform/packages/proxy/README.md b/platform/packages/proxy/README.md index bd21ab382..ecd1f3d45 100644 --- a/platform/packages/proxy/README.md +++ b/platform/packages/proxy/README.md @@ -47,10 +47,10 @@ seconds; non-email routes return 413 for oversized bodies and 408 for timeouts. Each deployed Worker holds its project token in the `__VOID_PROXY_TOKEN` binding. Both forms are derived from `JWT_SECRET`: -| Form | Generation | Value | -| ---------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| Legacy | 0 | base64url `HMAC-SHA256(JWT_SECRET, projectId)` | -| `v2..` | `` | `` = base64url `HMAC-SHA256(K, "void-proxy-token:v2:" + projectId + ":" + gen)`, `K` = HKDF of `JWT_SECRET` (salt `void-proxy-token`, info `v2`) | +| Form | Generation | Value | +| ------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Legacy | 0 | base64url `HMAC-SHA256(JWT_SECRET, projectId)` | +| `.` | 1 and up | `` = base64url `HMAC-SHA256(K, "void-proxy-token:" + projectId + ":" + gen)`, `K` = HKDF of `JWT_SECRET` (salt `void-proxy-token`, info `proxy-token`) | The ROUTING key `proxy-token:{projectId}` holds the project's accept floor, `{"gen": N}`. A token is accepted when its MAC is valid and its generation is at @@ -73,8 +73,8 @@ generation. It never sends a generation above the issued one. The proxy authenticates its own `resolve-worker`, `resolve-resource`, and `verify-prerender-build` calls with `x-void-proxy-internal`, an HMAC under a -separate HKDF key (salt `void-proxy-internal`, info `v1`) bound to the route and -project. That value never leaves the platform. The proxy also sends the legacy +separate HKDF key (salt `void-proxy-internal`, info `proxy-internal`) bound to +the route and project. That value never leaves the platform. The proxy also sends the legacy `x-void-internal` header so an older API keeps working. ## Endpoints diff --git a/platform/packages/proxy/test/unit/proxy.test.ts b/platform/packages/proxy/test/unit/proxy.test.ts index 5af5b4699..6b1f7f048 100644 --- a/platform/packages/proxy/test/unit/proxy.test.ts +++ b/platform/packages/proxy/test/unit/proxy.test.ts @@ -1153,7 +1153,7 @@ describe('proxy worker', () => { vi.restoreAllMocks(); }); - it('accepts the legacy and v2 tokens while no floor is set', async () => { + it('accepts the legacy and generation 1 tokens while no floor is set', async () => { for (const gen of [0, 1]) { clearProxyTokenStateCache(); const env = createEnv(); From a1431cc57bc4229bf2a0f3fe2af5fe715d08d79e Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 7 Oct 2026 19:57:44 +0800 Subject: [PATCH 12/12] Send one proxy token and never replay a rejected request A tenant Worker runs its own middleware and routes before the generated /__void/* binding, cron and queue handlers check the proxy token. A request rejected with 401/403 has therefore already run user code, and sending it again with a second token would run that code twice. The proxy, the cron fire consumer and the queue relay now send exactly one token per call. The proxy sends the generation resolve-worker reports, or the floor's generation when none is reported. A rejected call is returned (proxy), acked with AUTH_DRIFT (cron) or retried by the queue as before. The fallback token fields, the 403 body sniffing and sendGenerations are removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- platform/packages/api/src/cron-fires-queue.ts | 45 +----- .../packages/api/src/cron-scheduler-client.ts | 2 - platform/packages/api/src/cron-scheduler.ts | 5 +- platform/packages/api/src/proxy-token.ts | 11 -- .../api/test/unit/cron-fires-queue.test.ts | 147 ++--------------- .../api/test/unit/cron-scheduler.test.ts | 67 -------- .../api/test/unit/proxy-token.test.ts | 20 +-- platform/packages/dispatch/src/index.ts | 67 ++------ .../packages/dispatch/test/unit/index.test.ts | 102 ++---------- platform/packages/proxy/README.md | 7 +- platform/packages/proxy/src/dispatch.ts | 71 ++------- platform/packages/proxy/src/proxy-token.ts | 15 +- .../packages/proxy/test/unit/dispatch.test.ts | 150 +++--------------- 13 files changed, 99 insertions(+), 610 deletions(-) diff --git a/platform/packages/api/src/cron-fires-queue.ts b/platform/packages/api/src/cron-fires-queue.ts index c78d088ba..00a91b460 100644 --- a/platform/packages/api/src/cron-fires-queue.ts +++ b/platform/packages/api/src/cron-fires-queue.ts @@ -15,8 +15,6 @@ export type CronFireMessage = Readonly<{ cron: string; scheduledTime: number; proxyToken?: string; - /** Second token to try when the Worker rejects `proxyToken`. */ - proxyTokenFallback?: string; }>; type InvokeOutcome = Readonly<{ @@ -26,27 +24,8 @@ type InvokeOutcome = Readonly<{ bodySnippet?: string; /** True only when the worker script no longer exists. */ unreachable?: boolean; - /** True when the worker refused the token before running the cron handler. */ - tokenRejected?: boolean; }>; -/** The generated cron route's reject for a token it does not hold. */ -function isTokenRejection(status: number, text: string): boolean { - if (status !== 401 && status !== 403) { - return false; - } - try { - const data: unknown = JSON.parse(text); - return ( - typeof data === 'object' && - data !== null && - (data as { error?: unknown }).error === 'unauthorized' - ); - } catch { - return false; - } -} - /** * Cloudflare's WfP dispatch surfaces a missing-worker error as a thrown * Error whose message starts with "Worker not found". This predicate is @@ -61,11 +40,10 @@ function isWorkerNotFoundError(err: unknown): boolean { async function invokeWorker( env: ApiEnv['Bindings'], message: CronFireMessage, - token: string | undefined, ): Promise { const headers: Record = { 'Content-Type': 'application/json' }; - if (token) { - headers['x-void-internal'] = token; + if (message.proxyToken) { + headers['x-void-internal'] = message.proxyToken; } const body = JSON.stringify({ cron: message.cron, scheduledTime: message.scheduledTime }); @@ -102,7 +80,6 @@ async function invokeWorker( responseReceived: true, status: res.status, bodySnippet: text.slice(0, 200), - tokenRejected: isTokenRejection(res.status, text), }; } // Fully consume the response body so the dispatch subrequest finalizes cleanly, @@ -126,13 +103,7 @@ export async function processCronFiresBatch( ): Promise { await Promise.allSettled( batch.messages.map(async (msg) => { - let outcome = await invokeWorker(env, msg.body, msg.body.proxyToken); - // While a token rotation is in progress the worker may hold either - // token. Its reject comes before the cron handler runs, so a second - // attempt with the other token cannot run the handler twice. - if (outcome.tokenRejected && msg.body.proxyTokenFallback) { - outcome = await invokeWorker(env, msg.body, msg.body.proxyTokenFallback); - } + const outcome = await invokeWorker(env, msg.body); // A Response came back — the user worker's handler ran (or the runtime // made a routing decision, e.g. 404 on missing handler). Either way the @@ -142,11 +113,11 @@ export async function processCronFiresBatch( if (outcome.responseReceived) { const status = outcome.status; if (status === 401 || status === 403) { - // Auth drift: the user worker's baked __VOID_PROXY_TOKEN matches - // neither the stored proxyToken nor its fallback, if any. Retrying - // would yield the same 401 (same token in, same status out), so we - // ack; the next deploy's /sync resyncs the token. Logged with a - // stable greppable marker so ops can alert on this. + // Auth drift between CronScheduler's stored proxyToken and the + // user worker's baked __VOID_PROXY_TOKEN. Retrying would yield + // the same 401 (same token in, same status out), so we ack; the + // next deploy's /sync resyncs the token. Logged with a stable + // greppable marker so ops can alert on this. console.error( `[cron-fires] AUTH_DRIFT non-delivery for ${msg.body.workerName} (cron ${msg.body.cron}): status=${status} body=${outcome.bodySnippet ?? ''}; acking — next deploy will resync token`, ); diff --git a/platform/packages/api/src/cron-scheduler-client.ts b/platform/packages/api/src/cron-scheduler-client.ts index 6acd2c4e1..8a393aa45 100644 --- a/platform/packages/api/src/cron-scheduler-client.ts +++ b/platform/packages/api/src/cron-scheduler-client.ts @@ -4,8 +4,6 @@ export type CronSyncState = Readonly<{ workerName: string; schedules: ReadonlyArray<{ name: string; cron: string }>; proxyToken?: string; - /** Second token to try when the Worker rejects `proxyToken`. */ - proxyTokenFallback?: string; }>; /** diff --git a/platform/packages/api/src/cron-scheduler.ts b/platform/packages/api/src/cron-scheduler.ts index 01e6802b7..24845fb94 100644 --- a/platform/packages/api/src/cron-scheduler.ts +++ b/platform/packages/api/src/cron-scheduler.ts @@ -183,8 +183,8 @@ export class CronScheduler extends DurableObject { private async handleStatus(): Promise { const state = await this.getState(); const alarm = await this.ctx.storage.getAlarm(); - // Omit the proxy tokens from the status response to avoid leaking secrets. - const { proxyToken: _, proxyTokenFallback: __, ...safeState } = state ?? {}; + // Omit proxyToken from status response to avoid leaking secrets. + const { proxyToken: _, ...safeState } = state ?? {}; return Response.json({ state: state ? safeState : null, nextAlarm: alarm }); } @@ -240,7 +240,6 @@ export class CronScheduler extends DurableObject { cron, scheduledTime: now, proxyToken: state.proxyToken, - ...(state.proxyTokenFallback ? { proxyTokenFallback: state.proxyTokenFallback } : {}), }, })); const chunks = chunkMessages(messages, SEND_BATCH_CHUNK_SIZE); diff --git a/platform/packages/api/src/proxy-token.ts b/platform/packages/api/src/proxy-token.ts index 794801168..dd31d528c 100644 --- a/platform/packages/api/src/proxy-token.ts +++ b/platform/packages/api/src/proxy-token.ts @@ -85,17 +85,6 @@ export function isAcceptedGeneration(state: ProxyTokenState, gen: number): boole return gen >= state.gen; } -/** - * Generations a sender tries, in order, for the issued generation: the - * current one, then the one before it while Workers may still hold it. - */ -export function sendGenerations(issueGen: number): ReadonlyArray { - if (!Number.isSafeInteger(issueGen) || issueGen < 0) { - throw new RangeError(`invalid proxy token generation: ${issueGen}`); - } - return issueGen >= 1 ? [issueGen, issueGen - 1] : [0]; -} - // --- Keys --- const encoder = new TextEncoder(); diff --git a/platform/packages/api/test/unit/cron-fires-queue.test.ts b/platform/packages/api/test/unit/cron-fires-queue.test.ts index f9926ed24..34cd019ac 100644 --- a/platform/packages/api/test/unit/cron-fires-queue.test.ts +++ b/platform/packages/api/test/unit/cron-fires-queue.test.ts @@ -108,83 +108,14 @@ describe('processCronFiresBatch', () => { } }); - describe('fallback token', () => { - const unauthorized = () => Response.json({ error: 'unauthorized' }, { status: 401 }); - - function rotatingMessage() { - return makeMessage({ - workerName: 'user-foo-dpl_a', - cron: '* * * * *', - scheduledTime: 1_700_000_000_000, - proxyToken: 'old-tok', - proxyTokenFallback: 'new-tok', - }); - } - - function authDriftLogs(errSpy: { mock: { calls: Array> } }) { - return errSpy.mock.calls.filter((args) => - typeof args[0] === 'string' ? args[0].includes('AUTH_DRIFT') : false, - ); - } - - it('invokes once more with the fallback when the worker rejects the token', async () => { + it.each([401, 403])( + 'invokes the cron handler once when the worker rejects the token with %i', + async (status) => { + // The Worker's own middleware runs before the cron route checks the + // token, so a rejected fire is never sent again with another token. const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); try { - const seen: Array<{ token: string | null; body: string }> = []; - const env = makeEnv(async (req) => { - const token = req.headers.get('x-void-internal'); - seen.push({ token, body: await req.text() }); - return token === 'new-tok' ? Response.json({ ok: true }) : unauthorized(); - }); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), env); - - expect(seen.map((call) => call.token)).toEqual(['old-tok', 'new-tok']); - expect(seen[1].body).toBe(seen[0].body); - expect(msg.ack).toHaveBeenCalledOnce(); - expect(msg.retry).not.toHaveBeenCalled(); - expect(authDriftLogs(errSpy)).toHaveLength(0); - } finally { - errSpy.mockRestore(); - } - }); - - it('invokes once more after a 403 unauthorized reject', async () => { - const fetchSpy = vi.fn(async (req: Request) => - req.headers.get('x-void-internal') === 'new-tok' - ? Response.json({ ok: true }) - : Response.json({ error: 'unauthorized' }, { status: 403 }), - ); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); - - expect(fetchSpy).toHaveBeenCalledTimes(2); - expect(msg.ack).toHaveBeenCalledOnce(); - }); - - it('acks with AUTH_DRIFT when the fallback is rejected too', async () => { - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - try { - const fetchSpy = vi.fn(async () => unauthorized()); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); - - expect(fetchSpy).toHaveBeenCalledTimes(2); - expect(msg.ack).toHaveBeenCalledOnce(); - expect(msg.retry).not.toHaveBeenCalled(); - expect(authDriftLogs(errSpy)).toHaveLength(1); - } finally { - errSpy.mockRestore(); - } - }); - - it('does not invoke again without a fallback', async () => { - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - try { - const fetchSpy = vi.fn(async () => unauthorized()); + const fetchSpy = vi.fn(async () => Response.json({ error: 'unauthorized' }, { status })); const msg = makeMessage({ workerName: 'user-foo-dpl_a', cron: '* * * * *', @@ -194,72 +125,18 @@ describe('processCronFiresBatch', () => { await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); - expect(fetchSpy).toHaveBeenCalledOnce(); - expect(msg.ack).toHaveBeenCalledOnce(); - expect(authDriftLogs(errSpy)).toHaveLength(1); - } finally { - errSpy.mockRestore(); - } - }); - - it('does not invoke again for a 401 with another body', async () => { - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - try { - const fetchSpy = vi.fn(async () => Response.json({ error: 'expired' }, { status: 401 })); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); - - expect(fetchSpy).toHaveBeenCalledOnce(); - expect(msg.ack).toHaveBeenCalledOnce(); - } finally { - errSpy.mockRestore(); - } - }); - - it('does not invoke again after a thrown fetch', async () => { - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - try { - const fetchSpy = vi.fn(async () => { - throw new Error('connection reset'); - }); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), makeEnv(fetchSpy)); - expect(fetchSpy).toHaveBeenCalledOnce(); expect(msg.ack).toHaveBeenCalledOnce(); expect(msg.retry).not.toHaveBeenCalled(); + const drift = errSpy.mock.calls.filter((args) => + typeof args[0] === 'string' ? args[0].includes('AUTH_DRIFT') : false, + ); + expect(drift).toHaveLength(1); } finally { errSpy.mockRestore(); } - }); - - it('uses the legacy /__scheduled path for the fallback too', async () => { - const seen: Array<{ path: string; token: string | null }> = []; - const env = makeEnv(async (req) => { - const path = new URL(req.url).pathname; - const token = req.headers.get('x-void-internal'); - seen.push({ path, token }); - if (path === '/__void/scheduled') { - return new Response('not found', { status: 404 }); - } - return token === 'new-tok' ? Response.json({ ok: true }) : unauthorized(); - }); - const msg = rotatingMessage(); - - await processCronFiresBatch(makeBatch([msg]), env); - - expect(seen).toEqual([ - { path: '/__void/scheduled', token: 'old-tok' }, - { path: '/__scheduled', token: 'old-tok' }, - { path: '/__void/scheduled', token: 'new-tok' }, - { path: '/__scheduled', token: 'new-tok' }, - ]); - expect(msg.ack).toHaveBeenCalledOnce(); - expect(msg.retry).not.toHaveBeenCalled(); - }); - }); + }, + ); it('acks 5xx responses and does not retry the user worker', async () => { // A Response (any status) means the user worker received the request. diff --git a/platform/packages/api/test/unit/cron-scheduler.test.ts b/platform/packages/api/test/unit/cron-scheduler.test.ts index f81474e88..e7b3519f2 100644 --- a/platform/packages/api/test/unit/cron-scheduler.test.ts +++ b/platform/packages/api/test/unit/cron-scheduler.test.ts @@ -246,73 +246,6 @@ describe('CronScheduler.alarm', () => { ); } - it('copies proxyTokenFallback into each message when present', async () => { - const ctx = makeStubCtx({ - state: { - workerName: 'user-foo', - schedules: [ - { name: 'every', cron: '* * * * *' }, - { name: 'fives', cron: '*/5 * * * *' }, - ], - proxyToken: 'old-tok', - proxyTokenFallback: 'new-tok', - }, - }); - const sendBatch = vi.fn(async (_batch?: unknown) => {}); - const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch } }); - - await scheduler.alarm({ - scheduledTime: Date.UTC(2026, 0, 1), - isRetry: false, - retryCount: 0, - }); - - const messages = sendBatch.mock.calls[0][0] as Array<{ body: Record }>; - expect(messages).toHaveLength(2); - for (const message of messages) { - expect(message.body.proxyToken).toBe('old-tok'); - expect(message.body.proxyTokenFallback).toBe('new-tok'); - } - }); - - it('leaves proxyTokenFallback out of messages when the state has none', async () => { - const ctx = makeStubCtx({ - state: { - workerName: 'user-foo', - schedules: [{ name: 'every', cron: '* * * * *' }], - proxyToken: 'tok', - }, - }); - const sendBatch = vi.fn(async (_batch?: unknown) => {}); - const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch } }); - - await scheduler.alarm(); - - const messages = sendBatch.mock.calls[0][0] as Array<{ body: Record }>; - expect(messages[0].body.proxyToken).toBe('tok'); - expect(messages[0].body).not.toHaveProperty('proxyTokenFallback'); - }); - - it('omits both proxy tokens from /status', async () => { - const ctx = makeStubCtx({ - state: { - workerName: 'user-foo', - schedules: [{ name: 'every', cron: '* * * * *' }], - proxyToken: 'old-tok', - proxyTokenFallback: 'new-tok', - }, - }); - const scheduler = makeScheduler(ctx, { CRON_FIRES_QUEUE: { sendBatch: async () => {} } }); - - const res = await scheduler.fetch(new Request('https://do/status')); - const body = await res.json<{ state: Record }>(); - - expect(body.state).toEqual({ - workerName: 'user-foo', - schedules: [{ name: 'every', cron: '* * * * *' }], - }); - }); - it('throws when ALL sendBatch chunks fail (DO alarm framework will retry)', async () => { // At-most-once contract: when no fires were enqueued, throwing surfaces // the failure to the DO runtime so it retries with backoff. Safe because diff --git a/platform/packages/api/test/unit/proxy-token.test.ts b/platform/packages/api/test/unit/proxy-token.test.ts index 3244dfa70..4d4af8959 100644 --- a/platform/packages/api/test/unit/proxy-token.test.ts +++ b/platform/packages/api/test/unit/proxy-token.test.ts @@ -19,7 +19,6 @@ import { type ProxyTokenState, ProxyTokenStateError, readProxyTokenState, - sendGenerations, serializeProxyTokenState, verifyProxyInternalAuth, verifyProxyToken, @@ -157,12 +156,12 @@ describe('parseProxyTokenState', () => { describe('state table', () => { it.each([ - { raw: null, accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, - { raw: '{"gen":0}', accepted: [0, 1, 5], rejected: [], send: [0], issue: 0 }, - { raw: '{"gen":1}', accepted: [1, 2], rejected: [0], send: [1, 0], issue: 1 }, - { raw: '{"gen":2}', accepted: [2, 3], rejected: [0, 1], send: [2, 1], issue: 2 }, - { raw: '{"gen":3}', accepted: [3, 4], rejected: [0, 1, 2], send: [3, 2], issue: 3 }, - ])('floor $raw', async ({ raw, accepted, rejected, send, issue }) => { + { raw: null, accepted: [0, 1, 5], rejected: [], issue: 0 }, + { raw: '{"gen":0}', accepted: [0, 1, 5], rejected: [], issue: 0 }, + { raw: '{"gen":1}', accepted: [1, 2], rejected: [0], issue: 1 }, + { raw: '{"gen":2}', accepted: [2, 3], rejected: [0, 1], issue: 2 }, + { raw: '{"gen":3}', accepted: [3, 4], rejected: [0, 1, 2], issue: 3 }, + ])('floor $raw', async ({ raw, accepted, rejected, issue }) => { const { routing } = kvStub(raw === null ? {} : { [proxyTokenStateKey(PROJECT)]: raw }); const state = await readProxyTokenState(routing, PROJECT); for (const gen of accepted) { @@ -173,7 +172,6 @@ describe('state table', () => { expect(isAcceptedGeneration(state, gen)).toBe(false); expect(await verify(await deriveProxyToken(SECRET, PROJECT, gen), state).result).toBe(false); } - expect(sendGenerations(state.gen)).toEqual(send); expect(await issueProxyToken(routing, SECRET, PROJECT)).toBe( await deriveProxyToken(SECRET, PROJECT, issue), ); @@ -184,12 +182,6 @@ describe('state table', () => { await expect(readProxyTokenState(routing, PROJECT)).rejects.toThrow(ProxyTokenStateError); await expect(issueProxyToken(routing, SECRET, PROJECT)).rejects.toThrow(ProxyTokenStateError); }); - - it('sends the issued generation, then the one before it', () => { - expect(sendGenerations(0)).toEqual([0]); - expect(sendGenerations(1)).toEqual([1, 0]); - expect(sendGenerations(5)).toEqual([5, 4]); - }); }); describe('verifyProxyToken', () => { diff --git a/platform/packages/dispatch/src/index.ts b/platform/packages/dispatch/src/index.ts index 18623bbeb..5beef1314 100644 --- a/platform/packages/dispatch/src/index.ts +++ b/platform/packages/dispatch/src/index.ts @@ -98,8 +98,6 @@ type QueueRoutingEntry = Readonly<{ // without a matching `x-void-internal` header with 401. Absent on entries // written by older API versions. proxyToken?: string; - // Second token to try when the Worker rejects `proxyToken`. - proxyTokenFallback?: string; }>; const CLIENT_REVALIDATE_CACHE_CONTROL = 'public, max-age=0, must-revalidate'; @@ -1921,9 +1919,7 @@ export default { batch.retryAll(); return; } - const { workerName, queueName, proxyToken, proxyTokenFallback } = JSON.parse( - routingRaw, - ) as QueueRoutingEntry; + const { workerName, queueName, proxyToken } = JSON.parse(routingRaw) as QueueRoutingEntry; // Serialize batch for relay const messages = Array.from(batch.messages, (message) => ({ @@ -1936,39 +1932,27 @@ export default { const worker = env.DISPATCH.get(workerName); try { const body = JSON.stringify({ queue: queueName, messages }); - const relay = async (token: string | undefined): Promise => { - const queueHeaders: Record = { 'Content-Type': 'application/json' }; - if (token) { - queueHeaders['x-void-internal'] = token; - } - const res = await worker.fetch( - new Request('https://internal/__void/queue', { + const queueHeaders: Record = { 'Content-Type': 'application/json' }; + if (proxyToken) { + queueHeaders['x-void-internal'] = proxyToken; + } + let res = await worker.fetch( + new Request('https://internal/__void/queue', { + method: 'POST', + headers: queueHeaders, + body, + }), + ); + // Legacy: fall back for workers deployed before the /__void/ migration. + // TODO: remove once all deployed workers use /__void/* paths. + if (res.status === 404) { + res = await worker.fetch( + new Request('https://internal/__queue', { method: 'POST', headers: queueHeaders, body, }), ); - // Legacy: fall back for workers deployed before the /__void/ migration. - // TODO: remove once all deployed workers use /__void/* paths. - if (res.status === 404) { - return worker.fetch( - new Request('https://internal/__queue', { - method: 'POST', - headers: queueHeaders, - body, - }), - ); - } - return res; - }; - - let res = await relay(proxyToken); - // While a token rotation is in progress the Worker may hold either - // token. Its reject comes before any queue handler runs, so a second - // attempt with the other token cannot deliver the batch twice. - if (proxyTokenFallback && (await isQueueTokenRejection(res))) { - await res.body?.cancel(); - res = await relay(proxyTokenFallback); } if (!res.ok) { @@ -2041,23 +2025,6 @@ export default { }, } satisfies ExportedHandler; -/** The generated queue route's reject for a token it does not hold. */ -async function isQueueTokenRejection(res: Response): Promise { - if (res.status !== 401) { - return false; - } - try { - const data: unknown = await res.clone().json(); - return ( - typeof data === 'object' && - data !== null && - (data as { error?: unknown }).error === 'unauthorized' - ); - } catch { - return false; - } -} - class StaticAssetReadError extends Error { constructor(cause: unknown) { super('Static asset storage read failed', { cause }); diff --git a/platform/packages/dispatch/test/unit/index.test.ts b/platform/packages/dispatch/test/unit/index.test.ts index e0b13cb7f..6bd2e8507 100644 --- a/platform/packages/dispatch/test/unit/index.test.ts +++ b/platform/packages/dispatch/test/unit/index.test.ts @@ -4555,84 +4555,20 @@ describe('queue relay', () => { } }); - describe('fallback token', () => { - const rotatingRouting = JSON.stringify({ - workerName: 'user-app', - projectId: 'proj-1', - queueName: 'emails', - proxyToken: 'old', - proxyTokenFallback: 'new', - }); - const unauthorized = () => Response.json({ error: 'unauthorized' }, { status: 401 }); - - function setup(routingRaw: string, respond: (token: string | null) => Promise) { - const batch = createMockBatch('test-queue', [{ id: 'm1', body: 'hello' }]); - const { env, workerFetch } = createQueueEnv({ routingRaw }); - const seen: Array<{ token: string | null; body: string }> = []; - workerFetch.mockImplementation(async (req: Request) => { - const token = req.headers.get('x-void-internal'); - seen.push({ token, body: await req.clone().text() }); - return respond(token); - }); - return { batch, env, seen }; - } - - it('relays once more with the fallback when the Worker rejects the token', async () => { - const { batch, env, seen } = setup(rotatingRouting, async (token) => - token === 'new' ? Response.json({ ok: true }) : unauthorized(), - ); - - await app.queue(batch, env); - - expect(seen.map((call) => call.token)).toEqual(['old', 'new']); - expect(seen[1].body).toBe(seen[0].body); - expect(batch.ackAll).toHaveBeenCalledOnce(); - expect(batch.retryAll).not.toHaveBeenCalled(); - }); - - it('retries the batch without a second relay when there is no fallback', async () => { + it.each([401, 403])( + 'relays once and retries the batch when the Worker rejects the token with %i', + async (status) => { const routing = JSON.stringify({ workerName: 'user-app', projectId: 'proj-1', queueName: 'emails', - proxyToken: 'old', - }); - const { batch, env, seen } = setup(routing, async () => unauthorized()); - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - try { - await app.queue(batch, env); - } finally { - errSpy.mockRestore(); - } - - expect(seen).toHaveLength(1); - expect(batch.retryAll).toHaveBeenCalledOnce(); - }); - - it.each([ - ['a 401 with another body', () => Response.json({ error: 'expired' }, { status: 401 })], - ['a 401 with a non-JSON body', () => new Response('unauthorized', { status: 401 })], - ['a 403 unauthorized', () => Response.json({ error: 'unauthorized' }, { status: 403 })], - ['a 500', () => new Response('boom', { status: 500 })], - ])('does not relay again after %s', async (_label, response) => { - const { batch, env, seen } = setup(rotatingRouting, async () => response()); - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - try { - await app.queue(batch, env); - } finally { - errSpy.mockRestore(); - } - - expect(seen).toHaveLength(1); - expect(batch.retryAll).toHaveBeenCalledOnce(); - }); - - it('does not relay again after a thrown fetch', async () => { - const { batch, env, seen } = setup(rotatingRouting, async () => { - throw new Error('connection reset'); + proxyToken: 'tok', }); + const batch = createMockBatch('test-queue', [{ id: 'm1', body: 'hello' }]); + const { env, workerFetch } = createQueueEnv({ routingRaw: routing }); + workerFetch.mockImplementation(async () => + Response.json({ error: 'unauthorized' }, { status }), + ); const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); try { @@ -4641,25 +4577,11 @@ describe('queue relay', () => { errSpy.mockRestore(); } - expect(seen).toHaveLength(1); - expect(batch.retryAll).toHaveBeenCalledOnce(); - }); - - it('retries the batch after two relays when both tokens are rejected', async () => { - const { batch, env, seen } = setup(rotatingRouting, async () => unauthorized()); - const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - try { - await app.queue(batch, env); - } finally { - errSpy.mockRestore(); - } - - expect(seen.map((call) => call.token)).toEqual(['old', 'new']); + expect(workerFetch).toHaveBeenCalledOnce(); expect(batch.retryAll).toHaveBeenCalledOnce(); expect(batch.ackAll).not.toHaveBeenCalled(); - }); - }); + }, + ); it('forwards x-void-internal on legacy /__queue fallback too', async () => { const routing = JSON.stringify({ diff --git a/platform/packages/proxy/README.md b/platform/packages/proxy/README.md index ecd1f3d45..d5b89e762 100644 --- a/platform/packages/proxy/README.md +++ b/platform/packages/proxy/README.md @@ -67,9 +67,10 @@ forwards the token, and the API checks it against the same floor. When the proxy calls a tenant Worker's `/__void/*` handler, it sends the token for the generation `resolve-worker` reports (`proxyTokenGen`), or for the floor -when the API does not report one. If that Worker answers 403 -`{"error":"unauthorized"}`, the proxy retries once with the previous -generation. It never sends a generation above the issued one. +when the API does not report one. It sends each call once. The Worker's own +middleware runs before its `/__void/*` handler checks the token, so a 403 +`{"error":"unauthorized"}` is returned to the caller and never replayed with +another token. Cron fires and queue relays follow the same rule. The proxy authenticates its own `resolve-worker`, `resolve-resource`, and `verify-prerender-build` calls with `x-void-proxy-internal`, an HMAC under a diff --git a/platform/packages/proxy/src/dispatch.ts b/platform/packages/proxy/src/dispatch.ts index 17659de47..60362804f 100644 --- a/platform/packages/proxy/src/dispatch.ts +++ b/platform/packages/proxy/src/dispatch.ts @@ -4,13 +4,13 @@ import { ProxyTokenStateError, } from '@voidcloud/api/proxy-token'; import { hmacSign } from './hmac.ts'; -import { tenantSendTokens } from './proxy-token.ts'; +import { tenantSendToken } from './proxy-token.ts'; import type { Env } from './types.ts'; type WorkerResolution = Readonly<{ workerName: string; deploymentId: string; - /** Generation of the token the platform issues to this project's Workers. */ + /** Generation of the token the active Worker holds, when the API reports it. */ proxyTokenGen?: number; }>; @@ -65,49 +65,6 @@ function isGeneration(value: unknown): value is number { return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; } -/** The binding handler's reject is `{"error":"unauthorized"}`; anything larger is not it. */ -const TOKEN_REJECTION_MAX_BYTES = 256; - -/** - * The binding handler's reject for a token it does not hold, sent before any handler runs. - * The tenant Worker controls this response, so only a small prefix is ever read and the - * returned response still streams unbuffered. - */ -async function isTokenRejection(res: Response): Promise { - if (res.status !== 403 || !res.body) { - return false; - } - if (Number(res.headers.get('content-length')) > TOKEN_REJECTION_MAX_BYTES) { - return false; - } - const reader = res.clone().body!.getReader(); - const chunks: Array = []; - let size = 0; - try { - for (;;) { - const chunk = await reader.read(); - if (chunk.done) { - break; - } - size += chunk.value.byteLength; - if (size > TOKEN_REJECTION_MAX_BYTES) { - return false; - } - chunks.push(chunk.value); - } - const data: unknown = JSON.parse(await new Blob(chunks).text()); - return ( - typeof data === 'object' && - data !== null && - (data as { error?: unknown }).error === 'unauthorized' - ); - } catch { - return false; - } finally { - void reader.cancel().catch(() => {}); - } -} - /** * Dispatch a binding operation to the user's deployed worker via Workers for Platforms. * The user worker handles it via native D1/KV/R2 bindings at the `/__void/*` route. @@ -134,9 +91,9 @@ export async function dispatchToWorker( return null; } - let tokens: ReadonlyArray; + let token: string; try { - tokens = await tenantSendTokens(env, projectId, resolution.proxyTokenGen); + token = await tenantSendToken(env, projectId, resolution.proxyTokenGen); } catch (error) { if (!(error instanceof ProxyTokenStateError)) { throw error; @@ -154,27 +111,19 @@ export async function dispatchToWorker( return null; } - const payload = JSON.stringify(body); - const send = (token: string) => - worker.fetch(`https://worker/__void${path}`, { + try { + // Sent exactly once. The Worker's own middleware runs before its + // `/__void/*` handler checks the token, so a rejected request is not free + // to replay. + const res = await worker.fetch(`https://worker/__void${path}`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-void-internal': token, }, - body: payload, + body: JSON.stringify(body), }); - try { - let res = await send(tokens[0]); - // While a token rotation is in progress the Worker may still hold the - // previous token. Its reject comes before any handler runs, so a second - // attempt cannot repeat a write. - if (tokens.length > 1 && (await isTokenRejection(res))) { - await res.body?.cancel(); - res = await send(tokens[1]); - } - // 404 means old deploy without /__void handler → fall back if (res.status === 404) { return null; diff --git a/platform/packages/proxy/src/proxy-token.ts b/platform/packages/proxy/src/proxy-token.ts index 874d31fd0..e761093d4 100644 --- a/platform/packages/proxy/src/proxy-token.ts +++ b/platform/packages/proxy/src/proxy-token.ts @@ -2,7 +2,6 @@ import { createProxyTokenStateCache, deriveProxyToken, ProxyTokenStateError, - sendGenerations, verifyProxyToken, } from '@voidcloud/api/proxy-token'; import type { Env } from './types.ts'; @@ -42,17 +41,15 @@ export async function verifyTenantProxyToken( } /** - * Tokens to present to a tenant Worker, in order. Without the issued - * generation, sends for the cached floor, which throws `ProxyTokenStateError` - * when the floor cannot be read. + * The one token to present to a tenant Worker. Without the issued generation, + * sends for the cached floor, which throws `ProxyTokenStateError` when the + * floor cannot be read. */ -export async function tenantSendTokens( +export async function tenantSendToken( env: Pick, projectId: string, issueGen: number | undefined, -): Promise> { +): Promise { const gen = issueGen ?? (await stateCache.read(env.ROUTING, projectId)).gen; - return Promise.all( - sendGenerations(gen).map((sendGen) => deriveProxyToken(env.JWT_SECRET, projectId, sendGen)), - ); + return deriveProxyToken(env.JWT_SECRET, projectId, gen); } diff --git a/platform/packages/proxy/test/unit/dispatch.test.ts b/platform/packages/proxy/test/unit/dispatch.test.ts index fdd9a2d7d..738c1452f 100644 --- a/platform/packages/proxy/test/unit/dispatch.test.ts +++ b/platform/packages/proxy/test/unit/dispatch.test.ts @@ -714,16 +714,7 @@ describe('proxy token generations', () => { expect(sent.map((s) => s.token)).toEqual([legacy]); }); - it('sends the floor token, then the previous one, when there is no issued generation', async () => { - const { env, sent } = setup({ floor: '{"gen":2}', held: await tok(1) }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(200); - expect(sent.map((s) => s.token)).toEqual([await tok(2), await tok(1)]); - }); - - it('sends one request when the Worker holds the floor token', async () => { + it('sends the floor token when there is no issued generation', async () => { const { env, sent } = setup({ floor: '{"gen":1}', held: await tok(1) }); const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); @@ -732,138 +723,41 @@ describe('proxy token generations', () => { expect(sent.map((s) => s.token)).toEqual([await tok(1)]); }); - it('retries generation 1 with the legacy token and the same body', async () => { - const legacy = await tok(0); + it('sends the reported generation without reading the floor', async () => { const { env, sent, floorReads } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - held: legacy, + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, + held: await tok(2), }); const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); expect(res?.status).toBe(200); - expect(await res?.json()).toBe('hello'); - expect(sent.map((s) => s.token)).toEqual([await tok(1), legacy]); - expect(sent[1].body).toEqual(sent[0].body); + expect(sent.map((s) => s.token)).toEqual([await tok(2)]); expect(sent[0].body).toEqual(KV_BODY); expect(floorReads()).toBe(0); }); it.each([ - { held: 1, tokens: [2, 1] }, - { held: 2, tokens: [2] }, - ])('sends issued generation 2 to a Worker holding $held', async ({ held, tokens }) => { - const { env, sent, floorReads } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, - held: await tok(held), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(200); - expect(sent.map((s) => s.token)).toEqual(await Promise.all(tokens.map(tok))); - expect(floorReads()).toBe(0); - }); - - it('does not retry a 403 with another error', async () => { - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - respond: () => Response.json({ error: 'forbidden' }, { status: 403 }), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(403); - expect(await res?.json()).toEqual({ error: 'forbidden' }); - expect(sent).toHaveLength(1); - }); - - it('does not retry a 403 whose body is not JSON', async () => { - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - respond: () => new Response('denied', { status: 403 }), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(403); - expect(await res?.text()).toBe('denied'); - expect(sent).toHaveLength(1); - }); - - it('retries a reject that is streamed without a declared length', async () => { - const legacy = await tok(0); - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - respond: (token) => - token === legacy - ? Response.json('hello') - : new Response(new Blob(['{"error":"unauthorized"}']).stream(), { status: 403 }), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(200); - expect(sent.map((s) => s.token)).toEqual([await tok(1), legacy]); - }); - - it('does not read a 403 that declares a large body', async () => { - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - respond: () => - new Response('{"error":"unauthorized"}', { - status: 403, - headers: { 'content-length': String(64 * 1024 * 1024) }, - }), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(403); - expect(sent).toHaveLength(1); - }); - - it('stops reading a streamed 403 once it outgrows a reject', async () => { - let pulls = 0; - const chunk = new TextEncoder().encode(' '.repeat(64)); - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, - respond: () => - new Response( - new ReadableStream({ - pull(controller) { - pulls++; - controller.enqueue(chunk); - }, - }), - { status: 403 }, - ), - }); - - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); - - expect(res?.status).toBe(403); - expect(sent).toHaveLength(1); - expect(pulls).toBeLessThan(16); - const reader = res!.body!.getReader(); - expect((await reader.read()).value).toEqual(chunk); - await reader.cancel(); - }); - - it('returns the second reject when the Worker holds neither token', async () => { - const { env, sent } = setup({ - resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 2 }, - held: await tok(0), - }); + { label: 'a reported generation', resolution: { proxyTokenGen: 2 }, floor: null }, + { label: 'the floor', resolution: {}, floor: '{"gen":2}' }, + ])( + 'returns the reject from a Worker holding another token after one request ($label)', + async ({ resolution, floor }) => { + const { env, sent } = setup({ + resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', ...resolution }, + floor, + held: await tok(1), + }); - const res = await dispatchToWorker(env, PROJECT_ID, '/kv/get', KV_BODY); + const res = await dispatchToWorker(env, PROJECT_ID, '/kv/put', KV_BODY); - expect(res?.status).toBe(403); - expect(await res?.json()).toEqual({ error: 'unauthorized' }); - expect(sent.map((s) => s.token)).toEqual([await tok(2), await tok(1)]); - }); + expect(res?.status).toBe(403); + expect(await res?.json()).toEqual({ error: 'unauthorized' }); + expect(sent.map((s) => s.token)).toEqual([await tok(2)]); + }, + ); - it('does not retry after a thrown fetch', async () => { + it('does not replay after a thrown fetch', async () => { const { env, sent } = setup({ resolution: { workerName: 'user-my-app', deploymentId: 'dpl_active', proxyTokenGen: 1 }, respond: () => {