Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/896-webhook-classification.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A webhook send that may have reached the destination is now classified `unknown` even when Undici puts the socket code on `error.cause`, so it is not resent automatically; a transient resolver failure is retried instead of being marked `blocked` permanently.
4 changes: 3 additions & 1 deletion docs/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,9 @@ changed address is not mailed the old copy. A queued webhook row stores no URL:
the configured `NOTIFY_WEBHOOK_URL` is read at send time under the same safety
rule as a live send (https or allowlisted host, no credentials in the URL, no
private or reserved address, bounded redirects), and a refused destination is
`blocked` rather than retried. The admin outbox page renders transport, state,
`blocked` rather than retried. A resolution failure (the resolver throws or
returns no address) is transient and stays `retry`; only the policy checks
(scheme, credentials, allowlist, private or reserved address) are `blocked`. The admin outbox page renders transport, state,
attempt count, last error and delivery time with the address masked; it never
renders the mail or webhook body. A delivered row is kept for 30 days, `dead`
and `blocked` rows for 90, and `unknown` for 365 because it is the only state
Expand Down
54 changes: 37 additions & 17 deletions src/lib/webhook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ export interface WebhookSafetyOptions {
fetch?: typeof fetch
}

/** A destination refusal (`blocked`) is terminal; a resolution failure is transient. */
export class WebhookAddressError extends Error {
constructor(message: string, readonly kind: 'blocked' | 'retry') {
super(message)
this.name = 'WebhookAddressError'
}
}

const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
const MAX_REDIRECTS = 5

Expand All @@ -29,16 +37,16 @@ function parseWebhookUrl(raw: string): URL {
try {
url = new URL(raw)
} catch {
throw new Error('Refusing notification webhook: not an absolute URL')
throw new WebhookAddressError('Refusing notification webhook: not an absolute URL', 'blocked')
}
if (url.username !== '' || url.password !== '') {
throw new Error(`Refusing notification webhook: host "${normalizeHost(url.hostname)}" must not include credentials`)
throw new WebhookAddressError(`Refusing notification webhook: host "${normalizeHost(url.hostname)}" must not include credentials`, 'blocked')
}
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
throw new Error(`Refusing notification webhook: host "${normalizeHost(url.hostname)}" must use http:// or https://`)
throw new WebhookAddressError(`Refusing notification webhook: host "${normalizeHost(url.hostname)}" must use http:// or https://`, 'blocked')
}
if (url.hostname === '') {
throw new Error('Refusing notification webhook: missing host')
throw new WebhookAddressError('Refusing notification webhook: missing host', 'blocked')
}
return url
}
Expand All @@ -47,7 +55,7 @@ async function resolveWebhookHost(host: string, resolve: WebhookResolver): Promi
try {
return await resolve(host)
} catch {
throw new Error(`Refusing notification webhook: cannot resolve host "${host}"`)
throw new WebhookAddressError(`Refusing notification webhook: cannot resolve host "${host}"`, 'retry')
}
}

Expand All @@ -58,28 +66,28 @@ export async function assertSafeWebhookUrl(raw: string, options: WebhookSafetyOp
const allowlisted = allowedHosts.includes(host)

if (allowedHosts.length > 0 && !allowlisted) {
throw new Error(`Refusing notification webhook: host "${host}" is not in NOTIFY_WEBHOOK_ALLOWED_HOSTS`)
throw new WebhookAddressError(`Refusing notification webhook: host "${host}" is not in NOTIFY_WEBHOOK_ALLOWED_HOSTS`, 'blocked')
}
if (allowlisted) return url

if (url.protocol !== 'https:') {
throw new Error(`Refusing notification webhook: host "${host}" must use https://`)
throw new WebhookAddressError(`Refusing notification webhook: host "${host}" must use https://`, 'blocked')
}
if (isIP(host) !== 0) {
if (isBlockedIpAddress(host)) {
throw new Error(`Refusing notification webhook: host "${host}" is a private or reserved address`)
throw new WebhookAddressError(`Refusing notification webhook: host "${host}" is a private or reserved address`, 'blocked')
}
return url
}

const resolve = options.resolve ?? ((name: string) => lookup(name, { all: true }))
const addresses = await resolveWebhookHost(host, resolve)
if (addresses.length === 0) {
throw new Error(`Refusing notification webhook: cannot resolve host "${host}"`)
throw new WebhookAddressError(`Refusing notification webhook: cannot resolve host "${host}"`, 'retry')
}
for (const entry of addresses) {
if (isBlockedIpAddress(entry.address)) {
throw new Error(`Refusing notification webhook: host "${host}" resolves to a private or reserved address`)
throw new WebhookAddressError(`Refusing notification webhook: host "${host}" resolves to a private or reserved address`, 'blocked')
}
}
return url
Expand Down Expand Up @@ -112,9 +120,10 @@ async function postWebhook(
try {
safeUrl = await assertSafeWebhookUrl(current.toString(), options)
} catch (error) {
// A destination the safety rule refuses is not transient: retrying it
// forever would never change the answer, so the outbox records it blocked.
throw new NotificationWebhookError(String(error), 'blocked')
// A policy refusal is terminal; a resolver failure is transient. The
// address layer says which one this is, so a DNS blip is retried instead
// of losing the notification as `blocked`.
throw new NotificationWebhookError(String(error), error instanceof WebhookAddressError ? error.kind : 'blocked')
}
let response: Response
try {
Expand Down Expand Up @@ -164,11 +173,22 @@ export class NotificationWebhookError extends Error {
// Connection refused, DNS and TLS-setup failures are not.
const AMBIGUOUS_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'EPIPE', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT'])

function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null
}

function webhookErrorCode(error: unknown): string | undefined {
if (!isRecord(error)) return undefined
if (typeof error.code === 'string') return error.code
if (isRecord(error.cause) && typeof error.cause.code === 'string') return error.cause.code
return undefined
}

function isAmbiguousWebhookFailure(error: unknown): boolean {
if (typeof error !== 'object' || error === null) return false
const record = error as { name?: unknown; code?: unknown }
if (record.name === 'TimeoutError' || record.name === 'AbortError') return true
return typeof record.code === 'string' && AMBIGUOUS_CODES.has(record.code)
if (!isRecord(error)) return false
if (error.name === 'TimeoutError' || error.name === 'AbortError') return true
const code = webhookErrorCode(error)
return code !== undefined && AMBIGUOUS_CODES.has(code)
}

/**
Expand Down
42 changes: 41 additions & 1 deletion test/notificationOutbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ test('the webhook transport maps a status to sent, retry or dead, and a timeout
assert.equal((await blocked.send(row('off'))).state, 'blocked')
})

test('a webhook that fails DNS is retried, but a private destination is blocked', async () => {
test('a fetch-level network error is retried, but a private destination is blocked', async () => {
const config = testConfig({ notifyWebhookUrl: 'https://hooks.example.com/fledge' })
const resolve = async () => [{ address: '203.0.113.10', family: 4 }]
const dns = webhookTransport(config, {
Expand All @@ -236,6 +236,46 @@ test('a webhook that fails DNS is retried, but a private destination is blocked'
assert.equal(refused.state, 'blocked', 'a refused destination is not retried forever')
})

test('a reset on the fetch cause is unknown and is not claimed on the next pass', async () => {
const config = testConfig({ notifyWebhookUrl: 'https://hooks.example.com/fledge' })
const resolve = async () => [{ address: '203.0.113.10', family: 4 }]
const transport = webhookTransport(config, {
resolve,
fetch: async () => { throw Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } }) },
})
assert.equal((await transport.send(row('cause-reset'))).state, 'unknown')

const h = harness('cause-reset')
try {
enqueue(h.store, 'webhook:cause-reset')
const first = await deliverOutbox(h.store, {
email: { send: async () => ({ state: 'sent' }) },
webhook: transport,
}, { now: T0, leaseMs: OUTBOX_LEASE_MS })
assert.equal(first.unknown, 1)

const second = await deliverOutbox(h.store, {
email: { send: async () => ({ state: 'sent' }) },
webhook: { send: async () => { throw new Error('must not be called') } },
}, { now: T0 + 60_000, leaseMs: OUTBOX_LEASE_MS })
assert.equal(second.claimed, 0)
} finally {
h.store.close()
}
})

test('a webhook resolver failure is retryable, while a private destination stays blocked', async () => {
const config = testConfig({ notifyWebhookUrl: 'https://hooks.example.com/fledge' })
const retry = webhookTransport(config, {
resolve: async () => { throw new Error('getaddrinfo EAI_AGAIN') },
fetch: async () => new Response('', { status: 200 }),
})
assert.equal((await retry.send(row('resolver-fail'))).state, 'retry')

const blocked = webhookTransport(config, { resolve: async () => [{ address: '10.0.0.1', family: 4 }] })
assert.equal((await blocked.send(row('resolver-private'))).state, 'blocked')
})

// Rule the whole slice turns on: a timeout is not a failure to send, it is a
// send whose outcome nobody knows. It must not be retried by the machine.
test('a timeout outcome is unknown and is not claimed on the next pass', async () => {
Expand Down
Loading