Skip to content

Commit 7e7459b

Browse files
waleedlatif1waleedlatif1
committed
improvement(media): record a hosted-key auth failure when fal refuses Sim's account
Mothership's generate_audio and generate_video call fal on Sim's hosted keys outside the tool executor, so a refused account (revoked key, or a lock for an exhausted balance) only reached logs. The submit path now records a Sim/HostedKey Failed point with reason auth on a 401, 402 or 403, the same series the hosted-key dashboard already plots for executor tools, so an alarm on that reason covers fal too.
1 parent 395a394 commit 7e7459b

5 files changed

Lines changed: 155 additions & 17 deletions

File tree

‎apps/sim/lib/media/falai-audio.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,13 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
152152
ref_audio_url: params.voiceSampleDataUri,
153153
model_type: 'F5-TTS',
154154
}
155-
const { requestId, data } = await runFalQueue(model, input, apiKey, params.signal)
155+
const { requestId, data } = await runFalQueue(
156+
model,
157+
input,
158+
apiKey,
159+
'generate_audio',
160+
params.signal
161+
)
156162
const cost = await recordFalCost(apiKey, model, requestId, params.onCost)
157163
const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
158164
if (!url) throw new Error('No audio URL in Fal.ai clone response')
@@ -172,7 +178,13 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
172178
const input = buildInput(type, params, model)
173179

174180
// For fal audio models the model ID is the queue endpoint.
175-
const { requestId, data } = await runFalQueue(model, input, apiKey, params.signal)
181+
const { requestId, data } = await runFalQueue(
182+
model,
183+
input,
184+
apiKey,
185+
'generate_audio',
186+
params.signal
187+
)
176188
const cost = await recordFalCost(apiKey, model, requestId, params.onCost)
177189
const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
178190
if (!url) throw new Error('No audio URL in Fal.ai response')

‎apps/sim/lib/media/falai-hosted-key-metrics.test.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { jsonResponse } from '@sim/testing/helpers/http'
2+
import { inputValidationMock } from '@sim/testing/mocks/input-validation.mock'
3+
import { afterEach, describe, expect, it, vi } from 'vitest'
4+
5+
const { sentMetricData } = vi.hoisted(() => ({
6+
sentMetricData: [] as Array<{
7+
MetricName?: string
8+
Dimensions?: Array<{ Name?: string; Value?: string }>
9+
}>,
10+
}))
11+
12+
vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
13+
vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
14+
...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
15+
isHosted: true,
16+
}))
17+
vi.mock('@aws-sdk/client-cloudwatch', () => ({
18+
StandardUnit: { Count: 'Count', None: 'None', Milliseconds: 'Milliseconds' },
19+
PutMetricDataCommand: class {
20+
constructor(readonly input: { MetricData: typeof sentMetricData }) {}
21+
},
22+
CloudWatchClient: class {
23+
async send(command: { input: { MetricData: typeof sentMetricData } }) {
24+
sentMetricData.push(...command.input.MetricData)
25+
}
26+
},
27+
}))
28+
29+
import { runFalQueue } from '@/lib/media/falai'
30+
import { flushHostedKeyMetrics } from '@/lib/monitoring/metrics'
31+
32+
function stubFalSubmit(status: number, body: unknown) {
33+
vi.stubGlobal(
34+
'fetch',
35+
vi.fn(async () => jsonResponse(body, status))
36+
)
37+
}
38+
39+
async function failedHostedKeyReasons(): Promise<Array<Record<string, string | undefined>>> {
40+
await flushHostedKeyMetrics()
41+
return sentMetricData
42+
.filter((datum) => datum.MetricName === 'Failed')
43+
.map((datum) => Object.fromEntries((datum.Dimensions ?? []).map((d) => [d.Name, d.Value])))
44+
}
45+
46+
describe('a fal submit the hosted account refuses', () => {
47+
afterEach(() => {
48+
vi.unstubAllGlobals()
49+
sentMetricData.length = 0
50+
})
51+
52+
it.each([
53+
[403, { detail: 'User is locked. Reason: Exhausted balance.' }],
54+
[401, { detail: 'Invalid API key' }],
55+
])('records a hosted-key auth failure for a %i', async (status, body) => {
56+
stubFalSubmit(status, body)
57+
58+
await expect(runFalQueue('fal-ai/model', {}, 'key', 'generate_audio')).rejects.toThrow(
59+
`Fal.ai API error: ${status}`
60+
)
61+
62+
expect(await failedHostedKeyReasons()).toEqual([
63+
expect.objectContaining({ Provider: 'falai', Tool: 'generate_audio', Reason: 'auth' }),
64+
])
65+
})
66+
67+
it('records nothing when fal rejects the request input', async () => {
68+
stubFalSubmit(422, { detail: [{ msg: 'field required' }] })
69+
70+
await expect(runFalQueue('fal-ai/model', {}, 'key', 'generate_video')).rejects.toThrow(
71+
'Fal.ai API error: 422'
72+
)
73+
74+
expect(await failedHostedKeyReasons()).toEqual([])
75+
})
76+
})

‎apps/sim/lib/media/falai-video.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,13 @@ export async function generateFalVideo(params: GenerateFalVideoParams): Promise<
177177
input.prompt_optimizer = params.promptOptimizer
178178
}
179179

180-
const { requestId, data } = await runFalQueue(endpoint, input, apiKey, params.signal)
180+
const { requestId, data } = await runFalQueue(
181+
endpoint,
182+
input,
183+
apiKey,
184+
'generate_video',
185+
params.signal
186+
)
181187
const cost = await recordFalCost(apiKey, endpoint, requestId, params.onCost)
182188
const url = extractFalMediaUrl(data, ['video', 'output'])
183189
if (!url) throw new Error('No video URL in Fal.ai response')

‎apps/sim/lib/media/falai.test.ts‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,13 @@ describe('runFalQueue on a request that never finishes', () => {
6767
it('stops polling and cancels the request once its caller gives up', async () => {
6868
const fal = stubStuckFalQueue()
6969
const caller = new AbortController()
70-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal)
70+
const generation = runFalQueue(
71+
'fal-ai/model',
72+
{ prompt: 'p' },
73+
'key',
74+
'generate_video',
75+
caller.signal
76+
)
7177
const outcome = generation.then(
7278
() => 'resolved',
7379
() => 'rejected'
@@ -87,7 +93,13 @@ describe('runFalQueue on a request that never finishes', () => {
8793
it('ends a poll that never answers when its caller gives up', async () => {
8894
const fal = stubStuckFalQueue({ statusHangs: true })
8995
const caller = new AbortController()
90-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal)
96+
const generation = runFalQueue(
97+
'fal-ai/model',
98+
{ prompt: 'p' },
99+
'key',
100+
'generate_video',
101+
caller.signal
102+
)
91103
const outcome = generation.then(
92104
() => 'resolved',
93105
() => 'rejected'
@@ -104,7 +116,7 @@ describe('runFalQueue on a request that never finishes', () => {
104116

105117
it('fails and cancels after its own ceiling even when nothing aborts it', async () => {
106118
const fal = stubStuckFalQueue()
107-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key')
119+
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', 'generate_video')
108120
const outcome = generation.then(
109121
() => 'resolved',
110122
(error: unknown) => String(error)
@@ -119,9 +131,13 @@ describe('runFalQueue on a request that never finishes', () => {
119131
it('warns when fal refuses to cancel a request', async () => {
120132
const fal = stubStuckFalQueue({ cancelStatus: 500 })
121133
const caller = new AbortController()
122-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
123-
() => undefined
124-
)
134+
const generation = runFalQueue(
135+
'fal-ai/model',
136+
{ prompt: 'p' },
137+
'key',
138+
'generate_video',
139+
caller.signal
140+
).catch(() => undefined)
125141
await vi.advanceTimersByTimeAsync(3_500)
126142
caller.abort()
127143
await generation
@@ -137,7 +153,7 @@ describe('runFalQueue on a request that never finishes', () => {
137153
const fal = stubStuckFalQueue()
138154

139155
await expect(
140-
runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', AbortSignal.abort())
156+
runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', 'generate_video', AbortSignal.abort())
141157
).rejects.toThrow()
142158
expect(fal.calls).toEqual([])
143159
})
@@ -160,9 +176,13 @@ describe('runFalQueue on a request that never finishes', () => {
160176
})
161177
)
162178
const caller = new AbortController()
163-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
164-
() => 'rejected'
165-
)
179+
const generation = runFalQueue(
180+
'fal-ai/model',
181+
{ prompt: 'p' },
182+
'key',
183+
'generate_video',
184+
caller.signal
185+
).catch(() => 'rejected')
166186

167187
await vi.advanceTimersByTimeAsync(0)
168188
caller.abort(new Error('watchdog'))
@@ -175,9 +195,13 @@ describe('runFalQueue on a request that never finishes', () => {
175195
it('asks fal never to start the request past that ceiling', async () => {
176196
const fal = stubStuckFalQueue()
177197
const caller = new AbortController()
178-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
179-
() => undefined
180-
)
198+
const generation = runFalQueue(
199+
'fal-ai/model',
200+
{ prompt: 'p' },
201+
'key',
202+
'generate_video',
203+
caller.signal
204+
).catch(() => undefined)
181205
await vi.advanceTimersByTimeAsync(0)
182206
caller.abort()
183207
await generation

‎apps/sim/lib/media/falai.ts‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
readResponseTextWithLimit,
1414
readResponseToBufferWithLimit,
1515
} from '@/lib/core/utils/stream-limits'
16+
import { hostedKeyMetrics } from '@/lib/monitoring/metrics'
1617
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
1718

1819
const logger = createLogger('FalMediaClient')
@@ -33,6 +34,15 @@ const FAL_CANCEL_TIMEOUT_MS = 5000
3334
*/
3435
const FAL_SUBMIT_TIMEOUT_MS = 60_000
3536

37+
/** The Mothership tools that generate media on Sim's hosted Fal.ai keys. */
38+
type FalMediaTool = 'generate_audio' | 'generate_video'
39+
40+
/**
41+
* Statuses fal answers when the hosted account itself refuses the call (a revoked key, or an
42+
* account locked for an exhausted balance), so every generation fails until someone acts.
43+
*/
44+
const FAL_ACCOUNT_REFUSAL_STATUSES = new Set([401, 402, 403])
45+
3646
/**
3747
* Resolves a hosted Fal.ai API key from the numbered env pool
3848
* (FALAI_API_KEY_COUNT + FALAI_API_KEY_1..N), round-robined by minute,
@@ -101,6 +111,7 @@ export async function runFalQueue(
101111
endpoint: string,
102112
input: Record<string, unknown>,
103113
apiKey: string,
114+
tool: FalMediaTool,
104115
signal?: AbortSignal
105116
): Promise<FalQueueResult> {
106117
const ceiling = new AbortController()
@@ -110,7 +121,7 @@ export async function runFalQueue(
110121
)
111122
const queueSignal = signal ? AbortSignal.any([signal, ceiling.signal]) : ceiling.signal
112123
try {
113-
return await submitAndPollFalQueue(endpoint, input, apiKey, queueSignal)
124+
return await submitAndPollFalQueue(endpoint, input, apiKey, tool, queueSignal)
114125
} finally {
115126
clearTimeout(ceilingTimer)
116127
}
@@ -120,6 +131,7 @@ async function submitAndPollFalQueue(
120131
endpoint: string,
121132
input: Record<string, unknown>,
122133
apiKey: string,
134+
tool: FalMediaTool,
123135
queueSignal: AbortSignal
124136
): Promise<FalQueueResult> {
125137
// Only a submit already under way outlives its caller; a stopped call never starts one.
@@ -135,6 +147,14 @@ async function submitAndPollFalQueue(
135147
signal: AbortSignal.timeout(FAL_SUBMIT_TIMEOUT_MS),
136148
})
137149
if (!createResponse.ok) {
150+
if (FAL_ACCOUNT_REFUSAL_STATUSES.has(createResponse.status)) {
151+
hostedKeyMetrics.recordFailed({
152+
provider: 'falai',
153+
tool,
154+
key: 'FALAI_API_KEY',
155+
reason: 'auth',
156+
})
157+
}
138158
const err = await readResponseTextWithLimit(createResponse, {
139159
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
140160
label: 'Fal.ai create error response',

0 commit comments

Comments
 (0)