Skip to content

Commit 069d18e

Browse files
myxamediyarclaude
myxamediyar
and
claude
committed
feat(video): accept a start frame image for fal.ai video models
Fal models with an image-to-video sibling endpoint now take an optional start frame. When one is attached, requests go to the image-to-video endpoint with the image as a data URI in the model's image field, and the block's settings dropdowns follow that endpoint's options. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 99c5a92 commit 069d18e

11 files changed

Lines changed: 1053 additions & 161 deletions

File tree

‎apps/sim/blocks/blocks/video_generator.ts‎

Lines changed: 96 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@ import {
44
directVideoModels,
55
getDirectVideoModel,
66
} from '@/lib/media/direct-video-catalog'
7-
import { falVideoModels, getFalVideoModel } from '@/lib/media/fal-video-catalog'
7+
import {
8+
type FalVideoSettings,
9+
falVideoModels,
10+
getFalVideoModel,
11+
getFalVideoVariant,
12+
} from '@/lib/media/fal-video-catalog'
813
import { AuthMode, type BlockConfig, IntegrationType, type SubBlockConfig } from '@/blocks/types'
914
import { normalizeFileInput, parseOptionalBooleanInput } from '@/blocks/utils'
1015

@@ -42,40 +47,55 @@ const FALAI_AUDIO_DEFAULT_OFF_MODELS = ['kling-o3-pro', 'kling-o3-4k']
4247
const FAL_MODEL_DEFAULT = { id: 'default', label: 'Model default' } as const
4348
const endpointModels = falVideoModels.filter((model) => !model.legacyId)
4449

45-
function tuningField(
46-
id: string,
47-
title: string,
48-
key: 'durations' | 'aspectRatios' | 'resolutions' | 'audioField' | 'promptOptimizer'
49-
): SubBlockConfig {
50+
const START_FRAME_FIELDS = ['visualReferenceInput', 'visualReferenceUpload']
51+
const HIDDEN = { field: 'provider', value: [] }
52+
53+
function hasStartFrame(values?: Record<string, unknown>) {
54+
return [
55+
values?.visualReference,
56+
values?.visualReferenceInput,
57+
values?.visualReferenceUpload,
58+
].some((reference) => (Array.isArray(reference) ? reference.length > 0 : Boolean(reference)))
59+
}
60+
61+
/** Settings of the endpoint a fal run will call: image-to-video once a start frame is attached. */
62+
function selectedFalSettings(values?: Record<string, unknown>) {
63+
const model = getFalVideoModel(values?.model)
64+
return getFalVideoVariant(values?.model, Boolean(model?.image) && hasStartFrame(values))
65+
}
66+
67+
type FalSettingKey = 'durations' | 'aspectRatios' | 'resolutions' | 'audioField' | 'promptOptimizer'
68+
69+
function tuningField(id: string, title: string, key: FalSettingKey): SubBlockConfig {
5070
return {
5171
id,
5272
title,
5373
type: 'dropdown',
5474
value: () => FAL_MODEL_DEFAULT.id,
55-
condition: {
56-
field: 'provider',
57-
value: 'falai',
58-
and: {
59-
field: 'model',
60-
value: endpointModels.filter((model) => model[key]).map((model) => model.modelId),
61-
},
75+
condition: (values) => {
76+
const model = getFalVideoModel(values?.model)
77+
return model && !model.legacyId && selectedFalSettings(values)?.[key]
78+
? { field: 'provider', value: 'falai' }
79+
: HIDDEN
6280
},
63-
dependsOn: ['model'],
81+
dependsOn: ['model', ...START_FRAME_FIELDS],
6482
options: (params) => {
6583
if (!params) {
66-
const values = endpointModels.flatMap((model) => {
67-
const choices = model[key]
68-
return Array.isArray(choices)
69-
? choices.map((value) =>
70-
key === 'durations' ? String(Number.parseFloat(String(value))) : String(value)
71-
)
72-
: choices
73-
? ['true', 'false']
74-
: []
75-
})
84+
const values = endpointModels.flatMap((model) =>
85+
[model, model.image].flatMap((settings) => {
86+
const choices = settings?.[key]
87+
return Array.isArray(choices)
88+
? choices.map((value) =>
89+
key === 'durations' ? String(Number.parseFloat(String(value))) : String(value)
90+
)
91+
: choices
92+
? ['true', 'false']
93+
: []
94+
})
95+
)
7696
return [FAL_MODEL_DEFAULT, ...[...new Set(values)].map((id) => ({ id, label: id }))]
7797
}
78-
const model = getFalVideoModel(params?.values.model)
98+
const model = selectedFalSettings(params?.values)
7999
if (!model?.[key]) return []
80100
if (key === 'audioField' || key === 'promptOptimizer') {
81101
return [FAL_MODEL_DEFAULT, { id: 'true', label: 'On' }, { id: 'false', label: 'Off' }]
@@ -109,6 +129,15 @@ const falVideoFields = [
109129
tuningField('falPromptOptimizer', 'Prompt Optimizer', 'promptOptimizer'),
110130
]
111131

132+
/** Settings fields that models with a legacy ID keep, and the catalog setting each one sets. */
133+
const LEGACY_FAL_SETTING_KEYS: Record<string, keyof FalVideoSettings> = {
134+
duration: 'durations',
135+
aspectRatio: 'aspectRatios',
136+
resolution: 'resolutions',
137+
generateAudio: 'audioField',
138+
promptOptimizer: 'promptOptimizer',
139+
}
140+
112141
function withFalVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
113142
return subBlocks.flatMap((subBlock) => {
114143
const condition = subBlock.condition
@@ -141,17 +170,27 @@ function withFalVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
141170
]
142171
}
143172
if (subBlock.id === 'apiKey' && !condition.not) return [...falVideoFields, subBlock]
173+
const key = LEGACY_FAL_SETTING_KEYS[subBlock.id]
174+
if (key)
175+
return [
176+
{
177+
...subBlock,
178+
condition: (values) => {
179+
const image = getFalVideoModel(values?.model)?.image
180+
return image && !image[key] && hasStartFrame(values) ? HIDDEN : condition
181+
},
182+
dependsOn: [
183+
...(Array.isArray(subBlock.dependsOn) ? subBlock.dependsOn : []),
184+
...START_FRAME_FIELDS,
185+
],
186+
},
187+
]
144188
return [subBlock]
145189
})
146190
}
147191

148192
function selectedDirectModel(values?: Record<string, unknown>) {
149-
const withImage = [
150-
values?.visualReference,
151-
values?.visualReferenceInput,
152-
values?.visualReferenceUpload,
153-
].some((reference) => (Array.isArray(reference) ? reference.length > 0 : Boolean(reference)))
154-
return getDirectVideoModel(values?.provider, values?.model, withImage)
193+
return getDirectVideoModel(values?.provider, values?.model, hasStartFrame(values))
155194
}
156195

157196
const directVideoFields = (
@@ -168,7 +207,7 @@ const directVideoFields = (
168207
title,
169208
type: key === 'promptOptimizer' ? 'switch' : 'dropdown',
170209
value: key === 'promptOptimizer' ? undefined : () => FAL_MODEL_DEFAULT.id,
171-
dependsOn: ['provider', 'model', 'visualReferenceInput', 'visualReferenceUpload'],
210+
dependsOn: ['provider', 'model', ...START_FRAME_FIELDS],
172211
condition: (values) => ({
173212
field: 'provider',
174213
value: selectedDirectModel(values)?.settings?.[key] ? [String(values?.provider)] : [],
@@ -301,9 +340,18 @@ function withDirectVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
301340
dependsOn: ['provider', 'model'],
302341
},
303342
]
304-
if (field.id === 'visualReferenceUpload' || field.id === 'visualReferenceInput')
343+
if (START_FRAME_FIELDS.includes(field.id))
305344
return [
306-
{ ...field, required: { field: 'model', value: ['gen4_turbo', 'gen-4', 'gen-4-turbo'] } },
345+
{
346+
...field,
347+
condition: (values) =>
348+
values?.provider === 'runway' ||
349+
(values?.provider === 'falai' && getFalVideoModel(values.model)?.image)
350+
? { field: 'provider', value: ['runway', 'falai'] }
351+
: HIDDEN,
352+
dependsOn: ['provider', 'model'],
353+
required: { field: 'model', value: ['gen4_turbo', 'gen-4', 'gen-4-turbo'] },
354+
},
307355
]
308356
return [field]
309357
})
@@ -1928,8 +1976,20 @@ export const VideoGeneratorV3Block: BlockConfig = {
19281976
config: {
19291977
tool: (params) => VideoGeneratorV2Block.tools.config?.tool(params) ?? 'video_runway',
19301978
params: (params) => {
1931-
if (params.provider === 'falai')
1932-
return VideoGeneratorV2Block.tools.config?.params?.(params) ?? params
1979+
if (params.provider === 'falai') {
1980+
const falParams = VideoGeneratorV2Block.tools.config?.params?.(params) ?? params
1981+
if (!getFalVideoModel(params.model)?.image || !falParams.visualReference)
1982+
return { ...falParams, visualReference: undefined }
1983+
const image = getFalVideoVariant(params.model, true)
1984+
return {
1985+
...falParams,
1986+
duration: image?.durations ? falParams.duration : undefined,
1987+
aspectRatio: image?.aspectRatios ? falParams.aspectRatio : undefined,
1988+
resolution: image?.resolutions ? falParams.resolution : undefined,
1989+
generateAudio: image?.audioField ? falParams.generateAudio : undefined,
1990+
promptOptimizer: image?.promptOptimizer ? falParams.promptOptimizer : undefined,
1991+
}
1992+
}
19331993
const visualReference =
19341994
params.provider === 'runway'
19351995
? normalizeFileInput(params.visualReference, { single: true })

‎apps/sim/lib/internal/video/client.test.ts‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import {
33
executionLimitsMock,
44
executionLimitsMockFns,
55
} from '@sim/testing/mocks/execution-limits.mock'
6+
import {
7+
fileUtilsServerMock,
8+
fileUtilsServerMockFns,
9+
} from '@sim/testing/mocks/file-utils-server.mock'
610
import {
711
inputValidationMock,
812
inputValidationMockFns,
@@ -14,8 +18,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
1418
vi.mock('@/lib/uploads', () => uploadsMock)
1519
vi.mock('@/lib/core/execution-limits', () => executionLimitsMock)
1620
vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
21+
vi.mock('@/lib/uploads/utils/file.server', () => fileUtilsServerMock)
1722

18-
import { generateVideo } from '@/lib/internal/video/client'
23+
import { generateVideo, getVideoInputValidationError } from '@/lib/internal/video/client'
1924
import { executeVideoOperation } from '@/lib/internal/video/operations'
2025
import { observeServiceCosts } from '@/lib/mothership/billing/service-observer'
2126

@@ -472,6 +477,55 @@ describe('Video provider client', () => {
472477
expect(fetchMock).not.toHaveBeenCalled()
473478
})
474479

480+
it.each([
481+
{
482+
model: 'kling-v3-pro',
483+
endpoint: 'fal-ai/kling-video/v3/pro/image-to-video',
484+
imageField: 'start_image_url',
485+
},
486+
{
487+
model: 'veo-3.1',
488+
endpoint: 'fal-ai/veo3.1/image-to-video',
489+
imageField: 'image_url',
490+
},
491+
])('sends a start frame to $endpoint as $imageField', async ({ model, endpoint, imageField }) => {
492+
fileUtilsServerMockFns.mockDownloadFileFromStorage.mockResolvedValueOnce(Buffer.from('png'))
493+
const mockFetch = vi
494+
.fn<typeof fetch>()
495+
.mockResolvedValueOnce(jsonResponse({ request_id: 'fal-1' }))
496+
.mockResolvedValueOnce(jsonResponse({ status: 'COMPLETED' }))
497+
.mockResolvedValueOnce(jsonResponse({ video: { url: 'data:video/mp4;base64,dmlkZW8=' } }))
498+
vi.stubGlobal('fetch', mockFetch)
499+
const frame = { id: 'f', name: 'frame.png', url: '', size: 3, type: 'image/png', key: 'k' }
500+
501+
const result = generateVideo(
502+
{ provider: 'falai', apiKey: 'key', model, prompt: 'Pan left', visualReference: frame },
503+
{ requestId: 'request-1' }
504+
)
505+
await Promise.all([result, vi.advanceTimersByTimeAsync(5000)])
506+
507+
expect(String(mockFetch.mock.calls[0]?.[0])).toBe(`https://queue.fal.run/${endpoint}`)
508+
expect(JSON.parse(String(mockFetch.mock.calls[0]?.[1]?.body))).toEqual(
509+
expect.objectContaining({
510+
prompt: 'Pan left',
511+
[imageField]: `data:image/png;base64,${Buffer.from('png').toString('base64')}`,
512+
})
513+
)
514+
})
515+
516+
it('rejects a start frame for a fal model without an image-to-video endpoint', () => {
517+
const frame = { id: 'f', name: 'frame.png', url: '', size: 3, type: 'image/png', key: 'k' }
518+
expect(
519+
getVideoInputValidationError({
520+
provider: 'falai',
521+
apiKey: 'key',
522+
model: 'minimax/h3-max/styles/vhs',
523+
prompt: 'Pan left',
524+
visualReference: frame,
525+
})
526+
).toBe('Fal.ai model minimax/h3-max/styles/vhs does not accept a start frame image')
527+
})
528+
475529
it('retrieves a nested endpoint result when fal omits the queue URLs', async () => {
476530
const app = 'fal-ai/kandinsky5'
477531
const model = `${app}/text-to-video`

‎apps/sim/lib/internal/video/client.ts‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,22 @@ export async function generateVideo(
112112
: generateWithDirectProvider(input, context)
113113
}
114114

115+
/** Keeps the encoded image under 5 MB, the largest data URI Runway accepts. */
116+
const MAX_REFERENCE_IMAGE_BYTES = Math.floor(((5 * 1024 * 1024 - 512) * 3) / 4)
117+
118+
async function readReferenceImageDataUri(
119+
file: UserFile,
120+
requestId: string,
121+
label: string
122+
): Promise<string> {
123+
assertKnownSizeWithinLimit(file.size, MAX_REFERENCE_IMAGE_BYTES, label)
124+
const buffer = await downloadFileFromStorage(file, requestId, logger, {
125+
maxBytes: MAX_REFERENCE_IMAGE_BYTES,
126+
})
127+
assertKnownSizeWithinLimit(buffer.length, MAX_REFERENCE_IMAGE_BYTES, label)
128+
return `data:${file.type};base64,${buffer.toString('base64')}`
129+
}
130+
115131
async function generateWithDirectProvider(
116132
input: VideoGenerationInput,
117133
context: VideoGenerationContext
@@ -127,17 +143,11 @@ async function generateWithDirectProvider(
127143
if (protocol === 'runway') headers['X-Runway-Version'] = '2024-11-06'
128144

129145
if (input.visualReference) {
130-
const maxReferenceBytes = Math.floor(((5 * 1024 * 1024 - 512) * 3) / 4)
131-
assertKnownSizeWithinLimit(
132-
input.visualReference.size,
133-
maxReferenceBytes,
146+
prepared.body.promptImage = await readReferenceImageDataUri(
147+
input.visualReference,
148+
requestId,
134149
'Runway reference image'
135150
)
136-
const buffer = await downloadFileFromStorage(input.visualReference, requestId, logger, {
137-
maxBytes: maxReferenceBytes,
138-
})
139-
assertKnownSizeWithinLimit(buffer.length, maxReferenceBytes, 'Runway reference image')
140-
prepared.body.promptImage = `data:${input.visualReference.type};base64,${buffer.toString('base64')}`
141151
}
142152
signal?.throwIfAborted()
143153
const readProviderResponse = async (response: Response): Promise<Record<string, unknown>> => {
@@ -364,7 +374,15 @@ async function generateWithFalAI(
364374
const { requestId, signal } = context
365375
signal?.throwIfAborted()
366376
logger.info(`Starting Fal.ai generation with model: ${model}`, { requestId })
367-
const { endpoint, body } = buildFalVideoRequest(input)
377+
const { endpoint, imageField, body } = buildFalVideoRequest(input)
378+
if (input.visualReference && imageField) {
379+
body[imageField] = await readReferenceImageDataUri(
380+
input.visualReference,
381+
requestId,
382+
'Fal.ai start frame image'
383+
)
384+
signal?.throwIfAborted()
385+
}
368386
const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, {
369387
method: 'POST',
370388
redirect: 'error',

0 commit comments

Comments
 (0)