Skip to content

Commit 2eb80dd

Browse files
myxamediyarclaudewaleedlatif1
authored
feat(media): add Higgsfield to video and image generators, allow typed models (#9756)
* feat(media): add Higgsfield to video and image generators, allow typed models - Add Higgsfield as a provider in the Video Generator (Kling 3.0, Kling 2.5 Turbo, Seedance 2.0, MiniMax H3, Hailuo 2.3) and Image Generator (SOUL, SOUL V2, SOUL Cinema, Z-Image Turbo) - Switch the video and image model fields to comboboxes so users can type a model the catalog does not list - Send typed model IDs with the prompt only; validate them as safe model names or endpoint paths - Keep retired model errors, reject stale cross-provider values, and require a user Fal.ai key for custom Fal endpoints * fix(media): trim typed model IDs, guard reference images, restore default after provider switch - trim `model` at the image and video input schemas so a padded catalog ID still matches its catalog entry - reject `visualReference` for typed direct-video models, which only send the prompt - refill the combobox default after a provider switch even if the user had cleared the field - update the image operations test to the typed-model behavior Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(media): test typed image models at the request builder, document Higgsfield helpers - move the typed-model checks from mocked fetch calls to buildImageGenerationRequest, covering padded catalog and retired IDs - add TSDoc to higgsfieldStatusUrl and higgsfieldFailureMessage - say in the combobox comments that a cleared required field gets its default back when reopened Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor(higgsfield): share one API client between the block tools and media generators Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Waleed Latif <walif6@gmail.com>
1 parent 99c5a92 commit 2eb80dd

30 files changed

Lines changed: 813 additions & 75 deletions

File tree

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx‎

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,13 @@ const CREATE_ACTION_VALUE = '__sub-block-create-action__'
5050
*/
5151
type ComboBoxOption =
5252
| string
53-
| { label: string; id: string; icon?: React.ComponentType<{ className?: string }> }
53+
| {
54+
label: string
55+
id: string
56+
icon?: React.ComponentType<{ className?: string }>
57+
/** Left out of the list but still resolves a stored value to its label. */
58+
hidden?: boolean
59+
}
5460

5561
/**
5662
* Props for the ComboBox component
@@ -141,12 +147,12 @@ export const ComboBox = memo(function ComboBox({
141147
? options({ values: blockValues ?? {} })
142148
: (options ?? EMPTY_OPTIONS)
143149

144-
if (subBlockId === 'model') {
150+
if (subBlockId === 'model' && !config.mediaModelOptions) {
145151
return opts.filter((opt) => isModelUsable(typeof opt === 'string' ? opt : opt.id))
146152
}
147153

148154
return opts
149-
}, [options, blockValues, subBlockId, isModelUsable, deploymentShape])
155+
}, [options, blockValues, subBlockId, isModelUsable, deploymentShape, config.mediaModelOptions])
150156

151157
const [selectorSearch, setSelectorSearch] = useState('')
152158
const debouncedSelectorSearch = useDebounce(selectorSearch.trim(), SEARCH_DEBOUNCE_MS)
@@ -237,7 +243,12 @@ export const ComboBox = memo(function ComboBox({
237243
let opts: ComboBoxOption[] =
238244
isDynamic && normalizedFetchedOptions.length > 0 ? normalizedFetchedOptions : staticOptions
239245

240-
if (subBlockId === 'model' && isDynamic && normalizedFetchedOptions.length > 0) {
246+
if (
247+
subBlockId === 'model' &&
248+
!config.mediaModelOptions &&
249+
isDynamic &&
250+
normalizedFetchedOptions.length > 0
251+
) {
241252
opts = opts.filter((opt) => isModelUsable(typeof opt === 'string' ? opt : opt.id))
242253
}
243254

@@ -272,6 +283,7 @@ export const ComboBox = memo(function ComboBox({
272283
createdOption,
273284
subBlockId,
274285
isModelUsable,
286+
config.mediaModelOptions,
275287
])
276288

277289
// Convert options to Combobox format
@@ -280,7 +292,7 @@ export const ComboBox = memo(function ComboBox({
280292
if (typeof option === 'string') {
281293
return { label: option, value: option }
282294
}
283-
return { label: option.label, value: option.id, icon: option.icon }
295+
return { label: option.label, value: option.id, icon: option.icon, hidden: option.hidden }
284296
})
285297
return createOption ? [createOption, ...mapped] : mapped
286298
}, [evaluatedOptions, createOption])
@@ -325,8 +337,11 @@ export const ComboBox = memo(function ComboBox({
325337
return undefined
326338
}
327339

328-
if (evaluatedOptions.length > 0) {
329-
return getOptionValue(evaluatedOptions[0])
340+
const firstSelectable = evaluatedOptions.find(
341+
(option) => typeof option === 'string' || !option.hidden
342+
)
343+
if (firstSelectable) {
344+
return getOptionValue(firstSelectable)
330345
}
331346

332347
return undefined
@@ -356,16 +371,33 @@ export const ComboBox = memo(function ComboBox({
356371
setInputValue(displayValue)
357372
}
358373

374+
/**
375+
* The default in force when the user emptied the input, so the field is not refilled while they
376+
* type. A dependency change that brings a different default (a provider switch) ends it, and so
377+
* does a remount: a required field saved empty gets its default back when reopened.
378+
*/
379+
const userClearedDefaultRef = useRef<string | null>(null)
380+
359381
// Set default value once permissions are loaded
360382
useEffect(() => {
361383
if (isPermissionLoading) return
362384
if (defaultOptionValue === undefined) return
363385

364-
// Only set default when no value exists (initial block add)
365-
if (value === null || value === undefined) {
386+
// Set the default on initial block add, and after a dependency change clears an opted-in field
387+
const clearedByDependency =
388+
config.restoreDefaultWhenCleared &&
389+
value === '' &&
390+
userClearedDefaultRef.current !== defaultOptionValue
391+
if (value === null || value === undefined || clearedByDependency) {
366392
setStoreValue(defaultOptionValue)
367393
}
368-
}, [value, defaultOptionValue, setStoreValue, isPermissionLoading])
394+
}, [
395+
value,
396+
defaultOptionValue,
397+
setStoreValue,
398+
isPermissionLoading,
399+
config.restoreDefaultWhenCleared,
400+
])
369401

370402
/**
371403
* Handles wheel event for ReactFlow zoom control
@@ -521,9 +553,10 @@ export const ComboBox = memo(function ComboBox({
521553
? matchedOption
522554
: matchedOption.id
523555
: newValue
556+
userClearedDefaultRef.current = nextValue === '' ? (defaultOptionValue ?? null) : null
524557
setStoreValue(nextValue)
525558
},
526-
[isPreview, evaluatedOptions, setStoreValue]
559+
[isPreview, evaluatedOptions, setStoreValue, defaultOptionValue]
527560
)
528561

529562
return (

‎apps/sim/blocks/blocks/image_generator.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ export const ImageGeneratorV2Block: BlockConfig = {
281281
description: 'Generate images',
282282
authMode: AuthMode.ApiKey,
283283
longDescription:
284-
'Generate images using OpenAI GPT Image, Google Nano Banana, or Fal.ai image models.',
284+
'Generate images using OpenAI GPT Image, Google Nano Banana, Higgsfield SOUL and Z-Image, or Fal.ai image models.',
285285
docsLink: 'https://docs.sim.ai/integrations/image_generator',
286286
category: 'blocks',
287287
integrationType: IntegrationType.AI,
@@ -304,16 +304,20 @@ export const ImageGeneratorV2Block: BlockConfig = {
304304
options: [
305305
{ label: 'OpenAI', id: 'openai' },
306306
{ label: 'Google Gemini', id: 'gemini' },
307+
{ label: 'Higgsfield', id: 'higgsfield' },
307308
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
308309
],
309310
commandSearchable: true,
310311
value: () => 'falai',
311312
},
312-
...(['openai', 'gemini', 'falai'] as const).map(
313+
...(['openai', 'gemini', 'higgsfield', 'falai'] as const).map(
313314
(provider): SubBlockConfig => ({
314315
id: 'model',
315316
title: 'Model',
316-
type: 'dropdown',
317+
type: 'combobox',
318+
placeholder: 'Type or select a model...',
319+
mediaModelOptions: true,
320+
restoreDefaultWhenCleared: true,
317321
options: (params) =>
318322
imageModels
319323
.filter((model) => !params || model.provider === provider)

‎apps/sim/blocks/blocks/video_generator.ts‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,10 +243,11 @@ function withDirectVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
243243
{ id: 'veo', label: 'Google Gemini' },
244244
{ id: 'luma', label: 'Luma' },
245245
{ id: 'minimax', label: 'MiniMax' },
246+
{ id: 'higgsfield', label: 'Higgsfield' },
246247
{ id: 'falai', label: 'Fal.ai (Multi-Model)' },
247248
],
248249
},
249-
...(['runway', 'veo', 'luma', 'minimax'] as const).map(
250+
...(['runway', 'veo', 'luma', 'minimax', 'higgsfield'] as const).map(
250251
(provider): SubBlockConfig => ({
251252
id: 'model',
252253
title: 'Model',
@@ -1908,25 +1909,46 @@ export const VideoGeneratorV2Block: BlockConfig = {
19081909
},
19091910
}
19101911

1912+
/** Lets a user type a model the catalog does not list yet, as well as pick a listed one. */
1913+
function withTypedModelFields(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
1914+
return subBlocks.map((field) =>
1915+
field.id === 'model'
1916+
? {
1917+
...field,
1918+
type: 'combobox',
1919+
placeholder: 'Type or select a model...',
1920+
mediaModelOptions: true,
1921+
restoreDefaultWhenCleared: true,
1922+
}
1923+
: field
1924+
)
1925+
}
1926+
19111927
export const VideoGeneratorV3Block: BlockConfig = {
19121928
...VideoGeneratorV2Block,
19131929
sunset: undefined,
19141930
type: 'video_generator_v3',
19151931
name: 'Video Generator',
19161932
description: 'Generate videos from text using AI',
19171933
longDescription:
1918-
'Generate high-quality videos from text prompts using leading AI providers. Supports Runway, Google Gemini, Luma, MiniMax, and Fal.ai multi-model generation with provider-specific durations, aspect ratios, resolutions, prompt optimization, and native audio controls.',
1934+
'Generate high-quality videos from text prompts using leading AI providers. Supports Runway, Google Gemini, Luma, MiniMax, Higgsfield, and Fal.ai multi-model generation with provider-specific durations, aspect ratios, resolutions, prompt optimization, and native audio controls.',
19191935
docsLink: 'https://docs.sim.ai/integrations/video_generator',
19201936
category: 'blocks',
19211937
integrationType: IntegrationType.AI,
19221938
bgColor: '#181C1E',
19231939
icon: VideoIcon,
19241940
hideFromToolbar: false,
1925-
subBlocks: withDirectVideoCatalog(withFalVideoCatalog(VideoGeneratorV2Block.subBlocks)),
1941+
subBlocks: withTypedModelFields(
1942+
withDirectVideoCatalog(withFalVideoCatalog(VideoGeneratorV2Block.subBlocks))
1943+
),
19261944
tools: {
19271945
...VideoGeneratorV2Block.tools,
1946+
access: [...VideoGeneratorV2Block.tools.access, 'video_higgsfield'],
19281947
config: {
1929-
tool: (params) => VideoGeneratorV2Block.tools.config?.tool(params) ?? 'video_runway',
1948+
tool: (params) =>
1949+
params.provider === 'higgsfield'
1950+
? 'video_higgsfield'
1951+
: (VideoGeneratorV2Block.tools.config?.tool(params) ?? 'video_runway'),
19301952
params: (params) => {
19311953
if (params.provider === 'falai')
19321954
return VideoGeneratorV2Block.tools.config?.params?.(params) ?? params
@@ -1969,6 +1991,10 @@ export const VideoGeneratorV3Block: BlockConfig = {
19691991
},
19701992
inputs: {
19711993
...VideoGeneratorV2Block.inputs,
1994+
provider: {
1995+
type: 'string',
1996+
description: 'Video generation provider (runway, veo, luma, minimax, higgsfield, falai)',
1997+
},
19721998
duration: { type: 'number', description: 'Duration for the selected provider model' },
19731999
aspectRatio: {
19742000
type: 'string',

‎apps/sim/blocks/types.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,16 @@ export interface SubBlockConfig {
306306
* silently writes and persists a value the user never picked.
307307
*/
308308
emptyIsValid?: boolean
309+
/**
310+
* The `model` options are image or video generation models, not LLMs, so a combobox does not
311+
* run them through the permission group's LLM model filter (a dropdown never did).
312+
*/
313+
mediaModelOptions?: boolean
314+
/**
315+
* Refills a combobox's default when a dependency change clears it, as a dropdown does. Text
316+
* the user deletes themselves stays empty while they edit, so they can type a new value.
317+
*/
318+
restoreDefaultWhenCleared?: boolean
309319
/** Clears a persisted id after its authoritative single-option fetch confirms deletion. */
310320
clearOnMissingOption?: boolean
311321
/**

‎apps/sim/lib/api/contracts/tools/media/video.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/c
33
import { toolBooleanSchema } from '@/lib/api/contracts/tools/media/shared'
44
import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata'
55

6-
export const videoProviders = ['runway', 'veo', 'luma', 'minimax', 'falai'] as const
6+
export const videoProviders = ['runway', 'veo', 'luma', 'minimax', 'falai', 'higgsfield'] as const
77
const MISSING_VIDEO_FIELDS_ERROR = 'Missing required fields: provider, apiKey, and prompt'
88

99
export const videoToolBodySchema = z
@@ -15,7 +15,7 @@ export const videoToolBodySchema = z
1515
message: `Invalid provider. Must be one of: ${videoProviders.join(', ')}`,
1616
}),
1717
apiKey: z.string({ error: MISSING_VIDEO_FIELDS_ERROR }).min(1, MISSING_VIDEO_FIELDS_ERROR),
18-
model: z.string().optional(),
18+
model: z.string().trim().optional(),
1919
prompt: z.string({ error: MISSING_VIDEO_FIELDS_ERROR }).min(1, MISSING_VIDEO_FIELDS_ERROR),
2020
duration: z.coerce.number().optional(),
2121
aspectRatio: z.string().optional(),

‎apps/sim/lib/credentials/token-service-accounts/validators/higgsfield.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ import type {
77
TokenServiceAccountFields,
88
TokenServiceAccountValidationResult,
99
} from '@/lib/credentials/token-service-accounts/server'
10+
import { HIGGSFIELD_API_BASE, higgsfieldAuthHeaders } from '@/lib/media/higgsfield'
1011

1112
/**
1213
* Higgsfield has no identity endpoint, so the key is checked against the documented request
1314
* status route with an ID no account owns: an accepted key gets 404, a rejected one gets 401.
1415
*/
15-
const HIGGSFIELD_VERIFY_URL =
16-
'https://api.higgsfield.ai/requests/00000000-0000-4000-8000-000000000000/status'
16+
const HIGGSFIELD_VERIFY_URL = `${HIGGSFIELD_API_BASE}/requests/00000000-0000-4000-8000-000000000000/status`
1717
const KEY_PATTERN = /^([^:\s]+):[^:\s]+$/
1818

1919
/** Verifies a Higgsfield `KEY_ID:KEY_SECRET` pair. */
@@ -31,7 +31,7 @@ export async function validateHiggsfieldServiceAccount(
3131
const response = await fetchProvider(
3232
HIGGSFIELD_VERIFY_URL,
3333
{
34-
headers: { Authorization: `Key ${fields.apiToken.trim()}`, Accept: 'application/json' },
34+
headers: higgsfieldAuthHeaders(fields.apiToken),
3535
redirect: 'error',
3636
},
3737
'get_request_status'

‎apps/sim/lib/internal/image/operations.test.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,14 +178,13 @@ describe('image operations', () => {
178178
})
179179

180180
it.each(['openai', 'gemini'])(
181-
'does not purchase a fallback for an unknown %s model',
181+
'rejects a malformed typed %s model before any request',
182182
async (provider) => {
183-
mocks.fetch.mockResolvedValueOnce(new Response('Unexpected generation', { status: 502 }))
184183
const response = await executeImageGeneration(
185-
{ ...falInput, provider, model: 'unknown-model' },
184+
{ ...falInput, provider, model: '../unknown model' },
186185
{
187186
userId: 'user-1',
188-
requestId: 'unknown-model',
187+
requestId: 'malformed-model',
189188
}
190189
)
191190

0 commit comments

Comments
 (0)