Skip to content

Commit b5660ac

Browse files
authored
feat(video-generator): refresh native provider catalogs and request protocols (#9658)
* feat(video-generator): refresh native provider catalogs and request protocols * fix(video-generator): preserve saved settings and fix CI * fix(video-generator): preserve Agent audio and model prompt limits * fix(video-generator): retain empty controls and isolate CI scenarios
1 parent 1990323 commit b5660ac

23 files changed

Lines changed: 2140 additions & 738 deletions

‎apps/sim/blocks/blocks/video_generator.ts‎

Lines changed: 239 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
import { VideoIcon } from '@/components/icons'
2+
import {
3+
directVideoDefaults,
4+
directVideoModels,
5+
getDirectVideoModel,
6+
} from '@/lib/media/direct-video-catalog'
27
import { falVideoModels, getFalVideoModel } from '@/lib/media/fal-video-catalog'
38
import { AuthMode, type BlockConfig, IntegrationType, type SubBlockConfig } from '@/blocks/types'
49
import { normalizeFileInput, parseOptionalBooleanInput } from '@/blocks/utils'
@@ -57,6 +62,19 @@ function tuningField(
5762
},
5863
dependsOn: ['model'],
5964
options: (params) => {
65+
if (!params) {
66+
const values = endpointModels.flatMap((model) => {
67+
const choices = model[key]
68+
return Array.isArray(choices)
69+
? choices.map((value) =>
70+
key === 'durations' ? String(Number.parseFloat(String(value))) : String(value)
71+
)
72+
: choices
73+
? ['true', 'false']
74+
: []
75+
})
76+
return [FAL_MODEL_DEFAULT, ...[...new Set(values)].map((id) => ({ id, label: id }))]
77+
}
6078
const model = getFalVideoModel(params?.values.model)
6179
if (!model?.[key]) return []
6280
if (key === 'audioField' || key === 'promptOptimizer') {
@@ -127,6 +145,170 @@ function withFalVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
127145
})
128146
}
129147

148+
function selectedDirectModel(values?: Record<string, unknown>) {
149+
const withImage = [
150+
values?.visualReference,
151+
values?.visualReferenceInput,
152+
values?.visualReferenceUpload,
153+
].some((reference) => (Array.isArray(reference) ? reference.length > 0 : Boolean(reference)))
154+
return getDirectVideoModel(values?.provider, values?.model, withImage)
155+
}
156+
157+
const directVideoFields = (
158+
[
159+
['duration', 'Duration (seconds)', 'durations'],
160+
['aspectRatio', 'Aspect Ratio', 'aspectRatios'],
161+
['resolution', 'Resolution', 'resolutions'],
162+
['directGenerateAudio', 'Generate Audio', 'audio'],
163+
['promptOptimizer', 'Prompt Optimizer', 'promptOptimizer'],
164+
] as const
165+
).map(
166+
([id, title, key]): SubBlockConfig => ({
167+
id,
168+
title,
169+
type: key === 'promptOptimizer' ? 'switch' : 'dropdown',
170+
value: key === 'promptOptimizer' ? undefined : () => FAL_MODEL_DEFAULT.id,
171+
dependsOn: ['provider', 'model', 'visualReferenceInput', 'visualReferenceUpload'],
172+
condition: (values) => ({
173+
field: 'provider',
174+
value: selectedDirectModel(values)?.settings?.[key] ? [String(values?.provider)] : [],
175+
}),
176+
options: (params) => {
177+
if (!params) {
178+
const values = directVideoModels.flatMap((model) =>
179+
[model.text, model.image].flatMap((settings) => {
180+
const choices = settings?.[key]
181+
return Array.isArray(choices) ? choices.map(String) : choices ? ['true', 'false'] : []
182+
})
183+
)
184+
return [FAL_MODEL_DEFAULT, ...[...new Set(values)].map((id) => ({ id, label: id }))]
185+
}
186+
const model = selectedDirectModel(params?.values)
187+
const choices = model?.settings?.[key]
188+
if (!choices) return []
189+
if (typeof choices === 'boolean')
190+
return [FAL_MODEL_DEFAULT, { id: 'true', label: 'On' }, { id: 'false', label: 'Off' }]
191+
return [
192+
FAL_MODEL_DEFAULT,
193+
...choices
194+
.filter((value) => {
195+
if (model?.protocol !== 'minimax-v1') return true
196+
if (key === 'durations' && params?.values.resolution === '1080P') return value === 6
197+
if (key === 'resolutions' && Number(params?.values.duration) === 10)
198+
return value !== '1080P'
199+
return true
200+
})
201+
.map((value) => ({ id: String(value), label: String(value) })),
202+
]
203+
},
204+
})
205+
)
206+
207+
function withDirectVideoCatalog(subBlocks: SubBlockConfig[]): SubBlockConfig[] {
208+
return subBlocks.flatMap((field): SubBlockConfig[] => {
209+
if (
210+
field.id === 'model' &&
211+
typeof field.condition === 'object' &&
212+
field.condition.value === 'falai'
213+
) {
214+
const options = field.options
215+
return [
216+
{
217+
...field,
218+
options: (params) => {
219+
const falOptions = typeof options === 'function' ? options(params) : (options ?? [])
220+
return params
221+
? falOptions
222+
: [
223+
...falOptions,
224+
...directVideoModels.flatMap((model) => [
225+
{ id: model.id, label: model.label },
226+
...(model.aliases ?? []).map((id) => ({
227+
id,
228+
label: model.label,
229+
hidden: true,
230+
})),
231+
]),
232+
]
233+
},
234+
},
235+
]
236+
}
237+
if (field.id === 'provider')
238+
return [
239+
{
240+
...field,
241+
options: [
242+
{ id: 'runway', label: 'Runway' },
243+
{ id: 'veo', label: 'Google Gemini' },
244+
{ id: 'luma', label: 'Luma' },
245+
{ id: 'minimax', label: 'MiniMax' },
246+
{ id: 'falai', label: 'Fal.ai (Multi-Model)' },
247+
],
248+
},
249+
...(['runway', 'veo', 'luma', 'minimax'] as const).map(
250+
(provider): SubBlockConfig => ({
251+
id: 'model',
252+
title: 'Model',
253+
type: 'dropdown',
254+
searchable: true,
255+
condition: { field: 'provider', value: provider },
256+
options: directVideoModels
257+
.filter((model) => model.provider === provider)
258+
.flatMap((model) => [
259+
{ id: model.id, label: model.label },
260+
...(model.aliases ?? []).map((id) => ({ id, label: model.label, hidden: true })),
261+
]),
262+
value: () => directVideoDefaults[provider],
263+
dependsOn: ['provider'],
264+
required: true,
265+
})
266+
),
267+
]
268+
if (field.id === 'prompt') return [field, ...directVideoFields]
269+
const condition = field.condition
270+
const isDirect =
271+
typeof condition === 'object' &&
272+
condition.field === 'provider' &&
273+
typeof condition.value === 'string' &&
274+
condition.value !== 'falai'
275+
if (
276+
isDirect &&
277+
['model', 'duration', 'aspectRatio', 'resolution', 'promptOptimizer'].includes(field.id)
278+
)
279+
return []
280+
if (field.id === 'endpoint')
281+
return [
282+
{
283+
...field,
284+
condition: {
285+
field: 'provider',
286+
value: 'minimax',
287+
and: { field: 'model', value: ['hailuo-2.3', 'hailuo-02'] },
288+
},
289+
dependsOn: ['provider', 'model'],
290+
},
291+
]
292+
if (field.id === 'cameraControl')
293+
return [
294+
{
295+
...field,
296+
condition: {
297+
field: 'provider',
298+
value: 'luma',
299+
and: { field: 'model', value: ['ray-2', 'ray-flash-2'] },
300+
},
301+
dependsOn: ['provider', 'model'],
302+
},
303+
]
304+
if (field.id === 'visualReferenceUpload' || field.id === 'visualReferenceInput')
305+
return [
306+
{ ...field, required: { field: 'model', value: ['gen4_turbo', 'gen-4', 'gen-4-turbo'] } },
307+
]
308+
return [field]
309+
})
310+
}
311+
130312
export const VideoGeneratorBlock: BlockConfig = {
131313
type: 'video_generator',
132314
name: 'Video Generator (Legacy)',
@@ -1733,16 +1915,71 @@ export const VideoGeneratorV3Block: BlockConfig = {
17331915
name: 'Video Generator',
17341916
description: 'Generate videos from text using AI',
17351917
longDescription:
1736-
'Generate high-quality videos from text prompts using leading AI providers. Supports Runway, Google Veo, Luma, MiniMax, and Fal.ai multi-model generation with provider-specific durations, aspect ratios, resolutions, prompt optimization, and native audio controls.',
1918+
'Generate high-quality videos from text prompts using leading AI providers. Supports Runway, Google Gemini, Luma, MiniMax, and Fal.ai multi-model generation with provider-specific durations, aspect ratios, resolutions, prompt optimization, and native audio controls.',
17371919
docsLink: 'https://docs.sim.ai/integrations/video_generator',
17381920
category: 'blocks',
17391921
integrationType: IntegrationType.AI,
17401922
bgColor: '#181C1E',
17411923
icon: VideoIcon,
17421924
hideFromToolbar: false,
1743-
subBlocks: withFalVideoCatalog(VideoGeneratorV2Block.subBlocks),
1925+
subBlocks: withDirectVideoCatalog(withFalVideoCatalog(VideoGeneratorV2Block.subBlocks)),
1926+
tools: {
1927+
...VideoGeneratorV2Block.tools,
1928+
config: {
1929+
tool: (params) => VideoGeneratorV2Block.tools.config?.tool(params) ?? 'video_runway',
1930+
params: (params) => {
1931+
if (params.provider === 'falai')
1932+
return VideoGeneratorV2Block.tools.config?.params?.(params) ?? params
1933+
const visualReference =
1934+
params.provider === 'runway'
1935+
? normalizeFileInput(params.visualReference, { single: true })
1936+
: undefined
1937+
const model = getDirectVideoModel(params.provider, params.model, Boolean(visualReference))
1938+
const setting = (key: string, supported: unknown, fallbackKey?: string) => {
1939+
if (!supported) return undefined
1940+
const value = params[key] ?? (fallbackKey ? params[fallbackKey] : undefined)
1941+
return value === FAL_MODEL_DEFAULT.id ? undefined : value
1942+
}
1943+
const duration = setting('duration', model?.settings?.durations)
1944+
return {
1945+
provider: params.provider,
1946+
model: params.model,
1947+
apiKey: params.apiKey,
1948+
prompt: params.prompt,
1949+
visualReference,
1950+
duration: duration ? Number(duration) : undefined,
1951+
aspectRatio: setting('aspectRatio', model?.settings?.aspectRatios),
1952+
resolution: setting('resolution', model?.settings?.resolutions),
1953+
generateAudio: parseOptionalBooleanInput(
1954+
setting('directGenerateAudio', model?.settings?.audio, 'generateAudio')
1955+
),
1956+
promptOptimizer: parseOptionalBooleanInput(
1957+
setting('promptOptimizer', model?.settings?.promptOptimizer)
1958+
),
1959+
endpoint: model?.protocol === 'minimax-v1' ? params.endpoint : undefined,
1960+
cameraControl:
1961+
model?.protocol === 'luma-legacy' && params.cameraControl
1962+
? typeof params.cameraControl === 'string'
1963+
? JSON.parse(params.cameraControl)
1964+
: params.cameraControl
1965+
: undefined,
1966+
}
1967+
},
1968+
},
1969+
},
17441970
inputs: {
17451971
...VideoGeneratorV2Block.inputs,
1972+
duration: { type: 'number', description: 'Duration for the selected provider model' },
1973+
aspectRatio: {
1974+
type: 'string',
1975+
description: 'Frame size or aspect ratio for the selected provider model',
1976+
},
1977+
resolution: { type: 'string', description: 'Resolution for the selected provider model' },
1978+
directGenerateAudio: { type: 'boolean', description: 'Generate audio when supported' },
1979+
promptOptimizer: {
1980+
type: 'boolean',
1981+
description: 'Enable prompt optimization when supported',
1982+
},
17461983
falDuration: {
17471984
type: 'number',
17481985
description: 'Duration in seconds for the selected fal endpoint',

0 commit comments

Comments
 (0)