diff --git a/README.md b/README.md index ed1ebe2..0f49a26 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ A lightweight Node.js module for transcoding videos to web-friendly MP4 format u - Audio enhancement features (normalization, noise reduction, fades) - Thumbnail Generation at specified intervals or timestamps - Batch processing of multiple files with a fancy terminal UI +- Generative media (image, video, speech, music) behind one provider-agnostic interface - No file storage - just passes through to FFmpeg - Lightweight with minimal dependencies @@ -566,6 +567,111 @@ if (skippedFiles.length > 0) { } ``` +### Generating Media + +Alongside transcoding, the module can generate images, video, speech and music through one provider-agnostic interface. Generated assets come back as ordinary files, so the existing ffmpeg pipeline picks them up without any glue code. + +This adds **no runtime dependencies** — every provider is a plain `fetch` call against a documented REST endpoint. + +```javascript +import { generateImage, generateSpeech, transcodeAudio } from '@profullstack/transcoder'; + +// Generate an image +const image = await generateImage({ + prompt: 'A wide editorial photo of an empty recording studio at golden hour', + aspectRatio: '16:9' +}); +await image.toFile('./output/studio.png'); + +// Generate a voiceover, then transcode it with the existing pipeline +const speech = await generateSpeech({ text: 'Here is what changed in this release.' }); +const raw = await speech.toFile('./output/voiceover'); +await transcodeAudio(raw, './output/voiceover.mp3', { preset: 'audio-high' }); +``` + +#### Providers + +| Provider | Capabilities | Credentials | +|----------|--------------|-------------| +| `google` | image, video, speech, music | `GOOGLE_API_KEY` (Lyria also needs `GOOGLE_CLOUD_PROJECT` and `GOOGLE_ACCESS_TOKEN`) | +| `openai` | image, speech | `OPENAI_API_KEY` | +| `elevenlabs` | speech | `ELEVENLABS_API_KEY` | + +A provider is chosen automatically: the first preferred provider for that capability that actually has credentials configured. Speech prefers ElevenLabs, images prefer Google, and both can be overridden per call with `provider`, or globally with the `GENMEDIA_PROVIDER` environment variable. + +```javascript +import { describeProviders } from '@profullstack/transcoder'; + +// Which providers are usable right now? +console.log(describeProviders()); +// [{ name: 'google', capabilities: [...], envVars: [...], configured: false }, ...] +``` + +#### Video with synchronized audio + +Veo generates its own dialogue, effects and ambient audio, which removes the separate voiceover and mux stages from a typical short-video pipeline: + +```javascript +const clip = await generateVideo({ + prompt: 'Slow dolly across a quiet workshop, dust in the light, ambient room tone', + aspectRatio: '16:9', + onProgress: ({ elapsed }) => console.log(`rendering (${Math.round(elapsed / 1000)}s)`) +}); + +console.log(clip.meta.hasNativeAudio); // true +await clip.toFile('./output/clip.mp4'); +``` + +Video generation is a long-running operation and is polled internally until it completes, up to `maxWait` (default 10 minutes). + +#### Multi-speaker speech + +A single call produces a two-host conversation, with no editing step between the parts: + +```javascript +const dialogue = await generateSpeech({ + provider: 'google', + text: 'Host: So what shipped this week?\nGuest: The shared media layer.', + speakers: [ + { speaker: 'Host', voice: 'Kore' }, + { speaker: 'Guest', voice: 'Puck' } + ] +}); +``` + +Google's speech models return headerless PCM; it is wrapped in a WAV container automatically so ffmpeg does not need out-of-band format hints. + +#### Music beds + +```javascript +const bed = await generateMusic({ + prompt: 'An understated, optimistic instrumental bed with light percussion', + seed: 42 +}); +``` + +Generated music sidesteps the licensing problem that otherwise prevents user-facing video products from shipping with a soundtrack at all. + +#### Batch generation + +Providers rate limit, so an unbounded `Promise.all` over a storyboard is the quickest route to `429`s. `generateBatch` bounds concurrency, preserves input order, and captures per-item errors rather than failing the whole run: + +```javascript +const results = await generateBatch( + scenes.map(prompt => ({ kind: 'image', prompt })), + { concurrency: 2, onProgress: ({ completed, total }) => console.log(`${completed}/${total}`) } +); + +for (const { media, error, index } of results) { + if (error) continue; + await media.toFile(`./output/scene-${index + 1}`); +} +``` + +Rate limits and transient upstream faults are retried with exponential backoff and `Retry-After` support; client errors such as a rejected prompt fail immediately. + +See [examples/genmedia.js](examples/genmedia.js) for a runnable walkthrough. + ### Using the CLI Tool The module includes a command-line interface (CLI) for easy video transcoding, thumbnail generation, and watermarking directly from your terminal: @@ -838,6 +944,54 @@ Generates thumbnails from a video file without transcoding. - Promise that resolves with an array of thumbnail paths +### generateImage(options) / generateVideo(options) / generateSpeech(options) / generateMusic(options) + +Generates a media asset through the configured provider. + +**Common options:** + +- `provider` (string): Force a specific provider instead of resolving one +- `model` (string): Override the provider's default model +- `apiKey` (string): Credentials, otherwise read from the environment +- `timeout` (number): Per-request timeout in ms (default: `120000`) +- `retries` (number): Retries after the first attempt (default: `3`) +- `fetchImpl` (Function): Fetch implementation, useful in tests + +**Capability-specific options:** + +- `generateImage`: `prompt`, `aspectRatio`, `size`, `referenceImages` +- `generateVideo`: `prompt`, `aspectRatio`, `resolution`, `negativePrompt`, `image`, `pollInterval`, `maxWait`, `onProgress` +- `generateSpeech`: `text`, `voice`, `speakers`, `instructions`, `format` +- `generateMusic`: `prompt`, `negativePrompt`, `seed`, `projectId`, `location`, `accessToken` + +**Returns:** + +- Promise that resolves with a `GeneratedMedia` instance: `{ data, mimeType, provider, model, kind, meta }`, plus `size`, `extension`, `toFile(path)` and `toDataUri()` + +### generateBatch(requests, [options]) + +Generates several assets concurrently with a bounded number of requests in flight. + +**Parameters:** + +- `requests` (Array): Requests as `{ kind, ...options }` where `kind` is `image`, `video`, `speech` or `music` +- `options.concurrency` (number): Maximum requests in flight (default: `3`) +- `options.onProgress` (Function): Called with `{ index, total, completed, error }` + +**Returns:** + +- Promise that resolves with an array of `{ index, media, error }`, in the order the requests were given + +### describeProviders() + +**Returns:** + +- Array of `{ name, capabilities, defaultModels, envVars, configured }`, one entry per registered provider + +### registerProvider(provider) + +Registers a custom provider. A provider is an object with `name`, `capabilities`, `envVars`, and any of `generateImage`, `generateVideo`, `generateSpeech`, `generateMusic`. + ### BatchProcessEmitter Events The emitter returned by the batchProcessDirectory function emits the following events: diff --git a/examples/genmedia.js b/examples/genmedia.js new file mode 100644 index 0000000..64e9da6 --- /dev/null +++ b/examples/genmedia.js @@ -0,0 +1,173 @@ +/** + * Generative media example for the transcode module + * + * This example demonstrates generating an image, a voiceover, a music bed and a + * video clip through one interface, then handing the results to the existing + * ffmpeg pipeline. Nothing runs without credentials, so the script reports which + * providers are configured first and skips whatever it cannot reach. + */ + +// In a real project, you would import from the package: +// import { generateImage, generateSpeech, transcode } from '@profullstack/transcoder'; +// For this example, we're importing directly from the local file: +import { + generateImage, + generateVideo, + generateSpeech, + generateMusic, + generateBatch, + describeProviders, + hasCredentials +} from '../index.js'; +import fs from 'fs'; + +const outputDir = './test-videos/output/genmedia'; +if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); +} + +// Example 1: See what is actually usable in this environment +console.log('Example 1: Provider capabilities'); +for (const provider of describeProviders()) { + const state = provider.configured ? 'configured' : `needs ${provider.envVars[0]}`; + console.log(` ${provider.name.padEnd(12)} ${provider.capabilities.join(', ').padEnd(28)} ${state}`); +} + +// Example 2: An image, written straight to disk +async function imageExample() { + console.log('\nExample 2: Image generation'); + if (!hasCredentials('google') && !hasCredentials('openai')) { + console.log(' Skipped: no image provider configured'); + return; + } + + const image = await generateImage({ + prompt: 'A wide editorial photo of an empty recording studio at golden hour', + aspectRatio: '16:9' + }); + + const written = await image.toFile(`${outputDir}/studio`); + console.log(` ${image.provider}/${image.model} -> ${written} (${image.size} bytes)`); +} + +// Example 3: A voiceover, then transcode it to a web-friendly format +async function speechExample() { + console.log('\nExample 3: Voiceover'); + if (!hasCredentials('elevenlabs') && !hasCredentials('openai') && !hasCredentials('google')) { + console.log(' Skipped: no speech provider configured'); + return; + } + + const speech = await generateSpeech({ + text: 'Here is what changed in this release, in about forty five seconds.' + }); + + const written = await speech.toFile(`${outputDir}/voiceover`); + console.log(` ${speech.provider}/${speech.model} -> ${written} (${speech.size} bytes)`); + + // The result is a normal audio file, so the existing pipeline takes it from here: + // await transcodeAudio(written, `${outputDir}/voiceover.mp3`, { preset: 'audio-high' }); +} + +// Example 4: Two hosts in a single call, no editing between them +async function podcastExample() { + console.log('\nExample 4: Multi-speaker dialogue'); + if (!hasCredentials('google')) { + console.log(' Skipped: GOOGLE_API_KEY is not set'); + return; + } + + const dialogue = await generateSpeech({ + provider: 'google', + text: 'Host: So what actually shipped this week?\nGuest: The shared media layer, finally.', + speakers: [ + { speaker: 'Host', voice: 'Kore' }, + { speaker: 'Guest', voice: 'Puck' } + ] + }); + + console.log(` -> ${await dialogue.toFile(`${outputDir}/dialogue`)}`); +} + +// Example 5: A music bed. Generated audio sidesteps the licensing problem that +// otherwise stops user-facing video products from shipping with any soundtrack. +async function musicExample() { + console.log('\nExample 5: Music bed'); + if (!process.env.GOOGLE_CLOUD_PROJECT || !process.env.GOOGLE_ACCESS_TOKEN) { + console.log(' Skipped: Lyria needs GOOGLE_CLOUD_PROJECT and GOOGLE_ACCESS_TOKEN'); + return; + } + + const music = await generateMusic({ + prompt: 'An understated, optimistic instrumental bed with light percussion', + seed: 42 + }); + + console.log(` -> ${await music.toFile(`${outputDir}/bed`)}`); +} + +// Example 6: A video clip. Veo returns synchronized audio, so the usual +// generate-voiceover-then-mux stage is unnecessary here. +async function videoExample() { + console.log('\nExample 6: Video generation'); + if (!hasCredentials('google')) { + console.log(' Skipped: GOOGLE_API_KEY is not set'); + return; + } + + const clip = await generateVideo({ + prompt: 'Slow dolly across a quiet workshop, dust in the light, ambient room tone', + aspectRatio: '16:9', + onProgress: ({ elapsed }) => console.log(` still rendering (${Math.round(elapsed / 1000)}s)`) + }); + + const written = await clip.toFile(`${outputDir}/clip.mp4`); + console.log(` ${clip.model} -> ${written} (${clip.size} bytes, native audio: ${clip.meta.hasNativeAudio})`); +} + +// Example 7: A storyboard, generated concurrently but politely +async function batchExample() { + console.log('\nExample 7: Batch storyboard'); + if (!hasCredentials('google') && !hasCredentials('openai')) { + console.log(' Skipped: no image provider configured'); + return; + } + + const scenes = [ + 'Scene 1: a closed laptop on a workbench, morning light', + 'Scene 2: the same workbench, tools laid out in a row', + 'Scene 3: a wide shot of the finished piece' + ]; + + const results = await generateBatch( + scenes.map(prompt => ({ kind: 'image', prompt })), + { + concurrency: 2, + onProgress: ({ completed, total }) => console.log(` ${completed}/${total}`) + } + ); + + for (const result of results) { + if (result.error) { + console.log(` scene ${result.index + 1} failed: ${result.error.message}`); + continue; + } + console.log(` scene ${result.index + 1} -> ${await result.media.toFile(`${outputDir}/scene-${result.index + 1}`)}`); + } +} + +async function main() { + const examples = [imageExample, speechExample, podcastExample, musicExample, videoExample, batchExample]; + + for (const example of examples) { + try { + await example(); + } catch (error) { + console.error(` Error: ${error.message}`); + } + } + + console.log('\nDone.'); +} + +main(); diff --git a/package.json b/package.json index 4c0972b..47e00ec 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:image": "mocha test/image.test.js", "test:batch": "mocha test/batch.test.js", "test:terminal-ui": "mocha test/terminal-ui.test.js", + "test:genmedia": "mocha test/genmedia.test.js", "generate-test-video": "node scripts/generate-test-video.js", "generate-test-audio": "node scripts/generate-test-audio.js", "example": "node examples/basic-usage.js", @@ -41,6 +42,7 @@ "example:square": "node examples/square-padding.js", "example:batch": "node examples/batch-processing.js", "example:audio-enhancement": "node examples/audio-enhancement.js", + "example:genmedia": "node examples/genmedia.js", "example:cli": "./examples/example.sh", "install-ffmpeg": "./bin/build-ffmpeg.sh", "install-imagemagick": "./bin/install-imagemagick.sh", diff --git a/src/genmedia/errors.js b/src/genmedia/errors.js new file mode 100644 index 0000000..910a9a5 --- /dev/null +++ b/src/genmedia/errors.js @@ -0,0 +1,74 @@ +/** + * @profullstack/transcoder - Generative Media Errors + * Error types shared by every generative media provider + */ + +/** + * Base error for all generative media failures + */ +export class GenMediaError extends Error { + /** + * @param {string} message - Human readable description of the failure + * @param {Object} [details={}] - Additional context (provider, model, status) + */ + constructor(message, details = {}) { + super(message); + this.name = 'GenMediaError'; + this.provider = details.provider ?? null; + this.model = details.model ?? null; + this.details = details; + } +} + +/** + * Thrown when no credentials are available for a provider + */ +export class MissingCredentialsError extends GenMediaError { + constructor(provider, envVar) { + super( + `No credentials for provider "${provider}". Set ${envVar} or pass apiKey in options.`, + { provider, envVar } + ); + this.name = 'MissingCredentialsError'; + this.envVar = envVar; + } +} + +/** + * Thrown when a provider cannot service the requested capability + */ +export class UnsupportedCapabilityError extends GenMediaError { + constructor(provider, capability) { + super(`Provider "${provider}" cannot generate ${capability}.`, { provider, capability }); + this.name = 'UnsupportedCapabilityError'; + this.capability = capability; + } +} + +/** + * Thrown when a provider's HTTP API returns a non-success status + */ +export class ProviderResponseError extends GenMediaError { + /** + * @param {string} provider - Provider name + * @param {number} status - HTTP status code + * @param {string} body - Response body, truncated for readability + */ + constructor(provider, status, body) { + super(`Provider "${provider}" returned HTTP ${status}: ${body}`, { provider, status, body }); + this.name = 'ProviderResponseError'; + this.status = status; + this.body = body; + } +} + +/** + * Thrown when a long-running generation exceeds its poll budget + */ +export class GenerationTimeoutError extends GenMediaError { + constructor(provider, elapsedMs) { + super(`Provider "${provider}" did not finish within ${elapsedMs}ms.`, { provider, elapsedMs }); + this.name = 'GenerationTimeoutError'; + this.elapsedMs = elapsedMs; + } +} diff --git a/src/genmedia/http.js b/src/genmedia/http.js new file mode 100644 index 0000000..4bf8eff --- /dev/null +++ b/src/genmedia/http.js @@ -0,0 +1,148 @@ +/** + * @profullstack/transcoder - Generative Media HTTP + * A small fetch wrapper with timeouts, retries and consistent error reporting. + * Uses the global fetch shipped with Node 20+, so the module adds no dependencies. + */ + +import { ProviderResponseError } from './errors.js'; + +/** Status codes that are worth retrying: rate limits and transient upstream faults */ +const RETRYABLE_STATUS = new Set([408, 409, 425, 429, 500, 502, 503, 504]); + +export const DEFAULT_HTTP_OPTIONS = { + timeout: 120000, + retries: 3, + retryBaseDelay: 500, + maxRetryDelay: 8000 +}; + +/** + * Sleeps for a number of milliseconds + * + * @param {number} ms - Duration to wait + * @returns {Promise} + */ +export function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Computes an exponential backoff delay with full jitter + * + * @param {number} attempt - Zero-based attempt number + * @param {Object} settings - Merged HTTP settings + * @param {number} [retryAfterSeconds] - Value of a Retry-After header, when present + * @returns {number} - Delay in milliseconds + */ +export function backoffDelay(attempt, settings, retryAfterSeconds) { + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { + return Math.min(retryAfterSeconds * 1000, settings.maxRetryDelay); + } + const ceiling = Math.min(settings.retryBaseDelay * 2 ** attempt, settings.maxRetryDelay); + return Math.round(Math.random() * ceiling); +} + +/** + * Reads a Retry-After header as a number of seconds + * + * @param {Response} response - Fetch response + * @returns {number|undefined} - Seconds to wait, or undefined when absent + */ +function parseRetryAfter(response) { + const raw = response.headers?.get?.('retry-after'); + if (!raw) return undefined; + const seconds = Number(raw); + return Number.isFinite(seconds) ? seconds : undefined; +} + +/** + * Performs an HTTP request with timeout and retry handling + * + * @param {string} url - Request URL + * @param {Object} [init={}] - Fetch init options + * @param {Object} [options={}] - Behaviour options + * @param {string} [options.provider='unknown'] - Provider name used in error messages + * @param {Function} [options.fetchImpl] - Fetch implementation, injectable for tests + * @param {number} [options.timeout] - Per-attempt timeout in milliseconds + * @param {number} [options.retries] - Number of retries after the first attempt + * @returns {Promise} - Resolves with a successful response + */ +export async function request(url, init = {}, options = {}) { + const settings = { ...DEFAULT_HTTP_OPTIONS, ...options }; + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + const provider = options.provider ?? 'unknown'; + + if (typeof fetchImpl !== 'function') { + throw new TypeError('No fetch implementation available. Node 20+ or a fetchImpl option is required.'); + } + + let lastError = null; + + for (let attempt = 0; attempt <= settings.retries; attempt++) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), settings.timeout); + let response = null; + let networkError = null; + + try { + response = await fetchImpl(url, { ...init, signal: controller.signal }); + } catch (error) { + networkError = error; + } finally { + clearTimeout(timer); + } + + // Network faults and timeouts stay retryable until the budget runs out + if (networkError) { + lastError = networkError; + if (attempt === settings.retries) throw networkError; + await delay(backoffDelay(attempt, settings)); + continue; + } + + if (response.ok) { + return response; + } + + // Read the body once so the error message is useful, then decide on a retry + const body = await response.text().catch(() => ''); + const truncated = body.length > 500 ? `${body.slice(0, 500)}...` : body; + lastError = new ProviderResponseError(provider, response.status, truncated); + + // A 400 or a 401 will fail identically on every attempt, so fail fast + if (!RETRYABLE_STATUS.has(response.status) || attempt === settings.retries) { + throw lastError; + } + + await delay(backoffDelay(attempt, settings, parseRetryAfter(response))); + } + + throw lastError ?? new ProviderResponseError(provider, 0, 'Request failed with no response'); +} + +/** + * Performs a request and parses the response as JSON + * + * @param {string} url - Request URL + * @param {Object} [init={}] - Fetch init options + * @param {Object} [options={}] - Behaviour options + * @returns {Promise} - Parsed JSON body + */ +export async function requestJson(url, init = {}, options = {}) { + const response = await request(url, init, options); + return response.json(); +} + +/** + * Performs a request and returns the response body as a Buffer + * + * @param {string} url - Request URL + * @param {Object} [init={}] - Fetch init options + * @param {Object} [options={}] - Behaviour options + * @returns {Promise} - Response body + */ +export async function requestBuffer(url, init = {}, options = {}) { + const response = await request(url, init, options); + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); +} diff --git a/src/genmedia/index.js b/src/genmedia/index.js new file mode 100644 index 0000000..b252426 --- /dev/null +++ b/src/genmedia/index.js @@ -0,0 +1,156 @@ +/** + * @profullstack/transcoder - Generative Media + * + * One interface for generating images, video, speech and music, sitting next to + * the ffmpeg presets so a generated asset can be transcoded without leaving the + * module. Providers are swappable per call; nothing here is Google specific. + * + * @example + * import { generateVideo, transcode } from '@profullstack/transcoder'; + * + * const clip = await generateVideo({ prompt: 'a slow dolly across an empty studio' }); + * const raw = await clip.toFile('./out/clip.mp4'); + * await transcode(raw, './out/clip.web.mp4', { preset: 'web' }); + */ + +import { invoke, resolveProvider, describeProviders, hasCredentials } from './registry.js'; + +/** + * Generates an image + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Text prompt + * @param {string} [options.provider] - Force a specific provider + * @param {string} [options.model] - Model override + * @returns {Promise} - The generated image + */ +export function generateImage(options = {}) { + return invoke('image', options); +} + +/** + * Generates a video. + * + * With Veo the result already contains synchronized audio, so the usual + * generate-voiceover-then-mux stage can be skipped entirely. + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Text prompt + * @param {string} [options.provider] - Force a specific provider + * @returns {Promise} - The generated video + */ +export function generateVideo(options = {}) { + return invoke('video', options); +} + +/** + * Generates spoken audio + * + * @param {Object} options - Generation options + * @param {string} options.text - Text to speak + * @param {string} [options.voice] - Provider specific voice identifier + * @param {string} [options.provider] - Force a specific provider + * @returns {Promise} - The generated speech + */ +export function generateSpeech(options = {}) { + return invoke('speech', options); +} + +/** + * Generates music. + * + * Generated beds sidestep the licensing problem that blocks most user-facing + * video products from shipping with a soundtrack at all. + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Description of the music + * @param {string} [options.provider] - Force a specific provider + * @returns {Promise} - The generated music + */ +export function generateMusic(options = {}) { + return invoke('music', options); +} + +/** + * Generates several assets concurrently, with a bounded number in flight. + * + * Every provider rate limits, so an unbounded Promise.all over a storyboard is + * the fastest way to get 429s. This keeps the concurrency explicit and returns + * results in the order the requests were given. + * + * @param {Array} requests - Requests as {kind, ...options} + * @param {Object} [options={}] - Batch options + * @param {number} [options.concurrency=3] - Maximum requests in flight + * @param {Function} [options.onProgress] - Called with {index, total, error} + * @returns {Promise>} - Results as {index, media, error} + */ +export async function generateBatch(requests, options = {}) { + const concurrency = Math.max(1, options.concurrency ?? 3); + const results = new Array(requests.length); + let cursor = 0; + let completed = 0; + + const generators = { + image: generateImage, + video: generateVideo, + speech: generateSpeech, + music: generateMusic + }; + + async function worker() { + for (;;) { + const index = cursor++; + if (index >= requests.length) return; + + const { kind, ...rest } = requests[index]; + const generate = generators[kind]; + + if (!generate) { + results[index] = { index, media: null, error: new Error(`Unknown kind "${kind}"`) }; + } else { + try { + results[index] = { index, media: await generate(rest), error: null }; + } catch (error) { + results[index] = { index, media: null, error }; + } + } + + completed++; + if (typeof options.onProgress === 'function') { + options.onProgress({ + index, + total: requests.length, + completed, + error: results[index].error + }); + } + } + } + + await Promise.all( + Array.from({ length: Math.min(concurrency, requests.length) }, () => worker()) + ); + + return results; +} + +export { resolveProvider, describeProviders, hasCredentials }; +export { + registerProvider, + getProvider, + listProviders, + supports, + CAPABILITIES, + PROVIDER_PREFERENCE +} from './registry.js'; +export { GeneratedMedia, pcmToWav, parsePcmMimeType, extensionForMimeType } from './media.js'; +export { + GenMediaError, + MissingCredentialsError, + UnsupportedCapabilityError, + ProviderResponseError, + GenerationTimeoutError +} from './errors.js'; +export { googleProvider } from './providers/google.js'; +export { openaiProvider } from './providers/openai.js'; +export { elevenlabsProvider } from './providers/elevenlabs.js'; diff --git a/src/genmedia/media.js b/src/genmedia/media.js new file mode 100644 index 0000000..1ac136d --- /dev/null +++ b/src/genmedia/media.js @@ -0,0 +1,167 @@ +/** + * @profullstack/transcoder - Generated Media + * The single result shape every provider returns, plus the helpers needed to get + * that result onto disk and into the existing ffmpeg pipeline. + */ + +import fs from 'fs'; +import path from 'path'; +import { GenMediaError } from './errors.js'; + +/** File extensions for the mime types the providers actually return */ +const EXTENSION_BY_MIME = { + 'image/png': '.png', + 'image/jpeg': '.jpg', + 'image/webp': '.webp', + 'video/mp4': '.mp4', + 'video/webm': '.webm', + 'audio/mpeg': '.mp3', + 'audio/mp3': '.mp3', + 'audio/wav': '.wav', + 'audio/x-wav': '.wav', + 'audio/wave': '.wav', + 'audio/opus': '.opus', + 'audio/ogg': '.ogg', + 'audio/aac': '.aac', + 'audio/flac': '.flac' +}; + +/** + * Maps a mime type to a file extension + * + * @param {string} mimeType - Mime type, optionally with parameters + * @param {string} [fallback='.bin'] - Extension to use when the type is unknown + * @returns {string} - File extension including the leading dot + */ +export function extensionForMimeType(mimeType, fallback = '.bin') { + if (!mimeType) return fallback; + const base = String(mimeType).split(';')[0].trim().toLowerCase(); + return EXTENSION_BY_MIME[base] ?? fallback; +} + +/** + * Wraps raw little-endian PCM samples in a RIFF/WAVE container. + * + * Google's speech and music models return headerless PCM. ffmpeg will happily + * read that, but only if it is told the sample rate and channel count out of + * band, so it is far less error-prone to add the 44-byte header here. + * + * @param {Buffer} pcm - Raw PCM sample data + * @param {Object} [options={}] - PCM description + * @param {number} [options.sampleRate=24000] - Samples per second + * @param {number} [options.channels=1] - Channel count + * @param {number} [options.bitsPerSample=16] - Bit depth + * @returns {Buffer} - A complete WAV file + */ +export function pcmToWav(pcm, options = {}) { + const sampleRate = options.sampleRate ?? 24000; + const channels = options.channels ?? 1; + const bitsPerSample = options.bitsPerSample ?? 16; + + const blockAlign = (channels * bitsPerSample) / 8; + const byteRate = sampleRate * blockAlign; + const header = Buffer.alloc(44); + + header.write('RIFF', 0); + header.writeUInt32LE(36 + pcm.length, 4); + header.write('WAVE', 8); + header.write('fmt ', 12); + header.writeUInt32LE(16, 16); // PCM subchunk size + header.writeUInt16LE(1, 20); // Audio format 1 = PCM + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(byteRate, 28); + header.writeUInt16LE(blockAlign, 32); + header.writeUInt16LE(bitsPerSample, 34); + header.write('data', 36); + header.writeUInt32LE(pcm.length, 40); + + return Buffer.concat([header, pcm]); +} + +/** + * Parses the `audio/L16;rate=24000` style mime types Google returns for PCM + * + * @param {string} mimeType - Mime type to inspect + * @returns {Object|null} - PCM parameters, or null when the type is not raw PCM + */ +export function parsePcmMimeType(mimeType) { + if (!mimeType) return null; + const [base, ...params] = String(mimeType).split(';').map(part => part.trim()); + if (!/^audio\/l\d+$/i.test(base)) return null; + + const bitsPerSample = Number(base.slice('audio/l'.length)) || 16; + const rateParam = params.find(param => param.toLowerCase().startsWith('rate=')); + const sampleRate = rateParam ? Number(rateParam.split('=')[1]) : 24000; + const channelsParam = params.find(param => param.toLowerCase().startsWith('channels=')); + const channels = channelsParam ? Number(channelsParam.split('=')[1]) : 1; + + return { + bitsPerSample, + sampleRate: Number.isFinite(sampleRate) ? sampleRate : 24000, + channels: Number.isFinite(channels) ? channels : 1 + }; +} + +/** + * A generated asset, returned by every generate* function. + */ +export class GeneratedMedia { + /** + * @param {Object} params - Asset description + * @param {Buffer} params.data - Raw asset bytes + * @param {string} params.mimeType - Mime type of the asset + * @param {string} params.provider - Provider that produced it + * @param {string} params.model - Model that produced it + * @param {string} params.kind - One of image, video, speech, music + * @param {Object} [params.meta={}] - Provider specific extras + */ + constructor({ data, mimeType, provider, model, kind, meta = {} }) { + if (!Buffer.isBuffer(data)) { + throw new GenMediaError('GeneratedMedia requires a Buffer', { provider, model }); + } + this.data = data; + this.mimeType = mimeType; + this.provider = provider; + this.model = model; + this.kind = kind; + this.meta = meta; + } + + /** @returns {string} - The conventional file extension for this asset */ + get extension() { + return extensionForMimeType(this.mimeType); + } + + /** @returns {number} - Size of the asset in bytes */ + get size() { + return this.data.length; + } + + /** + * Writes the asset to disk, creating parent directories as needed. + * + * When the path has no extension, the one implied by the mime type is added, + * which keeps ffmpeg's format detection working downstream. + * + * @param {string} outputPath - Destination path + * @returns {Promise} - The path actually written + */ + async toFile(outputPath) { + const hasExtension = path.extname(outputPath) !== ''; + const target = hasExtension ? outputPath : `${outputPath}${this.extension}`; + + await fs.promises.mkdir(path.dirname(path.resolve(target)), { recursive: true }); + await fs.promises.writeFile(target, this.data); + + return target; + } + + /** + * @param {string} [prefix='data'] - Ignored, present for symmetry with toFile + * @returns {string} - The asset as a data URI + */ + toDataUri() { + return `data:${this.mimeType};base64,${this.data.toString('base64')}`; + } +} diff --git a/src/genmedia/providers/elevenlabs.js b/src/genmedia/providers/elevenlabs.js new file mode 100644 index 0000000..62f3dd6 --- /dev/null +++ b/src/genmedia/providers/elevenlabs.js @@ -0,0 +1,86 @@ +/** + * @profullstack/transcoder - ElevenLabs Generative Media Provider + * Speech only. Present because it is the incumbent voice provider across the + * existing pipelines and switching cost should be zero in either direction. + */ + +import { requestBuffer } from '../http.js'; +import { GeneratedMedia } from '../media.js'; +import { GenMediaError, MissingCredentialsError } from '../errors.js'; + +const API_BASE = 'https://api.elevenlabs.io/v1'; + +/** Rachel, ElevenLabs' long-standing default voice */ +const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; + +export const ELEVENLABS_DEFAULT_MODELS = { + speech: 'eleven_turbo_v2_5' +}; + +export const ELEVENLABS_CAPABILITIES = ['speech']; + +/** + * Resolves the API key from options or the environment + * + * @param {Object} options - Call options + * @returns {string} - The API key + */ +function resolveApiKey(options) { + const key = options.apiKey ?? process.env.ELEVENLABS_API_KEY; + if (!key) { + throw new MissingCredentialsError('elevenlabs', 'ELEVENLABS_API_KEY'); + } + return key; +} + +/** + * Generates speech with ElevenLabs text-to-speech + * + * @param {Object} options - Generation options + * @param {string} options.text - Text to speak + * @param {string} [options.voice] - Voice id + * @param {string} [options.model] - Model override + * @param {Object} [options.voiceSettings] - Stability and similarity settings + * @param {string} [options.outputFormat='mp3_44100_128'] - ElevenLabs output format + * @returns {Promise} - The generated speech + */ +export async function generateSpeech(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? ELEVENLABS_DEFAULT_MODELS.speech; + const voiceId = options.voice ?? process.env.ELEVENLABS_VOICE_ID ?? DEFAULT_VOICE_ID; + + if (!options.text) { + throw new GenMediaError('generateSpeech requires text', { provider: 'elevenlabs', model }); + } + + const outputFormat = options.outputFormat ?? 'mp3_44100_128'; + const body = { text: options.text, model_id: model }; + if (options.voiceSettings) body.voice_settings = options.voiceSettings; + + const data = await requestBuffer( + `${API_BASE}/text-to-speech/${voiceId}?output_format=${encodeURIComponent(outputFormat)}`, + { + method: 'POST', + headers: { 'content-type': 'application/json', 'xi-api-key': apiKey }, + body: JSON.stringify(body) + }, + { ...options, provider: 'elevenlabs' } + ); + + return new GeneratedMedia({ + data, + mimeType: outputFormat.startsWith('mp3') ? 'audio/mpeg' : 'audio/wav', + provider: 'elevenlabs', + model, + kind: 'speech', + meta: { voice: voiceId, outputFormat } + }); +} + +export const elevenlabsProvider = { + name: 'elevenlabs', + capabilities: ELEVENLABS_CAPABILITIES, + defaultModels: ELEVENLABS_DEFAULT_MODELS, + envVars: ['ELEVENLABS_API_KEY'], + generateSpeech +}; diff --git a/src/genmedia/providers/google.js b/src/genmedia/providers/google.js new file mode 100644 index 0000000..c960032 --- /dev/null +++ b/src/genmedia/providers/google.js @@ -0,0 +1,426 @@ +/** + * @profullstack/transcoder - Google Generative Media Provider + * Gemini Image, Veo, Gemini speech and Lyria, over the Generative Language and + * Vertex REST APIs. No SDK: every call is plain fetch against a documented endpoint. + */ + +import { requestJson, requestBuffer, delay } from '../http.js'; +import { GeneratedMedia, pcmToWav, parsePcmMimeType } from '../media.js'; +import { + GenMediaError, + MissingCredentialsError, + GenerationTimeoutError +} from '../errors.js'; + +const API_BASE = 'https://generativelanguage.googleapis.com/v1beta'; + +/** + * Default models. Google renames these often, so every generate call accepts a + * `model` option and these are only the starting point. + */ +export const GOOGLE_DEFAULT_MODELS = { + image: 'gemini-2.5-flash-image', + video: 'veo-3.1-generate-preview', + speech: 'gemini-2.5-flash-preview-tts', + music: 'lyria-002' +}; + +export const GOOGLE_CAPABILITIES = ['image', 'video', 'speech', 'music']; + +/** + * Resolves the API key from options or the environment + * + * @param {Object} options - Call options + * @returns {string} - The API key + */ +function resolveApiKey(options) { + const key = + options.apiKey ?? + process.env.GOOGLE_API_KEY ?? + process.env.GEMINI_API_KEY ?? + process.env.GOOGLE_GENAI_API_KEY; + + if (!key) { + throw new MissingCredentialsError('google', 'GOOGLE_API_KEY'); + } + return key; +} + +/** + * Builds the headers every Generative Language call needs + * + * @param {string} apiKey - API key + * @returns {Object} - Header map + */ +function jsonHeaders(apiKey) { + return { + 'content-type': 'application/json', + 'x-goog-api-key': apiKey + }; +} + +/** + * Pulls the first inline binary part out of a generateContent response + * + * @param {Object} payload - Parsed generateContent response + * @param {string} model - Model name, for error messages + * @returns {Object} - The inlineData object with mimeType and base64 data + */ +function firstInlinePart(payload, model) { + const parts = payload?.candidates?.[0]?.content?.parts ?? []; + const inline = parts.find(part => part.inlineData?.data); + + if (!inline) { + const blockReason = payload?.promptFeedback?.blockReason; + const finishReason = payload?.candidates?.[0]?.finishReason; + const detail = blockReason + ? `request was blocked (${blockReason})` + : `no inline media in response (finishReason: ${finishReason ?? 'unknown'})`; + throw new GenMediaError(`Google returned no media for ${model}: ${detail}`, { + provider: 'google', + model, + blockReason, + finishReason + }); + } + + return inline.inlineData; +} + +/** + * Generates an image with Gemini Image + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Text prompt + * @param {string} [options.model] - Model override + * @param {Array} [options.referenceImages] - Reference images as {data, mimeType} + * for character or product consistency across a set of generations + * @param {string} [options.aspectRatio] - Aspect ratio such as '16:9' + * @returns {Promise} - The generated image + */ +export async function generateImage(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? GOOGLE_DEFAULT_MODELS.image; + + if (!options.prompt) { + throw new GenMediaError('generateImage requires a prompt', { provider: 'google', model }); + } + + // Reference images ride along in the same parts array as the prompt + const parts = [{ text: options.prompt }]; + for (const reference of options.referenceImages ?? []) { + parts.push({ + inlineData: { + mimeType: reference.mimeType ?? 'image/png', + data: Buffer.isBuffer(reference.data) + ? reference.data.toString('base64') + : reference.data + } + }); + } + + const generationConfig = { responseModalities: ['IMAGE'] }; + if (options.aspectRatio) { + generationConfig.imageConfig = { aspectRatio: options.aspectRatio }; + } + + const payload = await requestJson( + `${API_BASE}/models/${model}:generateContent`, + { + method: 'POST', + headers: jsonHeaders(apiKey), + body: JSON.stringify({ contents: [{ parts }], generationConfig }) + }, + { ...options, provider: 'google' } + ); + + const inline = firstInlinePart(payload, model); + + return new GeneratedMedia({ + data: Buffer.from(inline.data, 'base64'), + mimeType: inline.mimeType ?? 'image/png', + provider: 'google', + model, + kind: 'image', + meta: { aspectRatio: options.aspectRatio ?? null } + }); +} + +/** + * Generates a video with Veo. + * + * Veo is a long-running operation: the first call returns an operation name that + * has to be polled. Veo also generates its own synchronized audio, which is why + * callers can skip the separate voiceover and mux stages entirely. + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Text prompt + * @param {string} [options.model] - Model override + * @param {Object} [options.image] - Optional first-frame image as {data, mimeType} + * @param {string} [options.negativePrompt] - What to avoid + * @param {string} [options.aspectRatio='16:9'] - Aspect ratio + * @param {string} [options.resolution] - Resolution hint such as '1080p' + * @param {number} [options.pollInterval=10000] - Milliseconds between poll attempts + * @param {number} [options.maxWait=600000] - Total milliseconds to wait before giving up + * @returns {Promise} - The generated video + */ +export async function generateVideo(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? GOOGLE_DEFAULT_MODELS.video; + + if (!options.prompt) { + throw new GenMediaError('generateVideo requires a prompt', { provider: 'google', model }); + } + + const instance = { prompt: options.prompt }; + if (options.image) { + instance.image = { + bytesBase64Encoded: Buffer.isBuffer(options.image.data) + ? options.image.data.toString('base64') + : options.image.data, + mimeType: options.image.mimeType ?? 'image/png' + }; + } + + const parameters = { aspectRatio: options.aspectRatio ?? '16:9' }; + if (options.negativePrompt) parameters.negativePrompt = options.negativePrompt; + if (options.resolution) parameters.resolution = options.resolution; + + const started = await requestJson( + `${API_BASE}/models/${model}:predictLongRunning`, + { + method: 'POST', + headers: jsonHeaders(apiKey), + body: JSON.stringify({ instances: [instance], parameters }) + }, + { ...options, provider: 'google' } + ); + + if (!started?.name) { + throw new GenMediaError('Veo did not return an operation name', { provider: 'google', model }); + } + + const operation = await pollOperation(started.name, apiKey, { ...options, model }); + const sample = + operation?.response?.generateVideoResponse?.generatedSamples?.[0] ?? + operation?.response?.generatedSamples?.[0]; + + if (!sample) { + const filteredReason = operation?.response?.raiMediaFilteredReasons?.[0]; + throw new GenMediaError( + `Veo returned no video${filteredReason ? `: ${filteredReason}` : ''}`, + { provider: 'google', model, operation: operation?.name } + ); + } + + // The sample is either an inline base64 blob or a URI that needs the key to fetch + const data = sample.video?.bytesBase64Encoded + ? Buffer.from(sample.video.bytesBase64Encoded, 'base64') + : await requestBuffer( + sample.video.uri, + { headers: { 'x-goog-api-key': apiKey } }, + { ...options, provider: 'google' } + ); + + return new GeneratedMedia({ + data, + mimeType: sample.video?.mimeType ?? 'video/mp4', + provider: 'google', + model, + kind: 'video', + meta: { + operation: operation?.name ?? null, + aspectRatio: parameters.aspectRatio, + hasNativeAudio: true + } + }); +} + +/** + * Polls a long-running operation until it reports completion + * + * @param {string} operationName - Fully qualified operation name + * @param {string} apiKey - API key + * @param {Object} options - Poll options + * @returns {Promise} - The completed operation + */ +export async function pollOperation(operationName, apiKey, options = {}) { + const pollInterval = options.pollInterval ?? 10000; + const maxWait = options.maxWait ?? 600000; + const startedAt = Date.now(); + + for (;;) { + const operation = await requestJson( + `${API_BASE}/${operationName}`, + { headers: { 'x-goog-api-key': apiKey } }, + { ...options, provider: 'google' } + ); + + if (operation?.done) { + if (operation.error) { + throw new GenMediaError( + `Veo operation failed: ${operation.error.message ?? 'unknown error'}`, + { provider: 'google', model: options.model, code: operation.error.code } + ); + } + return operation; + } + + const elapsed = Date.now() - startedAt; + if (elapsed >= maxWait) { + throw new GenerationTimeoutError('google', elapsed); + } + + if (typeof options.onProgress === 'function') { + options.onProgress({ elapsed, operation: operationName }); + } + + await delay(pollInterval); + } +} + +/** + * Generates speech with Gemini text-to-speech. + * + * The model returns headerless PCM, which is wrapped in a WAV container so the + * result drops straight into transcodeAudio without extra format hints. + * + * @param {Object} options - Generation options + * @param {string} options.text - Text to speak, may include style directions + * @param {string} [options.voice='Kore'] - Prebuilt voice name + * @param {Array} [options.speakers] - Multi-speaker config as {speaker, voice} + * @param {string} [options.model] - Model override + * @returns {Promise} - The generated speech + */ +export async function generateSpeech(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? GOOGLE_DEFAULT_MODELS.speech; + + if (!options.text) { + throw new GenMediaError('generateSpeech requires text', { provider: 'google', model }); + } + + // Multi-speaker turns a monologue into a two-host conversation in one call + const speechConfig = Array.isArray(options.speakers) && options.speakers.length > 0 + ? { + multiSpeakerVoiceConfig: { + speakerVoiceConfigs: options.speakers.map(entry => ({ + speaker: entry.speaker, + voiceConfig: { prebuiltVoiceConfig: { voiceName: entry.voice } } + })) + } + } + : { + voiceConfig: { prebuiltVoiceConfig: { voiceName: options.voice ?? 'Kore' } } + }; + + const payload = await requestJson( + `${API_BASE}/models/${model}:generateContent`, + { + method: 'POST', + headers: jsonHeaders(apiKey), + body: JSON.stringify({ + contents: [{ parts: [{ text: options.text }] }], + generationConfig: { responseModalities: ['AUDIO'], speechConfig } + }) + }, + { ...options, provider: 'google' } + ); + + const inline = firstInlinePart(payload, model); + const raw = Buffer.from(inline.data, 'base64'); + const pcm = parsePcmMimeType(inline.mimeType); + + return new GeneratedMedia({ + data: pcm ? pcmToWav(raw, pcm) : raw, + mimeType: pcm ? 'audio/wav' : inline.mimeType ?? 'audio/wav', + provider: 'google', + model, + kind: 'speech', + meta: { + voice: options.voice ?? 'Kore', + multiSpeaker: Array.isArray(options.speakers) && options.speakers.length > 0, + sourceMimeType: inline.mimeType ?? null + } + }); +} + +/** + * Generates music with Lyria. + * + * Lyria is served from Vertex AI rather than the Generative Language API, so it + * needs a project id and an OAuth access token instead of an API key. The usual + * way to get one on a workstation is `gcloud auth print-access-token`. + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Description of the music + * @param {string} [options.negativePrompt] - What to avoid + * @param {number} [options.seed] - Deterministic seed + * @param {string} [options.projectId] - GCP project, defaults to GOOGLE_CLOUD_PROJECT + * @param {string} [options.location='us-central1'] - Vertex region + * @param {string} [options.accessToken] - OAuth token, defaults to GOOGLE_ACCESS_TOKEN + * @param {string} [options.model] - Model override + * @returns {Promise} - The generated music + */ +export async function generateMusic(options = {}) { + const model = options.model ?? GOOGLE_DEFAULT_MODELS.music; + const projectId = options.projectId ?? process.env.GOOGLE_CLOUD_PROJECT; + const accessToken = options.accessToken ?? process.env.GOOGLE_ACCESS_TOKEN; + const location = options.location ?? process.env.GOOGLE_CLOUD_LOCATION ?? 'us-central1'; + + if (!options.prompt) { + throw new GenMediaError('generateMusic requires a prompt', { provider: 'google', model }); + } + if (!projectId) { + throw new MissingCredentialsError('google', 'GOOGLE_CLOUD_PROJECT'); + } + if (!accessToken) { + throw new MissingCredentialsError('google', 'GOOGLE_ACCESS_TOKEN'); + } + + const instance = { prompt: options.prompt }; + if (options.negativePrompt) instance.negative_prompt = options.negativePrompt; + if (Number.isFinite(options.seed)) instance.seed = options.seed; + + const url = + `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}` + + `/locations/${location}/publishers/google/models/${model}:predict`; + + const payload = await requestJson( + url, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${accessToken}` + }, + body: JSON.stringify({ instances: [instance], parameters: {} }) + }, + { ...options, provider: 'google' } + ); + + const prediction = payload?.predictions?.[0]; + if (!prediction?.bytesBase64Encoded) { + throw new GenMediaError('Lyria returned no audio', { provider: 'google', model }); + } + + return new GeneratedMedia({ + data: Buffer.from(prediction.bytesBase64Encoded, 'base64'), + mimeType: prediction.mimeType ?? 'audio/wav', + provider: 'google', + model, + kind: 'music', + meta: { seed: options.seed ?? null, location } + }); +} + +export const googleProvider = { + name: 'google', + capabilities: GOOGLE_CAPABILITIES, + defaultModels: GOOGLE_DEFAULT_MODELS, + envVars: ['GOOGLE_API_KEY', 'GEMINI_API_KEY', 'GOOGLE_GENAI_API_KEY'], + generateImage, + generateVideo, + generateSpeech, + generateMusic +}; diff --git a/src/genmedia/providers/openai.js b/src/genmedia/providers/openai.js new file mode 100644 index 0000000..f7dcdbc --- /dev/null +++ b/src/genmedia/providers/openai.js @@ -0,0 +1,151 @@ +/** + * @profullstack/transcoder - OpenAI Generative Media Provider + * Images and speech over the OpenAI REST API. This is the provider most of the + * existing pipelines already use, kept here so nothing has to be rewritten to + * adopt the shared layer. + */ + +import { requestJson, requestBuffer } from '../http.js'; +import { GeneratedMedia } from '../media.js'; +import { GenMediaError, MissingCredentialsError } from '../errors.js'; + +const API_BASE = 'https://api.openai.com/v1'; + +export const OPENAI_DEFAULT_MODELS = { + image: 'gpt-image-1', + speech: 'gpt-4o-mini-tts' +}; + +export const OPENAI_CAPABILITIES = ['image', 'speech']; + +/** + * Resolves the API key from options or the environment + * + * @param {Object} options - Call options + * @returns {string} - The API key + */ +function resolveApiKey(options) { + const key = options.apiKey ?? process.env.OPENAI_API_KEY; + if (!key) { + throw new MissingCredentialsError('openai', 'OPENAI_API_KEY'); + } + return key; +} + +/** + * Builds the auth headers for a JSON request + * + * @param {string} apiKey - API key + * @returns {Object} - Header map + */ +function jsonHeaders(apiKey) { + return { + 'content-type': 'application/json', + authorization: `Bearer ${apiKey}` + }; +} + +/** + * Generates an image with the OpenAI image API + * + * @param {Object} options - Generation options + * @param {string} options.prompt - Text prompt + * @param {string} [options.model] - Model override + * @param {string} [options.size='1024x1024'] - Output size + * @param {string} [options.quality] - Quality hint + * @param {string} [options.background] - Background handling, e.g. 'transparent' + * @returns {Promise} - The generated image + */ +export async function generateImage(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? OPENAI_DEFAULT_MODELS.image; + + if (!options.prompt) { + throw new GenMediaError('generateImage requires a prompt', { provider: 'openai', model }); + } + + const body = { + model, + prompt: options.prompt, + size: options.size ?? '1024x1024', + n: 1 + }; + if (options.quality) body.quality = options.quality; + if (options.background) body.background = options.background; + if (options.outputFormat) body.output_format = options.outputFormat; + + const payload = await requestJson( + `${API_BASE}/images/generations`, + { method: 'POST', headers: jsonHeaders(apiKey), body: JSON.stringify(body) }, + { ...options, provider: 'openai' } + ); + + const first = payload?.data?.[0]; + if (!first?.b64_json) { + throw new GenMediaError('OpenAI returned no image data', { provider: 'openai', model }); + } + + const format = options.outputFormat ?? 'png'; + + return new GeneratedMedia({ + data: Buffer.from(first.b64_json, 'base64'), + mimeType: `image/${format === 'jpg' ? 'jpeg' : format}`, + provider: 'openai', + model, + kind: 'image', + meta: { size: body.size, revisedPrompt: first.revised_prompt ?? null } + }); +} + +/** + * Generates speech with the OpenAI audio API + * + * @param {Object} options - Generation options + * @param {string} options.text - Text to speak + * @param {string} [options.voice='alloy'] - Voice name + * @param {string} [options.instructions] - Delivery instructions + * @param {string} [options.format='mp3'] - Output container + * @param {string} [options.model] - Model override + * @returns {Promise} - The generated speech + */ +export async function generateSpeech(options = {}) { + const apiKey = resolveApiKey(options); + const model = options.model ?? OPENAI_DEFAULT_MODELS.speech; + + if (!options.text) { + throw new GenMediaError('generateSpeech requires text', { provider: 'openai', model }); + } + + const format = options.format ?? 'mp3'; + const body = { + model, + input: options.text, + voice: options.voice ?? 'alloy', + response_format: format + }; + if (options.instructions) body.instructions = options.instructions; + + const data = await requestBuffer( + `${API_BASE}/audio/speech`, + { method: 'POST', headers: jsonHeaders(apiKey), body: JSON.stringify(body) }, + { ...options, provider: 'openai' } + ); + + return new GeneratedMedia({ + data, + mimeType: format === 'mp3' ? 'audio/mpeg' : `audio/${format}`, + provider: 'openai', + model, + kind: 'speech', + meta: { voice: body.voice } + }); +} + +export const openaiProvider = { + name: 'openai', + capabilities: OPENAI_CAPABILITIES, + defaultModels: OPENAI_DEFAULT_MODELS, + envVars: ['OPENAI_API_KEY'], + generateImage, + generateSpeech +}; diff --git a/src/genmedia/registry.js b/src/genmedia/registry.js new file mode 100644 index 0000000..dfc2f99 --- /dev/null +++ b/src/genmedia/registry.js @@ -0,0 +1,197 @@ +/** + * @profullstack/transcoder - Generative Media Registry + * Keeps track of which providers exist, what each can do, and which one to use + * when the caller does not name one. + */ + +import { googleProvider } from './providers/google.js'; +import { openaiProvider } from './providers/openai.js'; +import { elevenlabsProvider } from './providers/elevenlabs.js'; +import { UnsupportedCapabilityError, GenMediaError, MissingCredentialsError } from './errors.js'; + +/** Every capability the layer knows about */ +export const CAPABILITIES = ['image', 'video', 'speech', 'music']; + +/** + * Provider preference per capability, most preferred first. + * + * Speech leads with ElevenLabs because it is the incumbent across the existing + * pipelines and swapping it silently would change the sound of shipped products. + * Image leads with Google because Gemini Image renders legible text and holds + * character consistency across a set, which is what the asset pipelines need. + */ +export const PROVIDER_PREFERENCE = { + image: ['google', 'openai'], + video: ['google'], + speech: ['elevenlabs', 'google', 'openai'], + music: ['google'] +}; + +/** The method each capability maps to on a provider */ +const METHOD_BY_CAPABILITY = { + image: 'generateImage', + video: 'generateVideo', + speech: 'generateSpeech', + music: 'generateMusic' +}; + +const registry = new Map(); + +/** + * Registers a provider, replacing any existing one with the same name + * + * @param {Object} provider - Provider implementation + * @returns {Object} - The registered provider + */ +export function registerProvider(provider) { + if (!provider?.name) { + throw new GenMediaError('A provider must have a name'); + } + registry.set(provider.name, provider); + return provider; +} + +/** + * @param {string} name - Provider name + * @returns {Object|undefined} - The provider, if registered + */ +export function getProvider(name) { + return registry.get(name); +} + +/** + * @returns {Array} - Names of all registered providers + */ +export function listProviders() { + return [...registry.keys()]; +} + +/** + * Reports whether a provider has credentials available in the environment + * + * @param {string} name - Provider name + * @returns {boolean} - True when at least one of its env vars is set + */ +export function hasCredentials(name) { + const provider = registry.get(name); + if (!provider) return false; + return (provider.envVars ?? []).some(envVar => Boolean(process.env[envVar])); +} + +/** + * Describes every provider, what it can generate, and whether it is usable now. + * Useful for a `--capabilities` style CLI flag or a startup health check. + * + * @returns {Array} - One entry per registered provider + */ +export function describeProviders() { + return listProviders().map(name => { + const provider = registry.get(name); + return { + name, + capabilities: provider.capabilities ?? [], + defaultModels: provider.defaultModels ?? {}, + envVars: provider.envVars ?? [], + configured: hasCredentials(name) + }; + }); +} + +/** + * Picks the provider to use for a capability. + * + * Resolution order: an explicit option, then the GENMEDIA_PROVIDER environment + * variable, then the first preferred provider that actually has credentials, + * and finally the first preferred provider at all so the caller gets a clear + * MissingCredentialsError rather than a confusing "no provider" error. + * + * @param {string} capability - One of image, video, speech, music + * @param {Object} [options={}] - Call options + * @param {string} [options.provider] - Explicit provider name + * @returns {Object} - The resolved provider + */ +export function resolveProvider(capability, options = {}) { + if (!CAPABILITIES.includes(capability)) { + throw new GenMediaError(`Unknown capability "${capability}"`, { capability }); + } + + const explicit = options.provider ?? process.env.GENMEDIA_PROVIDER; + + if (explicit) { + const provider = registry.get(explicit); + if (!provider) { + throw new GenMediaError( + `Unknown provider "${explicit}". Registered: ${listProviders().join(', ')}`, + { provider: explicit } + ); + } + if (!supports(provider, capability)) { + throw new UnsupportedCapabilityError(explicit, capability); + } + return provider; + } + + const preference = PROVIDER_PREFERENCE[capability] ?? []; + const candidates = preference + .map(name => registry.get(name)) + .filter(provider => provider && supports(provider, capability)); + + if (candidates.length === 0) { + throw new GenMediaError(`No registered provider can generate ${capability}`, { capability }); + } + + return candidates.find(provider => hasCredentials(provider.name)) ?? candidates[0]; +} + +/** + * @param {Object} provider - Provider implementation + * @param {string} capability - Capability to check + * @returns {boolean} - True when the provider implements the capability + */ +export function supports(provider, capability) { + const method = METHOD_BY_CAPABILITY[capability]; + return typeof provider?.[method] === 'function'; +} + +/** + * Invokes a capability on the resolved provider + * + * @param {string} capability - Capability to invoke + * @param {Object} [options={}] - Options forwarded to the provider + * @returns {Promise} - The generated asset + */ +export async function invoke(capability, options = {}) { + const provider = resolveProvider(capability, options); + const method = METHOD_BY_CAPABILITY[capability]; + + if (options.fallback === false) { + return provider[method](options); + } + + try { + return await provider[method](options); + } catch (error) { + // A provider with no key configured is a setup problem, not a runtime one: + // fall through to the next preferred provider that is actually usable. + if (!(error instanceof MissingCredentialsError) || options.provider) { + throw error; + } + + const alternative = (PROVIDER_PREFERENCE[capability] ?? []) + .map(name => registry.get(name)) + .find( + candidate => + candidate && + candidate.name !== provider.name && + supports(candidate, capability) && + hasCredentials(candidate.name) + ); + + if (!alternative) throw error; + return alternative[method](options); + } +} + +registerProvider(googleProvider); +registerProvider(openaiProvider); +registerProvider(elevenlabsProvider); diff --git a/src/index.js b/src/index.js index 09eb232..d1bfe1b 100644 --- a/src/index.js +++ b/src/index.js @@ -84,4 +84,34 @@ export { export { createBatchUI, attachBatchUI -} from './terminal-ui.js'; \ No newline at end of file +} from './terminal-ui.js'; + +// Export generative media functionality +export { + generateImage, + generateVideo, + generateSpeech, + generateMusic, + generateBatch, + registerProvider, + getProvider, + listProviders, + describeProviders, + resolveProvider, + hasCredentials, + supports, + GeneratedMedia, + pcmToWav, + parsePcmMimeType, + extensionForMimeType, + GenMediaError, + MissingCredentialsError, + UnsupportedCapabilityError, + ProviderResponseError, + GenerationTimeoutError, + CAPABILITIES, + PROVIDER_PREFERENCE +} from './genmedia/index.js'; + +// Also available as a namespace: import { genmedia } from '@profullstack/transcoder' +export * as genmedia from './genmedia/index.js'; \ No newline at end of file diff --git a/test/genmedia.test.js b/test/genmedia.test.js new file mode 100644 index 0000000..11b7ea7 --- /dev/null +++ b/test/genmedia.test.js @@ -0,0 +1,763 @@ +/** + * Tests for the generative media layer. + * + * Every provider call is exercised through an injected fetch, so the suite runs + * offline and without credentials. + */ + +import { expect } from 'chai'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { + generateImage, + generateVideo, + generateSpeech, + generateMusic, + generateBatch, + registerProvider, + getProvider, + listProviders, + describeProviders, + resolveProvider, + hasCredentials, + supports, + GeneratedMedia, + pcmToWav, + parsePcmMimeType, + extensionForMimeType, + MissingCredentialsError, + UnsupportedCapabilityError, + ProviderResponseError, + PROVIDER_PREFERENCE +} from '../src/genmedia/index.js'; + +import { backoffDelay, request, DEFAULT_HTTP_OPTIONS } from '../src/genmedia/http.js'; + +/** Env vars the layer reads, cleared between tests so the host machine cannot leak in */ +const MANAGED_ENV = [ + 'GOOGLE_API_KEY', + 'GEMINI_API_KEY', + 'GOOGLE_GENAI_API_KEY', + 'GOOGLE_CLOUD_PROJECT', + 'GOOGLE_ACCESS_TOKEN', + 'GOOGLE_CLOUD_LOCATION', + 'OPENAI_API_KEY', + 'ELEVENLABS_API_KEY', + 'ELEVENLABS_VOICE_ID', + 'GENMEDIA_PROVIDER' +]; + +let savedEnv = {}; + +/** + * Builds a fetch stub that returns a queue of canned responses and records calls + * + * @param {Array} responses - Responses as {status, json, buffer, text, headers} + * @returns {Function} - A fetch implementation with a `calls` array attached + */ +function stubFetch(responses) { + const queue = [...responses]; + const impl = async (url, init) => { + impl.calls.push({ url, init, body: init?.body ? JSON.parse(init.body) : null }); + const next = queue.shift() ?? responses[responses.length - 1]; + const status = next.status ?? 200; + const headers = new Map(Object.entries(next.headers ?? {})); + + return { + ok: status >= 200 && status < 300, + status, + headers: { get: key => headers.get(key) ?? null }, + json: async () => next.json, + text: async () => next.text ?? JSON.stringify(next.json ?? ''), + arrayBuffer: async () => { + const buffer = next.buffer ?? Buffer.from(''); + return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); + } + }; + }; + impl.calls = []; + return impl; +} + +beforeEach(function () { + savedEnv = {}; + for (const key of MANAGED_ENV) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } +}); + +afterEach(function () { + for (const key of MANAGED_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = savedEnv[key]; + } + } +}); + +describe('GenMedia Media Helpers', function () { + describe('pcmToWav()', function () { + it('should prepend a valid 44-byte RIFF header', function () { + const pcm = Buffer.alloc(100, 7); + const wav = pcmToWav(pcm, { sampleRate: 24000, channels: 1, bitsPerSample: 16 }); + + expect(wav.length).to.equal(144); + expect(wav.toString('ascii', 0, 4)).to.equal('RIFF'); + expect(wav.toString('ascii', 8, 12)).to.equal('WAVE'); + expect(wav.toString('ascii', 36, 40)).to.equal('data'); + expect(wav.readUInt32LE(4)).to.equal(136); + expect(wav.readUInt32LE(40)).to.equal(100); + }); + + it('should compute byte rate and block align from the format', function () { + const wav = pcmToWav(Buffer.alloc(8), { sampleRate: 48000, channels: 2, bitsPerSample: 16 }); + + expect(wav.readUInt16LE(22)).to.equal(2); // channels + expect(wav.readUInt32LE(24)).to.equal(48000); // sample rate + expect(wav.readUInt32LE(28)).to.equal(192000); // byte rate + expect(wav.readUInt16LE(32)).to.equal(4); // block align + }); + + it('should default to 24kHz mono 16-bit', function () { + const wav = pcmToWav(Buffer.alloc(4)); + expect(wav.readUInt32LE(24)).to.equal(24000); + expect(wav.readUInt16LE(22)).to.equal(1); + expect(wav.readUInt16LE(34)).to.equal(16); + }); + }); + + describe('parsePcmMimeType()', function () { + it('should parse rate and bit depth out of an L16 type', function () { + expect(parsePcmMimeType('audio/L16;rate=24000')).to.deep.equal({ + bitsPerSample: 16, + sampleRate: 24000, + channels: 1 + }); + }); + + it('should read an explicit channel count', function () { + expect(parsePcmMimeType('audio/L16; rate=44100; channels=2')).to.deep.equal({ + bitsPerSample: 16, + sampleRate: 44100, + channels: 2 + }); + }); + + it('should return null for container formats', function () { + expect(parsePcmMimeType('audio/wav')).to.equal(null); + expect(parsePcmMimeType('audio/mpeg')).to.equal(null); + expect(parsePcmMimeType('')).to.equal(null); + expect(parsePcmMimeType(null)).to.equal(null); + }); + }); + + describe('extensionForMimeType()', function () { + it('should map known media types', function () { + expect(extensionForMimeType('image/png')).to.equal('.png'); + expect(extensionForMimeType('video/mp4')).to.equal('.mp4'); + expect(extensionForMimeType('audio/mpeg')).to.equal('.mp3'); + expect(extensionForMimeType('audio/wav')).to.equal('.wav'); + }); + + it('should ignore mime parameters and casing', function () { + expect(extensionForMimeType('IMAGE/PNG; charset=binary')).to.equal('.png'); + }); + + it('should fall back for unknown types', function () { + expect(extensionForMimeType('application/x-thing')).to.equal('.bin'); + expect(extensionForMimeType(undefined)).to.equal('.bin'); + }); + }); + + describe('GeneratedMedia', function () { + const sample = () => + new GeneratedMedia({ + data: Buffer.from('hello'), + mimeType: 'image/png', + provider: 'test', + model: 'test-model', + kind: 'image' + }); + + it('should expose size and extension', function () { + const media = sample(); + expect(media.size).to.equal(5); + expect(media.extension).to.equal('.png'); + }); + + it('should reject a non-Buffer payload', function () { + expect(() => new GeneratedMedia({ data: 'nope', mimeType: 'image/png' })).to.throw( + /requires a Buffer/ + ); + }); + + it('should build a data URI', function () { + expect(sample().toDataUri()).to.equal(`data:image/png;base64,${Buffer.from('hello').toString('base64')}`); + }); + + it('should write to disk and create parent directories', async function () { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'genmedia-')); + const target = path.join(dir, 'nested', 'deeper', 'out.png'); + + const written = await sample().toFile(target); + + expect(written).to.equal(target); + expect(fs.readFileSync(target).toString()).to.equal('hello'); + await fs.promises.rm(dir, { recursive: true, force: true }); + }); + + it('should append the mime-implied extension when the path has none', async function () { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'genmedia-')); + const written = await sample().toFile(path.join(dir, 'out')); + + expect(written).to.equal(path.join(dir, 'out.png')); + expect(fs.existsSync(written)).to.equal(true); + await fs.promises.rm(dir, { recursive: true, force: true }); + }); + }); +}); + +describe('GenMedia HTTP', function () { + describe('backoffDelay()', function () { + it('should honour Retry-After when present', function () { + expect(backoffDelay(0, DEFAULT_HTTP_OPTIONS, 2)).to.equal(2000); + }); + + it('should clamp Retry-After to the maximum delay', function () { + expect(backoffDelay(0, DEFAULT_HTTP_OPTIONS, 9999)).to.equal(DEFAULT_HTTP_OPTIONS.maxRetryDelay); + }); + + it('should stay within the exponential ceiling', function () { + for (let attempt = 0; attempt < 6; attempt++) { + const value = backoffDelay(attempt, DEFAULT_HTTP_OPTIONS); + expect(value).to.be.at.least(0); + expect(value).to.be.at.most(DEFAULT_HTTP_OPTIONS.maxRetryDelay); + } + }); + }); + + describe('request()', function () { + it('should retry a 429 and then succeed', async function () { + const fetchImpl = stubFetch([ + { status: 429, text: 'slow down', headers: { 'retry-after': '0' } }, + { status: 200, json: { ok: true } } + ]); + + const response = await request('https://example.test', {}, { + fetchImpl, + provider: 'test', + retryBaseDelay: 1 + }); + + expect(response.status).to.equal(200); + expect(fetchImpl.calls).to.have.length(2); + }); + + it('should not retry a 400', async function () { + const fetchImpl = stubFetch([{ status: 400, text: 'bad prompt' }]); + + try { + await request('https://example.test', {}, { fetchImpl, provider: 'test', retryBaseDelay: 1 }); + expect.fail('should have thrown'); + } catch (error) { + expect(error).to.be.instanceOf(ProviderResponseError); + expect(error.status).to.equal(400); + expect(fetchImpl.calls).to.have.length(1); + } + }); + + it('should give up after the retry budget', async function () { + const fetchImpl = stubFetch([{ status: 503, text: 'unavailable' }]); + + try { + await request('https://example.test', {}, { + fetchImpl, + provider: 'test', + retries: 2, + retryBaseDelay: 1 + }); + expect.fail('should have thrown'); + } catch (error) { + expect(error.status).to.equal(503); + expect(fetchImpl.calls).to.have.length(3); + } + }); + + it('should retry network faults', async function () { + let attempts = 0; + const fetchImpl = async () => { + attempts++; + if (attempts < 2) throw new Error('ECONNRESET'); + return { + ok: true, + status: 200, + headers: { get: () => null }, + json: async () => ({ ok: true }) + }; + }; + + const response = await request('https://example.test', {}, { + fetchImpl, + provider: 'test', + retryBaseDelay: 1 + }); + + expect(response.status).to.equal(200); + expect(attempts).to.equal(2); + }); + }); +}); + +describe('GenMedia Registry', function () { + it('should register the three built-in providers', function () { + expect(listProviders()).to.include.members(['google', 'openai', 'elevenlabs']); + }); + + it('should report capabilities per provider', function () { + expect(getProvider('google').capabilities).to.include.members(['image', 'video', 'speech', 'music']); + expect(getProvider('elevenlabs').capabilities).to.deep.equal(['speech']); + expect(supports(getProvider('openai'), 'video')).to.equal(false); + }); + + it('should detect credentials from the environment', function () { + expect(hasCredentials('openai')).to.equal(false); + process.env.OPENAI_API_KEY = 'sk-test'; + expect(hasCredentials('openai')).to.equal(true); + }); + + it('should prefer a configured provider over an unconfigured one', function () { + process.env.OPENAI_API_KEY = 'sk-test'; + expect(resolveProvider('image').name).to.equal('openai'); + + process.env.GOOGLE_API_KEY = 'g-test'; + expect(resolveProvider('image').name).to.equal('google'); + }); + + it('should fall back to the first preferred provider when nothing is configured', function () { + expect(resolveProvider('image').name).to.equal(PROVIDER_PREFERENCE.image[0]); + }); + + it('should honour an explicit provider option', function () { + expect(resolveProvider('speech', { provider: 'openai' }).name).to.equal('openai'); + }); + + it('should honour GENMEDIA_PROVIDER', function () { + process.env.GENMEDIA_PROVIDER = 'openai'; + expect(resolveProvider('image').name).to.equal('openai'); + }); + + it('should reject a provider that cannot do the job', function () { + expect(() => resolveProvider('video', { provider: 'elevenlabs' })).to.throw( + UnsupportedCapabilityError + ); + }); + + it('should reject an unknown provider by name', function () { + expect(() => resolveProvider('image', { provider: 'nope' })).to.throw(/Unknown provider/); + }); + + it('should describe configuration state for every provider', function () { + process.env.ELEVENLABS_API_KEY = 'el-test'; + const described = describeProviders(); + const elevenlabs = described.find(entry => entry.name === 'elevenlabs'); + + expect(elevenlabs.configured).to.equal(true); + expect(described.find(entry => entry.name === 'google').configured).to.equal(false); + }); + + it('should accept a custom provider', async function () { + registerProvider({ + name: 'stub', + capabilities: ['image'], + envVars: [], + generateImage: async () => + new GeneratedMedia({ + data: Buffer.from('x'), + mimeType: 'image/png', + provider: 'stub', + model: 'stub-1', + kind: 'image' + }) + }); + + const media = await generateImage({ provider: 'stub', prompt: 'hi' }); + expect(media.provider).to.equal('stub'); + }); + + it('should fall back to a configured provider when the preferred one has no key', async function () { + process.env.OPENAI_API_KEY = 'sk-test'; + + const fetchImpl = stubFetch([ + { status: 200, json: { data: [{ b64_json: Buffer.from('img').toString('base64') }] } } + ]); + + // Google is preferred for images but unconfigured, so this must land on OpenAI + const media = await generateImage({ prompt: 'a cat', fetchImpl }); + + expect(media.provider).to.equal('openai'); + }); + + it('should not silently fall back when a provider was named explicitly', async function () { + process.env.OPENAI_API_KEY = 'sk-test'; + + try { + await generateImage({ prompt: 'a cat', provider: 'google' }); + expect.fail('should have thrown'); + } catch (error) { + expect(error).to.be.instanceOf(MissingCredentialsError); + expect(error.provider).to.equal('google'); + } + }); +}); + +describe('GenMedia Google Provider', function () { + beforeEach(function () { + process.env.GOOGLE_API_KEY = 'g-test'; + }); + + it('should generate an image from inline response data', async function () { + const fetchImpl = stubFetch([ + { + status: 200, + json: { + candidates: [ + { + content: { + parts: [{ inlineData: { mimeType: 'image/png', data: Buffer.from('png-bytes').toString('base64') } }] + } + } + ] + } + } + ]); + + const media = await generateImage({ prompt: 'a studio', aspectRatio: '16:9', fetchImpl }); + + expect(media.provider).to.equal('google'); + expect(media.kind).to.equal('image'); + expect(media.data.toString()).to.equal('png-bytes'); + expect(media.extension).to.equal('.png'); + expect(fetchImpl.calls[0].init.headers['x-goog-api-key']).to.equal('g-test'); + expect(fetchImpl.calls[0].body.generationConfig.imageConfig.aspectRatio).to.equal('16:9'); + }); + + it('should send reference images alongside the prompt', async function () { + const fetchImpl = stubFetch([ + { + status: 200, + json: { + candidates: [{ content: { parts: [{ inlineData: { mimeType: 'image/png', data: 'AA==' } }] } }] + } + } + ]); + + await generateImage({ + prompt: 'same character, new scene', + referenceImages: [{ data: Buffer.from('ref'), mimeType: 'image/jpeg' }], + fetchImpl + }); + + const parts = fetchImpl.calls[0].body.contents[0].parts; + expect(parts).to.have.length(2); + expect(parts[1].inlineData.mimeType).to.equal('image/jpeg'); + expect(parts[1].inlineData.data).to.equal(Buffer.from('ref').toString('base64')); + }); + + it('should surface a blocked prompt clearly', async function () { + const fetchImpl = stubFetch([ + { status: 200, json: { promptFeedback: { blockReason: 'SAFETY' }, candidates: [] } } + ]); + + try { + await generateImage({ prompt: 'nope', fetchImpl }); + expect.fail('should have thrown'); + } catch (error) { + expect(error.message).to.match(/blocked \(SAFETY\)/); + } + }); + + it('should poll a Veo operation until it completes', async function () { + const fetchImpl = stubFetch([ + { status: 200, json: { name: 'models/veo/operations/abc' } }, + { status: 200, json: { done: false } }, + { + status: 200, + json: { + done: true, + name: 'models/veo/operations/abc', + response: { + generateVideoResponse: { + generatedSamples: [ + { video: { bytesBase64Encoded: Buffer.from('mp4-bytes').toString('base64'), mimeType: 'video/mp4' } } + ] + } + } + } + } + ]); + + const media = await generateVideo({ prompt: 'a dolly shot', fetchImpl, pollInterval: 1 }); + + expect(media.kind).to.equal('video'); + expect(media.data.toString()).to.equal('mp4-bytes'); + expect(media.meta.hasNativeAudio).to.equal(true); + expect(fetchImpl.calls).to.have.length(3); + }); + + it('should download a Veo result served as a URI', async function () { + const fetchImpl = stubFetch([ + { status: 200, json: { name: 'ops/1' } }, + { + status: 200, + json: { + done: true, + response: { + generateVideoResponse: { + generatedSamples: [{ video: { uri: 'https://files.test/v.mp4', mimeType: 'video/mp4' } }] + } + } + } + }, + { status: 200, buffer: Buffer.from('downloaded-mp4') } + ]); + + const media = await generateVideo({ prompt: 'x', fetchImpl, pollInterval: 1 }); + + expect(media.data.toString()).to.equal('downloaded-mp4'); + expect(fetchImpl.calls[2].url).to.equal('https://files.test/v.mp4'); + }); + + it('should fail when a Veo operation reports an error', async function () { + const fetchImpl = stubFetch([ + { status: 200, json: { name: 'ops/1' } }, + { status: 200, json: { done: true, error: { code: 3, message: 'invalid prompt' } } } + ]); + + try { + await generateVideo({ prompt: 'x', fetchImpl, pollInterval: 1 }); + expect.fail('should have thrown'); + } catch (error) { + expect(error.message).to.match(/invalid prompt/); + } + }); + + it('should wrap PCM speech in a WAV container', async function () { + const pcm = Buffer.alloc(64, 3); + const fetchImpl = stubFetch([ + { + status: 200, + json: { + candidates: [ + { + content: { + parts: [{ inlineData: { mimeType: 'audio/L16;rate=24000', data: pcm.toString('base64') } }] + } + } + ] + } + } + ]); + + const media = await generateSpeech({ text: 'hello there', provider: 'google', fetchImpl }); + + expect(media.mimeType).to.equal('audio/wav'); + expect(media.extension).to.equal('.wav'); + expect(media.data.length).to.equal(pcm.length + 44); + expect(media.data.toString('ascii', 0, 4)).to.equal('RIFF'); + expect(media.meta.sourceMimeType).to.equal('audio/L16;rate=24000'); + }); + + it('should build a multi-speaker speech config', async function () { + const fetchImpl = stubFetch([ + { + status: 200, + json: { + candidates: [{ content: { parts: [{ inlineData: { mimeType: 'audio/wav', data: 'AA==' } }] } }] + } + } + ]); + + await generateSpeech({ + text: 'Host: hi\nGuest: hello', + provider: 'google', + speakers: [ + { speaker: 'Host', voice: 'Kore' }, + { speaker: 'Guest', voice: 'Puck' } + ], + fetchImpl + }); + + const config = fetchImpl.calls[0].body.generationConfig.speechConfig; + expect(config.multiSpeakerVoiceConfig.speakerVoiceConfigs).to.have.length(2); + expect(config.multiSpeakerVoiceConfig.speakerVoiceConfigs[1].voiceConfig.prebuiltVoiceConfig.voiceName).to.equal('Puck'); + }); + + it('should require project and token for Lyria', async function () { + try { + await generateMusic({ prompt: 'gospel choir over a hip hop beat' }); + expect.fail('should have thrown'); + } catch (error) { + expect(error).to.be.instanceOf(MissingCredentialsError); + expect(error.envVar).to.equal('GOOGLE_CLOUD_PROJECT'); + } + }); + + it('should generate music from a Vertex prediction', async function () { + process.env.GOOGLE_CLOUD_PROJECT = 'proj'; + process.env.GOOGLE_ACCESS_TOKEN = 'ya29.test'; + + const fetchImpl = stubFetch([ + { status: 200, json: { predictions: [{ bytesBase64Encoded: Buffer.from('wav-bytes').toString('base64') }] } } + ]); + + const media = await generateMusic({ prompt: 'uplifting gospel', seed: 42, fetchImpl }); + + expect(media.kind).to.equal('music'); + expect(media.data.toString()).to.equal('wav-bytes'); + expect(media.meta.seed).to.equal(42); + expect(fetchImpl.calls[0].url).to.match(/us-central1-aiplatform\.googleapis\.com/); + expect(fetchImpl.calls[0].init.headers.authorization).to.equal('Bearer ya29.test'); + expect(fetchImpl.calls[0].body.instances[0].seed).to.equal(42); + }); + + it('should require a prompt', async function () { + try { + await generateImage({ provider: 'google' }); + expect.fail('should have thrown'); + } catch (error) { + expect(error.message).to.match(/requires a prompt/); + } + }); +}); + +describe('GenMedia OpenAI Provider', function () { + beforeEach(function () { + process.env.OPENAI_API_KEY = 'sk-test'; + }); + + it('should generate an image', async function () { + const fetchImpl = stubFetch([ + { status: 200, json: { data: [{ b64_json: Buffer.from('img').toString('base64'), revised_prompt: 'a cat, detailed' }] } } + ]); + + const media = await generateImage({ prompt: 'a cat', provider: 'openai', size: '1536x1024', fetchImpl }); + + expect(media.provider).to.equal('openai'); + expect(media.mimeType).to.equal('image/png'); + expect(media.meta.revisedPrompt).to.equal('a cat, detailed'); + expect(fetchImpl.calls[0].body.size).to.equal('1536x1024'); + }); + + it('should generate speech as binary audio', async function () { + const fetchImpl = stubFetch([{ status: 200, buffer: Buffer.from('mp3-bytes') }]); + + const media = await generateSpeech({ text: 'hello', provider: 'openai', voice: 'nova', fetchImpl }); + + expect(media.mimeType).to.equal('audio/mpeg'); + expect(media.data.toString()).to.equal('mp3-bytes'); + expect(fetchImpl.calls[0].body.voice).to.equal('nova'); + }); + + it('should require credentials', async function () { + delete process.env.OPENAI_API_KEY; + + try { + await generateImage({ prompt: 'x', provider: 'openai' }); + expect.fail('should have thrown'); + } catch (error) { + expect(error).to.be.instanceOf(MissingCredentialsError); + expect(error.envVar).to.equal('OPENAI_API_KEY'); + } + }); +}); + +describe('GenMedia ElevenLabs Provider', function () { + beforeEach(function () { + process.env.ELEVENLABS_API_KEY = 'el-test'; + }); + + it('should generate speech with the default voice', async function () { + const fetchImpl = stubFetch([{ status: 200, buffer: Buffer.from('mp3') }]); + + const media = await generateSpeech({ text: 'hi', fetchImpl }); + + expect(media.provider).to.equal('elevenlabs'); + expect(media.mimeType).to.equal('audio/mpeg'); + expect(fetchImpl.calls[0].url).to.match(/21m00Tcm4TlvDq8ikWAM/); + expect(fetchImpl.calls[0].init.headers['xi-api-key']).to.equal('el-test'); + }); + + it('should use a voice id from the environment', async function () { + process.env.ELEVENLABS_VOICE_ID = 'custom-voice'; + const fetchImpl = stubFetch([{ status: 200, buffer: Buffer.from('mp3') }]); + + await generateSpeech({ text: 'hi', fetchImpl }); + + expect(fetchImpl.calls[0].url).to.match(/custom-voice/); + }); +}); + +describe('GenMedia Batch', function () { + beforeEach(function () { + process.env.OPENAI_API_KEY = 'sk-test'; + }); + + it('should preserve request order and bound concurrency', async function () { + let inFlight = 0; + let peak = 0; + + registerProvider({ + name: 'counter', + capabilities: ['image'], + envVars: [], + generateImage: async options => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise(resolve => setTimeout(resolve, 5)); + inFlight--; + return new GeneratedMedia({ + data: Buffer.from(options.prompt), + mimeType: 'image/png', + provider: 'counter', + model: 'counter-1', + kind: 'image' + }); + } + }); + + const requests = ['a', 'b', 'c', 'd', 'e'].map(prompt => ({ + kind: 'image', + provider: 'counter', + prompt + })); + + const results = await generateBatch(requests, { concurrency: 2 }); + + expect(results.map(entry => entry.media.data.toString())).to.deep.equal(['a', 'b', 'c', 'd', 'e']); + expect(peak).to.be.at.most(2); + }); + + it('should capture per-item errors without failing the batch', async function () { + const results = await generateBatch([ + { kind: 'image', provider: 'openai', prompt: '' }, + { kind: 'nonsense', prompt: 'x' } + ]); + + expect(results[0].media).to.equal(null); + expect(results[0].error.message).to.match(/requires a prompt/); + expect(results[1].error.message).to.match(/Unknown kind/); + }); + + it('should report progress', async function () { + const seen = []; + await generateBatch([{ kind: 'nonsense' }, { kind: 'nonsense' }], { + onProgress: event => seen.push(event.completed) + }); + + expect(seen).to.deep.equal([1, 2]); + }); +});