diff --git a/Cargo.lock b/Cargo.lock index b7c0a0e1..6088042f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3402,6 +3402,7 @@ dependencies = [ "same-file", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", ] @@ -3531,6 +3532,7 @@ dependencies = [ "cap-std", "cpal", "futures", + "futures-util", "image", "libc", "objc2-app-kit", @@ -4180,6 +4182,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", "futures-util", "http", diff --git a/crates/opentake-agent/src/chat/loop.rs b/crates/opentake-agent/src/chat/loop.rs index 8a494ec7..8bb3665a 100644 --- a/crates/opentake-agent/src/chat/loop.rs +++ b/crates/opentake-agent/src/chat/loop.rs @@ -257,9 +257,9 @@ impl ChatLoop { /// When the dispatcher lacks a media bridge, hide the bridge-dependent /// tools instead of advertising tools that would only fail at runtime. fn tool_catalog(&self) -> Vec { - ToolName::ALL - .iter() - .copied() + self.dispatcher + .advertised_tools() + .into_iter() .filter(|tool| { self.dispatcher.has_media_bridge() || !matches!( diff --git a/crates/opentake-agent/src/mcp/dispatch.rs b/crates/opentake-agent/src/mcp/dispatch.rs index 9dd34b06..8b094886 100644 --- a/crates/opentake-agent/src/mcp/dispatch.rs +++ b/crates/opentake-agent/src/mcp/dispatch.rs @@ -22,8 +22,8 @@ use std::sync::{Arc, Mutex, RwLock}; use opentake_domain::{AnimPair, Crop, Interpolation, Keyframe, KeyframeTrack}; use opentake_domain::{ - ChromaKey, ColorGrade, Effect, LiftGammaGain, Mask, MaskShape, MediaManifest, Point2, Rgb, - Rgba, TextStyle, Timeline, Transform, VideoType, + ChromaKey, ColorGrade, Effect, GenerationJobStatus, LiftGammaGain, Mask, MaskShape, + MediaManifest, Point2, Rgb, Rgba, TextStyle, Timeline, Transform, VideoType, }; use opentake_media::analysis::{ detect_beats, detect_silences, BeatDetectionConfig, SilenceDetectionConfig, @@ -37,6 +37,7 @@ use serde_json::Value; use crate::mcp::core_handle::CoreHandle; use crate::mcp::gen_catalog; +use crate::mcp::generation::{GenerationBridge, GenerationRequest}; use crate::mcp::media_bridge::{ frame_to_block, media_frame_to_block, BridgeErrorKind, ImportSource, InspectMediaRequest, InspectMediaResult, InspectResult, MediaBridge, SearchCandidate, TranscriptSource, @@ -63,6 +64,16 @@ const INSPECT_MEDIA_MAX_FRAMES: usize = 12; const INSPECT_MEDIA_MAX_SEGMENTS: usize = 400; const INSPECT_MEDIA_MAX_WORDS: usize = 10_000; +fn is_generation_tool(tool: ToolName) -> bool { + matches!( + tool, + ToolName::GenerateVideo + | ToolName::GenerateImage + | ToolName::GenerateAudio + | ToolName::UpscaleMedia + ) +} + /// The in-process tool dispatcher. Holds the [`CoreHandle`] boundary, the plugin /// registry (read-locked for the active plugin), and a per-dispatcher agent-undo /// stack so `undo` only reverts edits this session made. @@ -74,6 +85,9 @@ pub struct Dispatcher { /// [`CoreHandle`] because those two capabilities reach into crates the agent /// layer does not link (`opentake-render`, the src-tauri import path). bridge: Option>, + /// Paid generation/upscale side-door. The desktop host injects this only + /// when it can persist jobs and run configured providers. + generation_bridge: Option>, /// Action names of agent edits applied through this dispatcher, newest last. /// Guards `undo`: we only revert when this session has pushed an edit. agent_undo: Mutex>, @@ -94,11 +108,22 @@ impl Dispatcher { handle: Arc, registry: Arc>, bridge: Option>, + ) -> Self { + Self::with_bridges(handle, registry, bridge, None) + } + + /// New dispatcher with independent media and generation host capabilities. + pub fn with_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> Self { Dispatcher { handle, registry, bridge, + generation_bridge, agent_undo: Mutex::new(Vec::new()), } } @@ -109,6 +134,20 @@ impl Dispatcher { self.bridge.is_some() } + pub fn can_generate(&self) -> bool { + self.generation_bridge + .as_ref() + .is_some_and(|bridge| bridge.can_generate()) + } + + pub fn advertised_tools(&self) -> Vec { + let mut tools = ToolName::ALL.to_vec(); + if self.can_generate() { + tools.extend(ToolName::GENERATION); + } + tools + } + /// Snapshot the current timeline from the bound core handle. pub fn timeline(&self) -> Timeline { self.handle.timeline() @@ -145,7 +184,9 @@ impl Dispatcher { error.message, ); } - if !ToolName::ALL.contains(&tool) { + if !(ToolName::ALL.contains(&tool) + || is_generation_tool(tool) && self.generation_bridge.is_some()) + { return ToolResult::public_error( PublicErrorKind::UnknownTool, format!("Tool is not advertised: {}", tool.as_str()), @@ -208,16 +249,15 @@ impl Dispatcher { ToolName::GetTimeline => { let a: GetTimelineArgs = decode_tool_args(args, "")?; let tl = self.handle.timeline(); - // canGenerate is gated by the (not-yet-wired) generation backend; - // false until that lands so the model never proposes generation. - let json = encode_timeline(&tl, a.start_frame, a.end_frame, false); + let json = encode_timeline(&tl, a.start_frame, a.end_frame, self.can_generate()); Ok(ToolResult::ok(json.to_string())) } ToolName::GetMedia => { let manifest = self.handle.media(); - let json = serde_json::to_value(&manifest) + let mut json = serde_json::to_value(&manifest) .map(round_floats_3dp) .map_err(|e| ToolError::new(format!("get_media: {e}")))?; + inject_generation_statuses(&mut json, &manifest); Ok(ToolResult::ok(json.to_string())) } ToolName::ListFolders => { @@ -278,17 +318,54 @@ impl Dispatcher { ToolName::GenerateVideo | ToolName::GenerateImage | ToolName::GenerateAudio - | ToolName::UpscaleMedia - | ToolName::AddMotionGraphic - | ToolName::EditMotionGraphic => Ok(ToolResult::error(format!( - "{}: capability is not advertised", - tool.as_str() - ))), + | ToolName::UpscaleMedia => self.submit_generation(tool, args, cancel), + ToolName::AddMotionGraphic | ToolName::EditMotionGraphic => Ok(ToolResult::error( + format!("{}: capability is not advertised", tool.as_str()), + )), } } // MARK: - Generative read bodies + fn submit_generation( + &self, + tool: ToolName, + args: &Value, + cancel: &opentake_media::MediaCancelToken, + ) -> Result { + let request = match tool { + ToolName::GenerateVideo => GenerationRequest::Video(decode_tool_args(args, "")?), + ToolName::GenerateImage => GenerationRequest::Image(decode_tool_args(args, "")?), + ToolName::GenerateAudio => GenerationRequest::Audio(decode_tool_args(args, "")?), + ToolName::UpscaleMedia => GenerationRequest::Upscale(decode_tool_args(args, "")?), + _ => return Err(ToolError::new("not a generation tool")), + }; + if !request.cost_authorized() { + return Ok(ToolResult::public_error( + PublicErrorKind::InvalidArguments(tool), + "costAuthorized must be true after explicit user approval", + )); + } + let Some(bridge) = self.generation_bridge.as_ref() else { + return Ok(ToolResult::public_error( + PublicErrorKind::CapabilityUnavailable(tool), + "Generation is not available in this build.", + )); + }; + if !bridge.can_generate() { + return Ok(ToolResult::public_error( + PublicErrorKind::CapabilityUnavailable(tool), + "Configure a compatible generation provider before submitting.", + )); + } + let submission = bridge + .submit(request, cancel) + .map_err(|_| ToolError::new("generation submission failed"))?; + let payload = serde_json::to_string(&submission) + .map_err(|_| ToolError::new("generation submission response failed"))?; + Ok(ToolResult::ok(payload)) + } + /// `list_models`: project the built-in static catalog from `opentake-gen` /// into the `{ models, loaded }` payload, optionally filtered by `?type=`. /// Fully local — no network, no BYOK key — so it runs synchronously here and @@ -323,6 +400,7 @@ impl Dispatcher { format!("Media not found: {}", a.media_ref), )); }; + ensure_generation_output_ready(entry, "inspect_media")?; if entry.kind == opentake_domain::ClipType::Text { return Ok(ToolResult::public_error( PublicErrorKind::CapabilityUnavailable(ToolName::InspectMedia), @@ -1029,6 +1107,13 @@ impl Dispatcher { let mut explicit_count = 0usize; for (i, raw) in a.entries.iter().enumerate() { let e: AddClipEntry = decode_tool_args(raw, &format!("entries[{i}]"))?; + if let Some(entry) = manifest + .entries + .iter() + .find(|entry| entry.id == e.media_ref) + { + ensure_generation_output_ready(entry, &format!("entries[{i}]"))?; + } let (media_type, has_audio) = resolve_media_kind(manifest, &e.media_ref); if e.track_index.is_some() { explicit_count += 1; @@ -1082,6 +1167,13 @@ impl Dispatcher { let mut entries = Vec::with_capacity(a.entries.len()); for (i, raw) in a.entries.iter().enumerate() { let e: InsertClipEntry = decode_tool_args(raw, &format!("entries[{i}]"))?; + if let Some(entry) = manifest + .entries + .iter() + .find(|entry| entry.id == e.media_ref) + { + ensure_generation_output_ready(entry, &format!("entries[{i}]"))?; + } let (media_type, has_audio) = resolve_media_kind(manifest, &e.media_ref); let duration_frames = match e.duration_frames { Some(d) => d, @@ -2274,6 +2366,63 @@ fn resolve_media_kind( .unwrap_or((opentake_domain::ClipType::Video, false)) } +fn generation_status_label(status: Option) -> &'static str { + match status { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => "generating", + Some(GenerationJobStatus::Downloading | GenerationJobStatus::Finalizing) => "downloading", + Some(GenerationJobStatus::Failed) => "failed", + Some(GenerationJobStatus::Cancelled) => "cancelled", + Some(GenerationJobStatus::Ready) | None => "none", + } +} + +fn ensure_generation_output_ready( + entry: &opentake_domain::MediaManifestEntry, + path: &str, +) -> Result<(), ToolError> { + let status = entry + .generation_input + .as_ref() + .and_then(|input| input.status); + if matches!(status, Some(GenerationJobStatus::Ready) | None) { + return Ok(()); + } + Err(ToolError::new(format!( + "{path}: generated media '{}' is not ready (status {})", + entry.id, + generation_status_label(status) + ))) +} + +fn inject_generation_statuses(payload: &mut Value, manifest: &MediaManifest) { + let Some(entries) = payload + .as_object_mut() + .and_then(|object| object.get_mut("entries")) + .and_then(Value::as_array_mut) + else { + return; + }; + for (payload_entry, manifest_entry) in entries.iter_mut().zip(&manifest.entries) { + let Some(object) = payload_entry.as_object_mut() else { + continue; + }; + let input = manifest_entry.generation_input.as_ref(); + object.insert( + "generationStatus".into(), + Value::String(generation_status_label(input.and_then(|value| value.status)).into()), + ); + if let Some(progress) = input.and_then(|value| value.progress) { + object.insert("generationProgress".into(), json_number(progress, 3)); + } + if let Some(error_code) = input.and_then(|value| value.error_code.as_ref()) { + object.insert( + "generationErrorCode".into(), + Value::String(error_code.clone()), + ); + } + } +} + /// Current `(track_index, start_frame)` of a clip on the timeline, or `(None, /// None)` if absent. Used to fill optional `move_clips` fields. fn clip_location(timeline: &Timeline, clip_id: &str) -> (Option, Option) { @@ -3026,7 +3175,18 @@ fn inspect_media_result( "duration".into(), json_number(inspected.duration_seconds, 3), ); - meta.insert("generationStatus".into(), Value::String("none".into())); + meta.insert( + "generationStatus".into(), + Value::String( + generation_status_label( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + ) + .into(), + ), + ); meta.insert("byteSize".into(), Value::from(inspected.byte_size)); if let Some(file_name) = manifest_file_name(entry) { meta.insert("fileName".into(), Value::String(file_name)); diff --git a/crates/opentake-agent/src/mcp/generation.rs b/crates/opentake-agent/src/mcp/generation.rs new file mode 100644 index 00000000..e17f91a7 --- /dev/null +++ b/crates/opentake-agent/src/mcp/generation.rs @@ -0,0 +1,223 @@ +//! Provider-neutral generation lifecycle coordination. +//! +//! The Agent dispatcher creates durable placeholders synchronously, while the +//! desktop bridge submits and watches the paid provider job off-thread. This +//! module owns the deterministic terminal pairing contract so every returned +//! result is applied to at most one placeholder and every placeholder reaches +//! one persisted terminal state. + +use std::path::PathBuf; + +use serde::Serialize; + +use crate::tools::args::{ + GenerateAudioArgs, GenerateImageArgs, GenerateVideoArgs, UpscaleMediaArgs, +}; + +/// Typed paid-generation request passed from the tool dispatcher to the +/// desktop runtime. The bridge owns model resolution, reference validation, +/// durable placeholder creation, provider submission, and background watch. +#[derive(Debug, Clone, PartialEq)] +pub enum GenerationRequest { + Video(GenerateVideoArgs), + Image(GenerateImageArgs), + Audio(GenerateAudioArgs), + Upscale(UpscaleMediaArgs), +} + +impl GenerationRequest { + pub fn cost_authorized(&self) -> bool { + match self { + Self::Video(args) => args.cost_authorized == Some(true), + Self::Image(args) => args.cost_authorized == Some(true), + Self::Audio(args) => args.cost_authorized == Some(true), + Self::Upscale(args) => args.cost_authorized == Some(true), + } + } +} + +/// Immediate response from an accepted asynchronous generation submission. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GenerationSubmission { + pub job_id: String, + pub placeholder_asset_ids: Vec, + pub status: String, +} + +/// Host boundary for production generation. Implementations must return only +/// after the placeholder/job record is durably committed, and must continue +/// provider work off the synchronous MCP dispatch thread. +pub trait GenerationBridge: Send + Sync { + /// True only when managed authorization or at least one compatible BYOK + /// credential is currently usable. + fn can_generate(&self) -> bool; + + fn submit( + &self, + request: GenerationRequest, + cancel: &opentake_media::MediaCancelToken, + ) -> Result; +} + +/// A provider result downloaded into a private staging location. The store +/// validates and commits it into the project before reporting success. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DownloadedGenerationArtifact { + pub path: PathBuf, + pub media_type: String, + pub byte_size: u64, +} + +/// Downloads one provider result without exposing credentials or signed URL +/// details to the persistence layer's public error contract. +pub trait GenerationArtifactDownloader { + fn download(&self, asset_id: &str, url: &str) -> Result; +} + +/// Durable state boundary implemented by the desktop project runtime. +pub trait GenerationFinalizationStore { + /// Atomically claim a terminal-finalization lease. `false` means the job is + /// already complete or another callback currently owns the lease. + fn claim_terminal(&self, job_id: &str) -> Result; + + /// Release a failed terminal-finalization lease so restart recovery or a + /// duplicate provider callback can retry. Output operations below must be + /// idempotent because a prior attempt may already have committed a prefix. + fn release_terminal(&self, job_id: &str) -> Result<(), String>; + + /// Commit one staged artifact to the matching placeholder identity. + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String>; + + /// Persist one fixed, non-sensitive terminal failure code. + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String>; + + /// Persist the aggregate job terminal state after every placeholder has a + /// terminal record. + fn complete_job(&self, job_id: &str, succeeded: usize, failed: usize) -> Result<(), String>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GenerationFinalizationSummary { + pub claimed: bool, + pub succeeded: usize, + pub failed: usize, + pub ignored_result_urls: usize, +} + +/// Pair provider URLs to placeholders in order and terminalize every +/// placeholder exactly once. Missing, malformed, download-failed, and +/// commit-failed results become fixed failure codes; extra URLs are ignored. +pub fn finalize_terminal_outputs( + store: &dyn GenerationFinalizationStore, + downloader: &dyn GenerationArtifactDownloader, + job_id: &str, + placeholder_ids: &[String], + result_urls: &[String], +) -> Result { + if !store.claim_terminal(job_id)? { + return Ok(GenerationFinalizationSummary { + claimed: false, + succeeded: 0, + failed: 0, + ignored_result_urls: 0, + }); + } + + let attempt = (|| { + let mut succeeded = 0; + let mut failed = 0; + for (index, asset_id) in placeholder_ids.iter().enumerate() { + let Some(url) = result_urls.get(index) else { + store.fail_output(asset_id, "GENERATION_RESULT_MISSING")?; + failed += 1; + continue; + }; + if !is_accepted_result_url(url) { + store.fail_output(asset_id, "GENERATION_RESULT_URL_INVALID")?; + failed += 1; + continue; + } + let artifact = match downloader.download(asset_id, url) { + Ok(artifact) => artifact, + Err(error) if error == "GENERATION_CANCELLED" => return Err(error), + Err(_) => { + store.fail_output(asset_id, "GENERATION_DOWNLOAD_FAILED")?; + failed += 1; + continue; + } + }; + if store.finalize_output(asset_id, artifact).is_err() { + store.fail_output(asset_id, "GENERATION_FINALIZE_FAILED")?; + failed += 1; + continue; + } + succeeded += 1; + } + store.complete_job(job_id, succeeded, failed)?; + Ok::<_, String>((succeeded, failed)) + })(); + + let (succeeded, failed) = match attempt { + Ok(summary) => summary, + Err(error) => { + if let Err(release_error) = store.release_terminal(job_id) { + return Err(format!( + "generation finalization failed and lease release failed: {release_error}" + )); + } + return Err(error); + } + }; + + Ok(GenerationFinalizationSummary { + claimed: true, + succeeded, + failed, + ignored_result_urls: result_urls.len().saturating_sub(placeholder_ids.len()), + }) +} + +fn is_accepted_result_url(raw: &str) -> bool { + let Ok(url) = reqwest::Url::parse(raw) else { + return false; + }; + match url.scheme() { + "https" => { + url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.port().is_none_or(|port| port == 443) + } + "data" => { + let value = url.as_str(); + value.starts_with("data:image/") + || value.starts_with("data:audio/") + || value.starts_with("data:video/") + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::is_accepted_result_url; + + #[test] + fn provider_result_urls_are_https_or_bounded_media_data_urls() { + assert!(is_accepted_result_url("https://cdn.test/result.png")); + assert!(is_accepted_result_url("data:image/png;base64,AAAA")); + assert!(is_accepted_result_url("data:audio/mpeg;base64,AAAA")); + assert!(is_accepted_result_url("data:video/mp4;base64,AAAA")); + assert!(!is_accepted_result_url("http://cdn.test/result.png")); + assert!(!is_accepted_result_url( + "https://user:secret@cdn.test/result.png" + )); + assert!(!is_accepted_result_url("file:///tmp/result.png")); + assert!(!is_accepted_result_url("not-a-url")); + } +} diff --git a/crates/opentake-agent/src/mcp/mod.rs b/crates/opentake-agent/src/mcp/mod.rs index 23362344..98c01322 100644 --- a/crates/opentake-agent/src/mcp/mod.rs +++ b/crates/opentake-agent/src/mcp/mod.rs @@ -13,5 +13,6 @@ pub mod convert; pub mod core_handle; pub mod dispatch; pub mod gen_catalog; +pub mod generation; pub mod media_bridge; pub mod server; diff --git a/crates/opentake-agent/src/mcp/server.rs b/crates/opentake-agent/src/mcp/server.rs index d95af475..3afc2b06 100644 --- a/crates/opentake-agent/src/mcp/server.rs +++ b/crates/opentake-agent/src/mcp/server.rs @@ -27,11 +27,13 @@ use serde_json::{Map, Value}; use crate::mcp::convert::to_call_tool_result; use crate::mcp::core_handle::CoreHandle; use crate::mcp::dispatch::Dispatcher; +use crate::mcp::generation::GenerationBridge; use crate::mcp::media_bridge::{MediaBridge, MCP_REQUEST_BODY_MAX}; use crate::plugin::registry::PluginRegistry; use crate::prompt::assemble::assemble_system_prompt; use crate::tools::descriptions::{description, input_schema}; use crate::tools::errors::first_non_finite_json_number_path; +#[cfg(test)] use crate::tools::names::ToolName; use crate::tools::panic_boundary::with_redacted_dispatch_panic; @@ -69,22 +71,37 @@ impl McpServer { handle: Arc, registry: Arc>, bridge: Option>, + ) -> Self { + Self::with_bridges(handle, registry, bridge, None) + } + + pub fn with_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> Self { let instructions = registry .read() .map(|r| assemble_system_prompt(&r, "default")) .unwrap_or_default(); McpServer { - dispatcher: Arc::new(Dispatcher::with_bridge(handle, registry, bridge)), + dispatcher: Arc::new(Dispatcher::with_bridges( + handle, + registry, + bridge, + generation_bridge, + )), instructions, } } - /// All tool schemas (1:1 with [`ToolName::ALL`]). - fn tools() -> Vec { - ToolName::ALL - .iter() - .map(|&t| { + /// Tool schemas for capabilities live in this exact host session. + fn tools(&self) -> Vec { + self.dispatcher + .advertised_tools() + .into_iter() + .map(|t| { let obj = input_schema(t) .as_object() .cloned() @@ -121,7 +138,7 @@ impl ServerHandler for McpServer { _context: RequestContext, ) -> Result { Ok(ListToolsResult { - tools: Self::tools(), + tools: self.tools(), next_cursor: None, meta: None, }) @@ -453,6 +470,16 @@ pub fn build_router_with_bridge_for_port( registry: Arc>, bridge: Option>, expected_port: u16, +) -> axum::Router { + build_router_with_bridges_for_port(handle, registry, bridge, None, expected_port) +} + +pub fn build_router_with_bridges_for_port( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + expected_port: u16, ) -> axum::Router { use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use rmcp::transport::streamable_http_server::{ @@ -463,10 +490,11 @@ pub fn build_router_with_bridge_for_port( let service = StreamableHttpService::new( move || { - Ok(McpServer::with_bridge( + Ok(McpServer::with_bridges( handle.clone(), registry.clone(), bridge.clone(), + generation_bridge.clone(), )) }, Arc::new(LocalSessionManager::default()), @@ -508,6 +536,16 @@ pub async fn serve_with_bridge( handle: Arc, registry: Arc>, bridge: Option>, +) -> std::io::Result<()> { + serve_with_bridges(addr, handle, registry, bridge, None).await +} + +pub async fn serve_with_bridges( + addr: SocketAddr, + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> std::io::Result<()> { if !addr.ip().is_loopback() { return Err(std::io::Error::new( @@ -517,7 +555,13 @@ pub async fn serve_with_bridge( } let listener = tokio::net::TcpListener::bind(addr).await?; let bound_addr = listener.local_addr()?; - let router = build_router_with_bridge_for_port(handle, registry, bridge, bound_addr.port()); + let router = build_router_with_bridges_for_port( + handle, + registry, + bridge, + generation_bridge, + bound_addr.port(), + ); tracing::info!("MCP server listening on http://{bound_addr}/mcp"); axum::serve(listener, router).await } @@ -567,12 +611,10 @@ mod tests { #[test] fn lists_every_advertised_tool() { - assert_eq!(McpServer::tools().len(), ToolName::ALL.len()); + let server = server(); + assert_eq!(server.tools().len(), ToolName::ALL.len()); // Names round-trip to the wire names. - let names: Vec = McpServer::tools() - .iter() - .map(|t| t.name.to_string()) - .collect(); + let names: Vec = server.tools().iter().map(|t| t.name.to_string()).collect(); assert!(names.contains(&"add_clips".to_string())); assert!(names.contains(&"detect_beats".to_string())); assert!(names.contains(&"activate_workflow".to_string())); diff --git a/crates/opentake-agent/src/tools/args.rs b/crates/opentake-agent/src/tools/args.rs index 1a40d730..cc67a9f2 100644 --- a/crates/opentake-agent/src/tools/args.rs +++ b/crates/opentake-agent/src/tools/args.rs @@ -682,6 +682,7 @@ impl ToolArgs for TightenSilencesArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateVideoArgs { + pub cost_authorized: Option, pub prompt: String, pub name: Option, pub model: Option, @@ -700,6 +701,7 @@ pub struct GenerateVideoArgs { impl ToolArgs for GenerateVideoArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "duration", @@ -720,23 +722,27 @@ impl ToolArgs for GenerateVideoArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateImageArgs { + pub cost_authorized: Option, pub prompt: String, pub name: Option, pub model: Option, pub aspect_ratio: Option, pub resolution: Option, pub quality: Option, + pub num_images: Option, pub reference_media_refs: Option>, pub folder_id: Option, } impl ToolArgs for GenerateImageArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "aspectRatio", "resolution", "quality", + "numImages", "referenceMediaRefs", "folderId", ]; @@ -746,6 +752,7 @@ impl ToolArgs for GenerateImageArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateAudioArgs { + pub cost_authorized: Option, pub prompt: Option, pub name: Option, pub model: Option, @@ -762,6 +769,7 @@ pub struct GenerateAudioArgs { impl ToolArgs for GenerateAudioArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "voice", @@ -780,12 +788,14 @@ impl ToolArgs for GenerateAudioArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct UpscaleMediaArgs { + pub cost_authorized: Option, pub media_ref: String, pub model: Option, pub source_clip_id: Option, } impl ToolArgs for UpscaleMediaArgs { - const ALLOWED_KEYS: &'static [&'static str] = &["mediaRef", "model", "sourceClipId"]; + const ALLOWED_KEYS: &'static [&'static str] = + &["costAuthorized", "mediaRef", "model", "sourceClipId"]; } // --- import_media --- diff --git a/crates/opentake-agent/src/tools/descriptions.rs b/crates/opentake-agent/src/tools/descriptions.rs index 3a5e1a5a..74042e7f 100644 --- a/crates/opentake-agent/src/tools/descriptions.rs +++ b/crates/opentake-agent/src/tools/descriptions.rs @@ -402,6 +402,7 @@ pub fn input_schema(tool: ToolName) -> Value { ToolName::GenerateVideo => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Text description of the video to generate"}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID (e.g. 'veo3.1-fast'). Use list_models to see options. Defaults to first available model."}, @@ -417,25 +418,28 @@ pub fn input_schema(tool: ToolName) -> Value { "referenceAudioMediaRefs": {"type": "array", "items": {"type": "string"}, "description": "Media asset IDs of audio references (Seedance only). Refer to them as @Audio1, @Audio2. See maxReferenceAudios and maxCombinedAudioRefSeconds."}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &["prompt"], + &["costAuthorized", "prompt"], ), ToolName::GenerateImage => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Text description of the image to generate"}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID (e.g. 'nano-banana-pro'). Use list_models to see options. Defaults to first available model."}, "aspectRatio": {"type": "string", "description": "Aspect ratio (e.g. '16:9', '9:16')"}, "resolution": {"type": "string", "description": "Resolution (e.g. '2K', '4K')"}, "quality": {"type": "string", "description": "Image quality (e.g. 'low', 'medium', 'high'). Only supported by some models — see list_models."}, + "numImages": {"type": "integer", "minimum": 1, "maximum": 4, "description": "Number of ordered image results to generate. Defaults to 1 and is capped at 4."}, "referenceMediaRefs": {"type": "array", "items": {"type": "string"}, "description": "Media asset IDs to use as reference images"}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &["prompt"], + &["costAuthorized", "prompt"], ), ToolName::GenerateAudio => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Required for TTS (the text to speak) and text-to-music (style/mood/genre; MiniMax needs ≥10 chars). For Lyria 3 Pro, include lyrics, tempo, language, and vocal style directly in the prompt. Optional style guide for video-to-music models."}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID. Use list_models with type='audio' to see options and their 'inputs'. Defaults to the first model."}, @@ -449,16 +453,17 @@ pub fn input_schema(tool: ToolName) -> Value { "videoSourceMediaRef": {"type": "string", "description": "Video-to-audio models only. Score this existing video asset instead of a timeline span. Mutually exclusive with the videoSource frames."}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &[], + &["costAuthorized"], ), ToolName::UpscaleMedia => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid upscale request in this conversation."}, "mediaRef": {"type": "string", "description": "ID of the video or image asset to upscale"}, "model": {"type": "string", "description": "Upscaler model ID (e.g. 'bytedance-upscaler', 'seedvr-image-upscaler'). Defaults to the first model that supports the asset's type."}, "sourceClipId": {"type": "string", "description": "Optional. Video clip id (from get_timeline) referencing mediaRef. When set and the clip is trimmed, only the clip's visible range is upscaled, not the full source."} }), - &["mediaRef"], + &["costAuthorized", "mediaRef"], ), ToolName::ImportMedia => object( diff --git a/crates/opentake-agent/src/tools/names.rs b/crates/opentake-agent/src/tools/names.rs index 5fee8fbf..52c48d39 100644 --- a/crates/opentake-agent/src/tools/names.rs +++ b/crates/opentake-agent/src/tools/names.rs @@ -157,6 +157,15 @@ impl ToolName { ToolName::ApplyEffect, ]; + /// Provider-backed tools appended to a host catalog only while its live + /// generation bridge reports usable authorization. + pub const GENERATION: [ToolName; 4] = [ + ToolName::GenerateVideo, + ToolName::GenerateImage, + ToolName::GenerateAudio, + ToolName::UpscaleMedia, + ]; + /// Every recognized schema/wire name, including capabilities deliberately /// hidden from discovery until a real backend exists. Keeping this set lets /// strict argument validation and compatibility tests cover future tools diff --git a/crates/opentake-agent/tests/generation_dispatch.rs b/crates/opentake-agent/tests/generation_dispatch.rs new file mode 100644 index 00000000..46c9e9af --- /dev/null +++ b/crates/opentake-agent/tests/generation_dispatch.rs @@ -0,0 +1,285 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use opentake_agent::mcp::core_handle::AppCoreHandle; +use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::mcp::generation::{ + finalize_terminal_outputs, DownloadedGenerationArtifact, GenerationArtifactDownloader, + GenerationBridge, GenerationFinalizationStore, GenerationRequest, GenerationSubmission, +}; +use opentake_agent::plugin::registry::PluginRegistry; +use opentake_core::AppCore; +use serde_json::json; + +#[derive(Default)] +struct RecordingStore { + claimed: Mutex>, + completed: Mutex>, + finalized: Mutex>, + failed: Mutex>, + completions: Mutex>, + output_write_failures_remaining: Mutex, +} + +impl GenerationFinalizationStore for RecordingStore { + fn claim_terminal(&self, job_id: &str) -> Result { + if self.completed.lock().unwrap().contains(job_id) { + return Ok(false); + } + Ok(self.claimed.lock().unwrap().insert(job_id.to_string())) + } + + fn release_terminal(&self, job_id: &str) -> Result<(), String> { + self.claimed.lock().unwrap().remove(job_id); + Ok(()) + } + + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String> { + let mut failures = self.output_write_failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("transient manifest write failure".to_string()); + } + self.finalized + .lock() + .unwrap() + .insert(asset_id.to_string(), artifact.path); + Ok(()) + } + + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String> { + let mut failures = self.output_write_failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("transient manifest write failure".to_string()); + } + self.failed + .lock() + .unwrap() + .insert(asset_id.to_string(), code.to_string()); + Ok(()) + } + + fn complete_job(&self, job_id: &str, succeeded: usize, failed: usize) -> Result<(), String> { + if !self.completed.lock().unwrap().insert(job_id.to_string()) { + return Ok(()); + } + self.completions + .lock() + .unwrap() + .push((job_id.to_string(), succeeded, failed)); + Ok(()) + } +} + +struct FixtureDownloader; + +impl GenerationArtifactDownloader for FixtureDownloader { + fn download(&self, asset_id: &str, url: &str) -> Result { + if url.contains("download-fails") { + return Err("provider download failed with private detail".to_string()); + } + Ok(DownloadedGenerationArtifact { + path: PathBuf::from(format!("/fixture/{asset_id}.bin")), + media_type: "application/octet-stream".to_string(), + byte_size: 7, + }) + } +} + +struct RecordingGenerationBridge { + available: AtomicBool, + submissions: AtomicUsize, +} + +impl RecordingGenerationBridge { + fn new(available: bool) -> Self { + Self { + available: AtomicBool::new(available), + submissions: AtomicUsize::new(0), + } + } +} + +impl GenerationBridge for RecordingGenerationBridge { + fn can_generate(&self) -> bool { + self.available.load(Ordering::Acquire) + } + + fn submit( + &self, + _request: GenerationRequest, + _cancel: &opentake_media::MediaCancelToken, + ) -> Result { + self.submissions.fetch_add(1, Ordering::AcqRel); + Ok(GenerationSubmission { + job_id: "job-dispatch".to_string(), + placeholder_asset_ids: vec!["asset-placeholder".to_string()], + status: "queued".to_string(), + }) + } +} + +fn dispatcher_with_generation_bridge(bridge: Arc) -> Dispatcher { + Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(AppCore::new())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ) +} + +#[test] +fn placeholder_persist_finalize_all_results_and_failures() { + let store = RecordingStore::default(); + let summary = finalize_terminal_outputs( + &store, + &FixtureDownloader, + "job-1", + &[ + "asset-a".into(), + "asset-b".into(), + "asset-c".into(), + "asset-d".into(), + ], + &[ + "https://results.test/a.png".into(), + "https://results.test/download-fails.png".into(), + "not-a-result-url".into(), + "https://results.test/extra.png".into(), + "https://results.test/ignored-extra.png".into(), + ], + ) + .expect("terminal outputs are recorded"); + + assert!(summary.claimed); + assert_eq!(summary.succeeded, 2); + assert_eq!(summary.failed, 2); + assert_eq!(summary.ignored_result_urls, 1); + assert_eq!( + store + .finalized + .lock() + .unwrap() + .keys() + .cloned() + .collect::>(), + vec!["asset-a", "asset-d"] + ); + assert_eq!( + store.failed.lock().unwrap().clone(), + BTreeMap::from([ + ( + "asset-b".to_string(), + "GENERATION_DOWNLOAD_FAILED".to_string() + ), + ( + "asset-c".to_string(), + "GENERATION_RESULT_URL_INVALID".to_string() + ), + ]) + ); + assert_eq!( + store.completions.lock().unwrap().as_slice(), + &[("job-1".to_string(), 2, 2)] + ); +} + +#[test] +fn placeholder_persists_and_every_terminal_result_finalizes_once() { + let store = RecordingStore::default(); + let placeholders = vec!["asset-a".to_string(), "asset-b".to_string()]; + let urls = vec!["https://results.test/a.png".to_string()]; + + let first = + finalize_terminal_outputs(&store, &FixtureDownloader, "job-once", &placeholders, &urls) + .expect("first terminal callback succeeds"); + let duplicate = + finalize_terminal_outputs(&store, &FixtureDownloader, "job-once", &placeholders, &urls) + .expect("duplicate terminal callback is idempotent"); + + assert!(first.claimed); + assert_eq!((first.succeeded, first.failed), (1, 1)); + assert!(!duplicate.claimed); + assert_eq!((duplicate.succeeded, duplicate.failed), (0, 0)); + assert_eq!(store.finalized.lock().unwrap().len(), 1); + assert_eq!( + store + .failed + .lock() + .unwrap() + .get("asset-b") + .map(String::as_str), + Some("GENERATION_RESULT_MISSING") + ); + assert_eq!(store.completions.lock().unwrap().len(), 1); + + let retryable = RecordingStore { + output_write_failures_remaining: Mutex::new(2), + ..Default::default() + }; + let first_attempt = finalize_terminal_outputs( + &retryable, + &FixtureDownloader, + "job-retry", + &["asset-retry".to_string()], + &["https://results.test/retry.png".to_string()], + ); + assert!(first_attempt.is_err()); + assert!(retryable.claimed.lock().unwrap().is_empty()); + + let recovered = finalize_terminal_outputs( + &retryable, + &FixtureDownloader, + "job-retry", + &["asset-retry".to_string()], + &["https://results.test/retry.png".to_string()], + ) + .expect("restart recovery can reacquire the released terminal lease"); + assert!(recovered.claimed); + assert_eq!((recovered.succeeded, recovered.failed), (1, 0)); + assert_eq!(retryable.completions.lock().unwrap().len(), 1); +} + +#[test] +fn configured_capability_and_cost_authorization_gate_dispatch() { + let bridge = Arc::new(RecordingGenerationBridge::new(false)); + let dispatcher = dispatcher_with_generation_bridge(bridge.clone()); + + let timeline = dispatcher.dispatch("get_timeline", json!({})); + assert!(!timeline.is_error); + assert!(timeline.text_joined().contains("\"canGenerate\":false")); + + let unavailable = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": true, "prompt": "fixture"}), + ); + assert!(unavailable.is_error); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 0); + + bridge.available.store(true, Ordering::Release); + let timeline = dispatcher.dispatch("get_timeline", json!({})); + assert!(timeline.text_joined().contains("\"canGenerate\":true")); + + let unauthorized = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": false, "prompt": "fixture"}), + ); + assert!(unauthorized.is_error); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 0); + + let accepted = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": true, "prompt": "fixture"}), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + assert!(accepted.text_joined().contains("job-dispatch")); + assert!(accepted.text_joined().contains("asset-placeholder")); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 1); +} diff --git a/crates/opentake-core/Cargo.toml b/crates/opentake-core/Cargo.toml index b8eee2ca..446fd06a 100644 --- a/crates/opentake-core/Cargo.toml +++ b/crates/opentake-core/Cargo.toml @@ -14,3 +14,6 @@ opentake-ops = { workspace = true } opentake-project = { workspace = true } thiserror = "2" same-file = "1.0.6" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/opentake-core/src/core.rs b/crates/opentake-core/src/core.rs index fc3fc772..cfe4f0b7 100644 --- a/crates/opentake-core/src/core.rs +++ b/crates/opentake-core/src/core.rs @@ -40,7 +40,10 @@ use same_file::Handle; use crate::deps::CoreDeps; use crate::error::{CoreError, Result}; use crate::events::{CoreEvent, EventBus, SubscriptionId}; -use crate::session::{EditorSession, ProbedMedia}; +use crate::session::{ + EditorSession, GenerationJobCommit, GenerationStateUpdate, PreparedGenerationJob, + PreparedGenerationOutput, ProbedMedia, +}; type ProjectIdentityTransitionListener = Arc; @@ -676,6 +679,152 @@ impl AppCore { self.lock().editor.generation_log().clone() } + /// Persist placeholder assets and the queued audit event before a paid + /// provider request is submitted. No ids are returned unless the project + /// snapshot is durable. + pub fn begin_generation_job_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + plan: PreparedGenerationJob, + ) -> Result { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.begin_generation_job(plan, ids), + ) + } + + pub fn update_generation_job_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + job_id: &str, + update: GenerationStateUpdate, + ) -> Result { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.update_generation_job(job_id, update, ids), + ) + } + + pub fn finalize_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + output: PreparedGenerationOutput, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.finalize_generation_output(output, ids), + ) + } + + /// Finalize a generated output and stream its media bytes into the same + /// complete-bundle publication as the manifest and generation log. + pub fn finalize_generation_output_with_media_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + output: PreparedGenerationOutput, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result<()> { + let expected_relative_path = format!("media/{media_leaf}"); + if output.relative_path != expected_relative_path { + return Err(CoreError::Media( + "generation output path does not match its media leaf".to_string(), + )); + } + self.persist_generation_mutation_using( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.finalize_generation_output(output, ids), + |editor| editor.save_generation_state_with_media(media_leaf, media_byte_size, media), + ) + } + + pub fn fail_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + asset_id: &str, + error_code: &str, + created_at: Option, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.fail_generation_output(asset_id, error_code, created_at, ids), + ) + } + + pub fn cancel_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + asset_id: &str, + created_at: Option, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.cancel_generation_output(asset_id, created_at, ids), + ) + } + + fn persist_generation_mutation( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + mutate: impl FnOnce(&mut EditorSession, &dyn IdGen) -> Result, + ) -> Result { + self.persist_generation_mutation_using( + expected_project_epoch, + expected_project_dir, + mutate, + |editor| editor.save_generation_state(), + ) + } + + fn persist_generation_mutation_using( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + mutate: impl FnOnce(&mut EditorSession, &dyn IdGen) -> Result, + persist: impl FnOnce(&mut EditorSession) -> Result, + ) -> Result { + let (value, count, written) = { + let mut session = self.lock(); + ensure_project_identity(&session, expected_project_epoch, expected_project_dir)?; + let checkpoint = session.editor.checkpoint_generation_state(); + let result = (|| { + let value = mutate(&mut session.editor, self.ids.as_ref())?; + let written = persist(&mut session.editor)?; + Ok((value, written)) + })(); + match result { + Ok((value, written)) => (value, session.editor.media().entries.len(), written), + Err(error) => { + session.editor.restore_generation_state(checkpoint); + return Err(error); + } + } + }; + self.events.emit(&CoreEvent::MediaChanged { + project_epoch: expected_project_epoch, + count, + }); + self.events.emit(&CoreEvent::ProjectSaved { + path: written.to_string_lossy().into_owned(), + project_epoch: expected_project_epoch, + }); + Ok(value) + } + /// The open project's `.opentake` bundle directory, or `None` for an unsaved /// project. Needed to resolve [`MediaSource::Project`](opentake_domain::MediaSource) /// relative paths to on-disk files (preview/composite read the original media). diff --git a/crates/opentake-core/src/lib.rs b/crates/opentake-core/src/lib.rs index ad4632b9..070c06eb 100644 --- a/crates/opentake-core/src/lib.rs +++ b/crates/opentake-core/src/lib.rs @@ -49,7 +49,8 @@ pub use crate::core::{ ProjectRevision, ProjectRuntimeSnapshot, TimelineSnapshot, }; pub use session::{ - importable_clip_type, EditorSession, ProbedMedia, SUPPORTED_AUDIO_EXTENSIONS, + importable_clip_type, EditorSession, GenerationJobCommit, GenerationStateUpdate, + PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, SUPPORTED_AUDIO_EXTENSIONS, SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, }; diff --git a/crates/opentake-core/src/session.rs b/crates/opentake-core/src/session.rs index 7bcc2113..ced3d523 100644 --- a/crates/opentake-core/src/session.rs +++ b/crates/opentake-core/src/session.rs @@ -34,10 +34,15 @@ use std::path::{Path, PathBuf}; -use opentake_domain::{ClipType, MediaAsset, MediaManifest, MediaManifestEntry, Timeline}; +use opentake_domain::{ + ClipType, GenerationInput, GenerationJobStatus, MediaAsset, MediaManifest, MediaManifestEntry, + MediaSource, Timeline, +}; use opentake_ops::command::{self, EditCommand, EditResult}; use opentake_ops::{EditorState, IdGen}; -use opentake_project::{GenerationLog, Project, ProjectCompatibility, ProjectRoot}; +use opentake_project::{ + GenerationLog, GenerationLogEntry, Project, ProjectCompatibility, ProjectRoot, +}; use same_file::Handle; use crate::error::{CoreError, Result}; @@ -65,6 +70,55 @@ pub struct ProbedMedia { pub has_audio: bool, } +/// Validated provider-neutral generation job prepared by the Agent/Tauri host. +/// Credentials, signed URLs, and provider diagnostics are deliberately absent. +#[derive(Clone, Debug)] +pub struct PreparedGenerationJob { + pub name: String, + pub kind: ClipType, + pub folder_id: Option, + pub provider: String, + pub input: GenerationInput, + pub output_count: usize, + pub source_asset_id: Option, + pub source_clip_id: Option, + pub estimated_cost_credits: Option, + pub created_at: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GenerationJobCommit { + pub job_id: String, + pub placeholder_asset_ids: Vec, +} + +/// One durable lifecycle update. Provider messages are reduced to a fixed +/// application-owned `error_code` before reaching this boundary. +#[derive(Clone, Debug)] +pub struct GenerationStateUpdate { + pub status: GenerationJobStatus, + pub progress: Option, + pub error_code: Option, + pub provider_job_id: Option, + pub cost_credits: Option, + pub created_at: Option, +} + +#[derive(Clone, Debug)] +pub struct PreparedGenerationOutput { + pub asset_id: String, + pub relative_path: String, + pub probe: ProbedMedia, + pub created_at: Option, +} + +#[derive(Clone)] +pub(crate) struct GenerationStateCheckpoint { + manifest: MediaManifest, + log: GenerationLog, + component_present: bool, +} + /// File extensions the importer accepts, grouped by the [`ClipType`] they map to. /// /// Upstream's picker (`MediaTab.swift:754` — `allowedContentTypes = [.movie, @@ -105,6 +159,104 @@ pub fn importable_clip_type(path: &Path) -> Option { } } +fn safe_provider_prefix(value: &str) -> bool { + !value.is_empty() + && value.len() <= 32 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +fn safe_generation_error_code(value: &str) -> bool { + !value.is_empty() + && value.len() <= 80 + && value + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn validate_generation_update(update: &GenerationStateUpdate) -> Result<()> { + if update.cost_credits.is_some_and(|credits| credits < 0) { + return Err(CoreError::Media( + "generation cost must not be negative".to_string(), + )); + } + if let Some(progress) = update.progress { + if !progress.is_finite() || !(0.0..=1.0).contains(&progress) { + return Err(CoreError::Media( + "generation progress must be finite and between 0 and 1".to_string(), + )); + } + } + if let Some(error_code) = update.error_code.as_deref() { + if !safe_generation_error_code(error_code) { + return Err(CoreError::Media( + "generation failure code is invalid".to_string(), + )); + } + } + if update.status == GenerationJobStatus::Failed && update.error_code.is_none() { + return Err(CoreError::Media( + "failed generation status requires an error code".to_string(), + )); + } + if update.status != GenerationJobStatus::Failed && update.error_code.is_some() { + return Err(CoreError::Media( + "generation error code is only valid for failed status".to_string(), + )); + } + if let Some(provider_job_id) = update.provider_job_id.as_deref() { + if provider_job_id.is_empty() + || provider_job_id.len() > 512 + || provider_job_id.chars().any(char::is_control) + || provider_job_id.contains("://") + { + return Err(CoreError::Media( + "provider job identity is invalid".to_string(), + )); + } + } + Ok(()) +} + +fn valid_generation_transition( + current: Option, + next: GenerationJobStatus, +) -> bool { + use GenerationJobStatus as Status; + match (current, next) { + (None, Status::Queued) => true, + (Some(current), next) if current == next => true, + (Some(Status::Queued), Status::Generating | Status::Failed | Status::Cancelled) => true, + (Some(Status::Generating), Status::Downloading | Status::Failed | Status::Cancelled) => { + true + } + ( + Some(Status::Downloading), + Status::Finalizing | Status::Ready | Status::Failed | Status::Cancelled, + ) => true, + (Some(Status::Finalizing), Status::Ready | Status::Failed | Status::Cancelled) => true, + (Some(Status::Failed | Status::Cancelled), Status::Queued) => true, + (Some(Status::Ready), Status::Ready) => true, + _ => false, + } +} + +fn validate_project_media_relative_path(value: &str) -> Result<()> { + let path = Path::new(value); + let mut components = path.components(); + if components.next() != Some(std::path::Component::Normal("media".as_ref())) + || components.clone().next().is_none() + || path.is_absolute() + || components.any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err(CoreError::Media( + "generated output path must be a safe media-relative path".to_string(), + )); + } + Ok(()) +} + /// The open document plus its project-level metadata. pub struct EditorSession { /// Authoritative editable state: timeline, manifest, undo/redo, version. @@ -534,6 +686,427 @@ impl EditorSession { &self.generation_log } + /// Create durable manifest placeholders and append the corresponding + /// queued audit events. The caller persists the returned in-memory state in + /// the same critical section before exposing the placeholder ids. + pub(crate) fn begin_generation_job( + &mut self, + mut plan: PreparedGenerationJob, + ids: &dyn IdGen, + ) -> Result { + self.ensure_mutable()?; + if self.project_dir.is_none() { + return Err(CoreError::NoProjectOpen); + } + if !(1..=4).contains(&plan.output_count) { + return Err(CoreError::Media( + "generation output count must be between 1 and 4".to_string(), + )); + } + if plan.input.model.trim().is_empty() || plan.provider.trim().is_empty() { + return Err(CoreError::Media( + "generation model and provider are required".to_string(), + )); + } + if !safe_provider_prefix(&plan.provider) { + return Err(CoreError::Media( + "generation provider prefix is invalid".to_string(), + )); + } + if let Some(folder_id) = plan.folder_id.as_deref() { + if !self + .state + .manifest + .folders + .iter() + .any(|folder| folder.id == folder_id) + { + return Err(CoreError::Media(format!( + "generation folder does not exist: {folder_id}" + ))); + } + } + if let Some(source_asset_id) = plan.source_asset_id.as_deref() { + if !self + .state + .manifest + .entries + .iter() + .any(|entry| entry.id == source_asset_id) + { + return Err(CoreError::Media(format!( + "generation source asset does not exist: {source_asset_id}" + ))); + } + } + + let job_id = ids.next_id(); + plan.input.job_id = Some(job_id.clone()); + plan.input.provider = Some(plan.provider.clone()); + plan.input.provider_job_id = None; + plan.input.status = Some(GenerationJobStatus::Queued); + plan.input.progress = Some(0.0); + plan.input.error_code = None; + plan.input.source_asset_id = plan.source_asset_id.clone(); + plan.input.source_clip_id = plan.source_clip_id.clone(); + plan.input.estimated_cost_credits = plan.estimated_cost_credits; + plan.input.created_at = plan.created_at; + + let mut placeholder_asset_ids = Vec::with_capacity(plan.output_count); + for output_index in 0..plan.output_count { + let asset_id = ids.next_id(); + let mut input = plan.input.clone(); + input.output_index = Some(output_index); + let display_name = if plan.output_count == 1 { + plan.name.clone() + } else { + format!("{} {}", plan.name, output_index + 1) + }; + self.state.manifest.entries.push(MediaManifestEntry { + id: asset_id.clone(), + name: display_name, + kind: plan.kind, + source: MediaSource::Project { + relative_path: format!("media/{asset_id}.pending"), + }, + duration: (input.duration.max(0)) as f64, + generation_input: Some(input.clone()), + source_width: None, + source_height: None, + source_fps: None, + has_audio: Some( + plan.kind == ClipType::Audio + || (plan.kind == ClipType::Video && input.generate_audio.unwrap_or(true)), + ), + folder_id: plan.folder_id.clone(), + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + self.generation_log + .entries + .push(GenerationLogEntry::job_event( + ids.next_id(), + job_id.clone(), + input.model.clone(), + None, + plan.provider.clone(), + None, + asset_id.clone(), + GenerationJobStatus::Queued, + Some(0.0), + None, + plan.created_at, + plan.source_asset_id.clone(), + plan.source_clip_id.clone(), + )); + placeholder_asset_ids.push(asset_id); + } + self.generation_log_component_present = true; + Ok(GenerationJobCommit { + job_id, + placeholder_asset_ids, + }) + } + + pub(crate) fn update_generation_job( + &mut self, + job_id: &str, + update: GenerationStateUpdate, + ids: &dyn IdGen, + ) -> Result { + self.ensure_mutable()?; + validate_generation_update(&update)?; + let mut events = Vec::new(); + for entry in &mut self.state.manifest.entries { + let Some(input) = entry.generation_input.as_mut() else { + continue; + }; + if input.job_id.as_deref() != Some(job_id) { + continue; + } + if !valid_generation_transition(input.status, update.status) { + return Err(CoreError::Media(format!( + "invalid generation transition from {:?} to {:?}", + input.status, update.status + ))); + } + input.status = Some(update.status); + input.progress = update.progress; + input.error_code = update.error_code.clone(); + if update.provider_job_id.is_some() { + input.provider_job_id = update.provider_job_id.clone(); + } + events.push((entry.id.clone(), input.clone())); + } + if events.is_empty() { + return Err(CoreError::Media(format!( + "generation job does not exist: {job_id}" + ))); + } + for (event_index, (asset_id, input)) in events.iter().enumerate() { + self.append_generation_event( + ids, + asset_id, + input, + update.status, + update.progress, + update.error_code.clone(), + (event_index == 0).then_some(update.cost_credits).flatten(), + update.created_at, + ); + } + Ok(events.len()) + } + + pub(crate) fn finalize_generation_output( + &mut self, + output: PreparedGenerationOutput, + ids: &dyn IdGen, + ) -> Result<()> { + self.ensure_mutable()?; + validate_project_media_relative_path(&output.relative_path)?; + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == output.asset_id) + .ok_or_else(|| { + CoreError::Media(format!( + "generation placeholder does not exist: {}", + output.asset_id + )) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if !valid_generation_transition(input.status, GenerationJobStatus::Ready) { + return Err(CoreError::Media(format!( + "generation placeholder cannot finalize from {:?}", + input.status + ))); + } + entry.source = MediaSource::Project { + relative_path: output.relative_path, + }; + entry.duration = output.probe.duration_secs; + entry.source_width = output.probe.width; + entry.source_height = output.probe.height; + entry.source_fps = output.probe.fps; + entry.has_audio = Some(output.probe.has_audio); + input.status = Some(GenerationJobStatus::Ready); + input.progress = Some(1.0); + input.error_code = None; + let asset_id = entry.id.clone(); + let input = input.clone(); + self.append_generation_event( + ids, + &asset_id, + &input, + GenerationJobStatus::Ready, + Some(1.0), + None, + None, + output.created_at, + ); + Ok(()) + } + + pub(crate) fn fail_generation_output( + &mut self, + asset_id: &str, + error_code: &str, + created_at: Option, + ids: &dyn IdGen, + ) -> Result<()> { + if !safe_generation_error_code(error_code) { + return Err(CoreError::Media( + "generation failure code is invalid".to_string(), + )); + } + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| { + CoreError::Media(format!("generation placeholder does not exist: {asset_id}")) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if matches!(input.status, Some(GenerationJobStatus::Ready)) { + return Ok(()); + } + input.status = Some(GenerationJobStatus::Failed); + input.progress = None; + input.error_code = Some(error_code.to_string()); + let input = input.clone(); + self.append_generation_event( + ids, + asset_id, + &input, + GenerationJobStatus::Failed, + None, + Some(error_code.to_string()), + None, + created_at, + ); + Ok(()) + } + + pub(crate) fn cancel_generation_output( + &mut self, + asset_id: &str, + created_at: Option, + ids: &dyn IdGen, + ) -> Result<()> { + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| { + CoreError::Media(format!("generation placeholder does not exist: {asset_id}")) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if matches!( + input.status, + Some( + GenerationJobStatus::Ready + | GenerationJobStatus::Failed + | GenerationJobStatus::Cancelled + ) + ) { + return Ok(()); + } + input.status = Some(GenerationJobStatus::Cancelled); + input.progress = None; + input.error_code = None; + let input = input.clone(); + self.append_generation_event( + ids, + asset_id, + &input, + GenerationJobStatus::Cancelled, + None, + None, + None, + created_at, + ); + Ok(()) + } + + pub(crate) fn save_generation_state(&mut self) -> Result { + self.ensure_mutable()?; + let target = self.project_dir.clone().ok_or(CoreError::NoProjectOpen)?; + let mut project = + Project::new_with_compatibility(target.clone(), self.compatibility.clone()); + project.timeline = self.state.timeline.clone(); + project.manifest = self.state.manifest.clone(); + project.generation_log = Some(self.generation_log.clone()); + // Generation spans media.json + generation-log.json. Publish a complete + // sibling bundle so both become visible at one rename commit point. + // The source root carries media/chat/thumbnail into the fresh stage. + let source_root = self.project_root.take().ok_or(CoreError::NoProjectOpen)?; + let new_root = match project.publish_complete_replacing_root(&target, source_root) { + Ok(root) => root, + Err(error) => { + // A pre-commit failure restores the original target; recover + // retained authority when possible while preserving the exact + // publication error for the caller. Post-commit ambiguity stays + // fail-closed if the target cannot be reopened. + self.project_root = ProjectRoot::open(&target).ok(); + return Err(error.into()); + } + }; + self.project_root = Some(new_root); + self.generation_log_component_present = true; + Ok(target) + } + + pub(crate) fn save_generation_state_with_media( + &mut self, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result { + self.ensure_mutable()?; + let target = self.project_dir.clone().ok_or(CoreError::NoProjectOpen)?; + let mut project = + Project::new_with_compatibility(target.clone(), self.compatibility.clone()); + project.timeline = self.state.timeline.clone(); + project.manifest = self.state.manifest.clone(); + project.generation_log = Some(self.generation_log.clone()); + let source_root = self.project_root.take().ok_or(CoreError::NoProjectOpen)?; + let new_root = match project.publish_complete_replacing_root_with_media( + &target, + source_root, + media_leaf, + media_byte_size, + media, + ) { + Ok(root) => root, + Err(error) => { + self.project_root = ProjectRoot::open(&target).ok(); + return Err(error.into()); + } + }; + self.project_root = Some(new_root); + self.generation_log_component_present = true; + Ok(target) + } + + pub(crate) fn checkpoint_generation_state(&self) -> GenerationStateCheckpoint { + GenerationStateCheckpoint { + manifest: self.state.manifest.clone(), + log: self.generation_log.clone(), + component_present: self.generation_log_component_present, + } + } + + pub(crate) fn restore_generation_state(&mut self, checkpoint: GenerationStateCheckpoint) { + self.state.manifest = checkpoint.manifest; + self.generation_log = checkpoint.log; + self.generation_log_component_present = checkpoint.component_present; + } + + #[allow(clippy::too_many_arguments)] + fn append_generation_event( + &mut self, + ids: &dyn IdGen, + asset_id: &str, + input: &GenerationInput, + status: GenerationJobStatus, + progress: Option, + error_code: Option, + cost_credits: Option, + created_at: Option, + ) { + self.generation_log + .entries + .push(GenerationLogEntry::job_event( + ids.next_id(), + input.job_id.clone().unwrap_or_default(), + input.model.clone(), + cost_credits, + input.provider.clone().unwrap_or_default(), + input.provider_job_id.clone(), + asset_id.to_string(), + status, + progress, + error_code, + created_at, + input.source_asset_id.clone(), + input.source_clip_id.clone(), + )); + self.generation_log_component_present = true; + } + /// Compatibility state inherited from the opened project. pub fn compatibility(&self) -> &ProjectCompatibility { &self.compatibility diff --git a/crates/opentake-core/tests/generation_persistence.rs b/crates/opentake-core/tests/generation_persistence.rs new file mode 100644 index 00000000..94bf0eec --- /dev/null +++ b/crates/opentake-core/tests/generation_persistence.rs @@ -0,0 +1,343 @@ +use std::fs; + +use opentake_core::{ + AppCore, GenerationStateUpdate, PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, +}; +use opentake_domain::{ + ClipType, GenerationInput, GenerationJobStatus, MediaManifestEntry, MediaSource, +}; +use opentake_project::Project; + +fn saved_project() -> (tempfile::TempDir, std::path::PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Generation.opentake"); + let mut project = Project::new(&bundle); + project.manifest.entries.push(MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(4), + source_height: Some(3), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + fs::create_dir_all(bundle.join("media")).unwrap(); + fs::write(bundle.join("media/source.png"), b"source-bytes").unwrap(); + fs::write(bundle.join("thumbnail.jpg"), b"cover").unwrap(); + (temp, bundle) +} + +fn upscale_plan() -> PreparedGenerationJob { + PreparedGenerationJob { + name: "Upscaled source".to_string(), + kind: ClipType::Image, + folder_id: None, + provider: "fal".to_string(), + input: GenerationInput { + prompt: String::new(), + model: "fal:fixture-upscaler".to_string(), + duration: 0, + aspect_ratio: String::new(), + ..Default::default() + }, + output_count: 1, + source_asset_id: Some("source-image".to_string()), + source_clip_id: Some("source-clip".to_string()), + estimated_cost_credits: Some(12), + created_at: Some(800_000_000.0), + } +} + +fn update(status: GenerationJobStatus, progress: Option) -> GenerationStateUpdate { + GenerationStateUpdate { + status, + progress, + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: Some(800_000_001.0), + } +} + +#[test] +fn placeholders_job_events_and_finalized_output_survive_restart() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, upscale_plan()) + .unwrap(); + assert_eq!(committed.placeholder_asset_ids.len(), 1); + let asset_id = committed.placeholder_asset_ids[0].clone(); + + let queued = Project::open(&bundle).unwrap(); + let placeholder = queued + .manifest + .entries + .iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + let input = placeholder.generation_input.as_ref().unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Queued)); + assert_eq!(input.source_asset_id.as_deref(), Some("source-image")); + assert_eq!(input.source_clip_id.as_deref(), Some("source-clip")); + assert_eq!(input.estimated_cost_credits, Some(12)); + assert_eq!(fs::read(bundle.join("thumbnail.jpg")).unwrap(), b"cover"); + assert_eq!( + queued.generation_log.as_ref().unwrap().entries[0].status, + Some(GenerationJobStatus::Queued) + ); + + let mut running = update(GenerationJobStatus::Generating, Some(0.2)); + running.provider_job_id = Some("fal::fixture-job".to_string()); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + running, + ) + .unwrap(); + let mut downloading = update(GenerationJobStatus::Downloading, Some(0.8)); + downloading.cost_credits = Some(11); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + downloading, + ) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + update(GenerationJobStatus::Finalizing, Some(0.9)), + ) + .unwrap(); + + let media_leaf = format!("{asset_id}.png"); + let relative_path = format!("media/{media_leaf}"); + let mut generated_media = std::io::Cursor::new(b"upscaled-bytes"); + core.finalize_generation_output_with_media_for_project( + runtime.project_epoch, + &bundle, + PreparedGenerationOutput { + asset_id: asset_id.clone(), + relative_path: relative_path.clone(), + probe: ProbedMedia { + duration_secs: 0.0, + width: Some(8), + height: Some(6), + fps: None, + has_audio: false, + }, + created_at: Some(800_000_002.0), + }, + &media_leaf, + 14, + &mut generated_media, + ) + .unwrap(); + + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + let media = reopened.media(); + let source = media + .entries + .iter() + .find(|entry| entry.id == "source-image") + .unwrap(); + assert_eq!(source.source_width, Some(4)); + assert_eq!(source.source_height, Some(3)); + assert_eq!( + fs::read(bundle.join("media/source.png")).unwrap(), + b"source-bytes" + ); + + let output = media + .entries + .iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!(output.source_width, Some(8)); + assert_eq!(output.source_height, Some(6)); + assert_eq!( + output.source, + MediaSource::Project { + relative_path: relative_path.clone() + } + ); + assert_eq!( + output.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Ready) + ); + assert_eq!( + fs::read(bundle.join(&relative_path)).unwrap(), + b"upscaled-bytes" + ); + assert_eq!(fs::read(bundle.join("thumbnail.jpg")).unwrap(), b"cover"); + + let log = reopened.generation_log(); + assert_eq!(log.entries.len(), 5); + assert_eq!(log.total_credits(), 11); + assert_eq!( + log.entries.last().and_then(|entry| entry.status), + Some(GenerationJobStatus::Ready) + ); + assert!(log.entries.iter().all(|entry| { + let json = serde_json::to_string(entry).unwrap(); + !json.contains("source-bytes") && !json.contains("https://") + })); +} + +#[test] +fn invalid_progress_and_error_codes_do_not_mutate_the_durable_job() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, upscale_plan()) + .unwrap(); + let before = fs::read(bundle.join("media.json")).unwrap(); + + let invalid = GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(f64::NAN), + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: None, + }; + assert!(core + .update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + invalid, + ) + .is_err()); + assert_eq!(fs::read(bundle.join("media.json")).unwrap(), before); + + assert!(core + .fail_generation_output_for_project( + runtime.project_epoch, + &bundle, + &committed.placeholder_asset_ids[0], + "provider leaked /private/path", + None, + ) + .is_err()); + assert_eq!(fs::read(bundle.join("media.json")).unwrap(), before); +} + +#[test] +fn cancelling_a_partially_finalized_job_preserves_ready_outputs() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + let mut plan = upscale_plan(); + plan.output_count = 2; + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, plan) + .unwrap(); + let ready_id = committed.placeholder_asset_ids[0].clone(); + let cancelled_id = committed.placeholder_asset_ids[1].clone(); + + for (status, progress) in [ + (GenerationJobStatus::Generating, Some(0.2)), + (GenerationJobStatus::Downloading, Some(0.8)), + (GenerationJobStatus::Finalizing, Some(0.9)), + ] { + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + update(status, progress), + ) + .unwrap(); + } + + let relative_path = format!("media/{ready_id}.png"); + fs::write(bundle.join(&relative_path), b"ready-output").unwrap(); + core.finalize_generation_output_for_project( + runtime.project_epoch, + &bundle, + PreparedGenerationOutput { + asset_id: ready_id.clone(), + relative_path: relative_path.clone(), + probe: ProbedMedia { + duration_secs: 0.0, + width: Some(8), + height: Some(6), + fps: None, + has_audio: false, + }, + created_at: Some(800_000_002.0), + }, + ) + .unwrap(); + core.cancel_generation_output_for_project( + runtime.project_epoch, + &bundle, + &ready_id, + Some(800_000_003.0), + ) + .unwrap(); + core.cancel_generation_output_for_project( + runtime.project_epoch, + &bundle, + &cancelled_id, + Some(800_000_003.0), + ) + .unwrap(); + + let reopened = Project::open(&bundle).unwrap(); + let ready = reopened + .manifest + .entries + .iter() + .find(|entry| entry.id == ready_id) + .unwrap(); + assert_eq!( + ready.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Ready) + ); + assert_eq!( + ready.source, + MediaSource::Project { + relative_path: relative_path.clone(), + } + ); + assert_eq!( + fs::read(bundle.join(relative_path)).unwrap(), + b"ready-output" + ); + + let cancelled = reopened + .manifest + .entries + .iter() + .find(|entry| entry.id == cancelled_id) + .unwrap(); + assert_eq!( + cancelled.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Cancelled) + ); + assert!(matches!( + cancelled.source, + MediaSource::Project { ref relative_path } if relative_path.ends_with(".pending") + )); + assert!(!bundle.join(format!("media/{cancelled_id}.png")).exists()); +} diff --git a/crates/opentake-domain/src/lib.rs b/crates/opentake-domain/src/lib.rs index c7efa6ac..18bb40c8 100644 --- a/crates/opentake-domain/src/lib.rs +++ b/crates/opentake-domain/src/lib.rs @@ -48,8 +48,8 @@ pub use keyframe::{ KeyframeInterpolatable, KeyframeTrack, }; pub use media::{ - GenerationInput, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, MediaManifestEntry, - MediaResolver, MediaSource, + GenerationInput, GenerationJobStatus, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, + MediaManifestEntry, MediaResolver, MediaSource, }; pub use signal::{ ContextSignal, EditingSkeleton, EditingStage, StageGuidance, TrackHint, TrackRole, diff --git a/crates/opentake-domain/src/media.rs b/crates/opentake-domain/src/media.rs index 3ae9ac47..36c27d15 100644 --- a/crates/opentake-domain/src/media.rs +++ b/crates/opentake-domain/src/media.rs @@ -23,6 +23,21 @@ use serde::{Deserialize, Serialize}; use crate::clip_type::ClipType; +/// Durable provider-neutral lifecycle for an asynchronous generated output. +/// Stored with `GenerationInput` so the manifest is the recovery source of +/// truth and never needs provider credentials or signed result URLs. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum GenerationJobStatus { + Queued, + Generating, + Downloading, + Finalizing, + Ready, + Failed, + Cancelled, +} + /// Where a media file lives. Encoded externally-tagged to match Swift's /// synthesized `Codable` for an enum with associated values: /// `{"external":{"absolutePath":"..."}}` / `{"project":{"relativePath":"..."}}`. @@ -100,6 +115,41 @@ pub struct GenerationInput { /// Apple-reference-date seconds (see module note on dates). #[serde(default, skip_serializing_if = "Option::is_none")] pub created_at: Option, + /// Local durable job identity. Provider job ids remain private to the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_id: Option, + /// Non-secret provider routing prefix (`fal`, `replicate`, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Provider job identity required for restart recovery. This is not a + /// credential and must never contain a result URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_job_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, + /// Normalized 0..1 progress. Providers without progress report phase-only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub progress: Option, + /// Fixed application-owned failure code; provider messages are never stored. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_code: Option, + /// Stable ordered output index for N-result generation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_index: Option, + /// Source asset provenance for upscale/edit flows. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_asset_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_clip_id: Option, + /// Timeline span provenance for video-to-audio generation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_start_frame: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_end_frame: Option, + /// Client-side estimate shown before submission; actual settled cost is + /// recorded once in the generation log when a provider supplies it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_cost_credits: Option, } /// Serializable manifest entry. 1:1 port of `MediaManifestEntry`. @@ -460,7 +510,28 @@ impl MediaAsset { source_fps: entry.source_fps, has_audio: entry.has_audio.unwrap_or(false), generation_input: entry.generation_input.clone(), - generation_status: GenerationStatus::None, + generation_status: match entry + .generation_input + .as_ref() + .and_then(|input| input.status) + { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => { + GenerationStatus::Generating + } + Some(GenerationJobStatus::Downloading) => GenerationStatus::Downloading, + Some(GenerationJobStatus::Finalizing) => GenerationStatus::Rendering, + Some(GenerationJobStatus::Failed) => GenerationStatus::Failed( + entry + .generation_input + .as_ref() + .and_then(|input| input.error_code.clone()) + .unwrap_or_else(|| "GENERATION_FAILED".to_string()), + ), + Some(GenerationJobStatus::Cancelled) => { + GenerationStatus::Failed("GENERATION_CANCELLED".to_string()) + } + Some(GenerationJobStatus::Ready) | None => GenerationStatus::None, + }, folder_id: entry.folder_id.clone(), pending_download_url: None, cached_remote_url: entry.cached_remote_url.clone(), diff --git a/crates/opentake-gen/src/build_params.rs b/crates/opentake-gen/src/build_params.rs index b6f6cef7..fb80cbd2 100644 --- a/crates/opentake-gen/src/build_params.rs +++ b/crates/opentake-gen/src/build_params.rs @@ -168,21 +168,44 @@ pub fn build_params( GenerationParams::Video(build_video_edit_params(input, uploaded)) } else { // Frame count: derive from start/end presence via image_urls slot. - let frame_count = input.image_urls.as_ref().map(|v| v.len()).unwrap_or(0); + let frame_count = input + .image_url_asset_ids + .as_ref() + .map(|values| values.len()) + .or_else(|| input.image_urls.as_ref().map(|values| values.len())) + .unwrap_or(0); let image_ref_count = input - .reference_image_urls + .reference_image_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_image_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); let video_ref_count = input - .reference_video_urls + .reference_video_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_video_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); let audio_ref_count = input - .reference_audio_urls + .reference_audio_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_audio_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); GenerationParams::Video(build_video_params( input, diff --git a/crates/opentake-media/src/lib.rs b/crates/opentake-media/src/lib.rs index 762bcd73..3d42d1b5 100644 --- a/crates/opentake-media/src/lib.rs +++ b/crates/opentake-media/src/lib.rs @@ -45,6 +45,78 @@ pub mod waveform; use std::path::{Path, PathBuf}; +/// Materialize an exact visible source range as an uploadable MP4. This is used +/// by generation/upscale when `sourceClipId` is supplied, so provider uploads +/// receive the clip's trimmed source window instead of the complete asset. +pub fn trim_video_range( + source: &Path, + destination: &Path, + start_seconds: f64, + end_seconds: f64, + cancel: &MediaCancelToken, +) -> Result<()> { + if !start_seconds.is_finite() + || !end_seconds.is_finite() + || start_seconds < 0.0 + || end_seconds <= start_seconds + { + return Err(MediaError::Ffmpeg( + "invalid trimmed generation source range".to_string(), + )); + } + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent)?; + } + let duration = end_seconds - start_seconds; + let mut child = std::process::Command::new(ff::ffmpeg_path()) + .args(["-hide_banner", "-loglevel", "error", "-nostdin", "-y"]) + .arg("-ss") + .arg(format!("{start_seconds:.6}")) + .arg("-i") + .arg(source) + .arg("-t") + .arg(format!("{duration:.6}")) + .args([ + "-map", + "0:v:0", + "-map", + "0:a?", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "18", + "-c:a", + "aac", + "-movflags", + "+faststart", + ]) + .arg(destination) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|error| MediaError::Ffmpeg(format!("trim source spawn: {error}")))?; + loop { + if cancel.is_cancelled() { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_file(destination); + return Err(MediaError::Cancelled); + } + if let Some(status) = child.try_wait()? { + if !status.success() || !destination.is_file() { + let _ = std::fs::remove_file(destination); + return Err(MediaError::Ffmpeg( + "trimmed generation source could not be materialized".to_string(), + )); + } + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + // --- flat re-exports of the public API --- pub use cancel::MediaCancelToken; @@ -457,4 +529,39 @@ mod tests { v_streams.trim() ); } + + #[test] + fn trim_video_range_materializes_only_visible_window_and_honors_cancel() { + use std::process::Command; + if !ff::ffmpeg_available() || !ff::ffprobe_available() { + eprintln!("skipping: ffmpeg/ffprobe unavailable"); + return; + } + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("source.mp4"); + let trimmed = temp.path().join("trimmed.mp4"); + let generated = Command::new(ff::ffmpeg_path()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "color=size=64x48:rate=24:duration=3"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&source) + .output() + .unwrap(); + assert!(generated.status.success()); + let source_before = std::fs::read(&source).unwrap(); + + trim_video_range(&source, &trimmed, 0.75, 1.75, &MediaCancelToken::new()).unwrap(); + let trimmed_probe = probe(&trimmed).unwrap(); + assert!((trimmed_probe.duration_secs - 1.0).abs() < 0.2); + assert_eq!(std::fs::read(&source).unwrap(), source_before); + + let cancelled_path = temp.path().join("cancelled.mp4"); + let cancel = MediaCancelToken::new(); + cancel.cancel(); + assert!(matches!( + trim_video_range(&source, &cancelled_path, 0.0, 2.0, &cancel), + Err(MediaError::Cancelled) + )); + assert!(!cancelled_path.exists()); + } } diff --git a/crates/opentake-project/src/bundle.rs b/crates/opentake-project/src/bundle.rs index 6278abf7..075a40c6 100644 --- a/crates/opentake-project/src/bundle.rs +++ b/crates/opentake-project/src/bundle.rs @@ -312,7 +312,65 @@ impl Project { if let Some(source) = media_source { source.copy_media_to(publisher.stage())?; source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + source.copy_thumbnail_to(publisher.stage())?; + } + } + publisher.publish() + } + + /// Replace the same bundle represented by an owned retained root. + /// + /// Windows refuses to rename a directory while a process still owns an + /// open directory handle to it. Complete same-target transactions therefore + /// stage and copy through the retained authority first, explicitly close + /// that authority, and only then enter the existing journaled publication + /// commit. Save-As keeps using [`Self::publish_complete_to`] because its + /// source and destination are distinct. + pub fn publish_complete_replacing_root( + &self, + bundle: impl AsRef, + media_source: ProjectRoot, + ) -> Result { + let encoded = EncodedProject::prepare(self)?; + let publisher = ProjectRoot::begin_replace(bundle.as_ref())?; + encoded.write_to(publisher.stage())?; + media_source.copy_media_to(publisher.stage())?; + media_source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + media_source.copy_thumbnail_to(publisher.stage())?; + } + drop(media_source); + publisher.publish() + } + + /// Replace the owned source bundle while adding one generated media leaf + /// directly to the unpublished sibling stage. + /// + /// This keeps the media bytes, `media.json`, and `generation-log.json` in + /// one directory-publication transaction. In particular, callers do not + /// need to retain an open handle inside the live target across its Windows + /// rename commit point. + pub fn publish_complete_replacing_root_with_media( + &self, + bundle: impl AsRef, + media_source: ProjectRoot, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result { + let encoded = EncodedProject::prepare(self)?; + let publisher = ProjectRoot::begin_replace(bundle.as_ref())?; + encoded.write_to(publisher.stage())?; + media_source.copy_media_to(publisher.stage())?; + publisher + .stage() + .write_new_media_leaf(media_leaf, media_byte_size, media)?; + media_source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + media_source.copy_thumbnail_to(publisher.stage())?; } + drop(media_source); publisher.publish() } } @@ -637,6 +695,103 @@ mod tests { ); } + #[test] + fn complete_publish_replaces_the_owned_source_root() { + let tmp = TmpDir::new("complete-same-target"); + let target = tmp.path().join("Project.opentake"); + let mut project = Project::new(&target); + project.timeline.fps = 24; + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/clip.bin"), b"media").unwrap(); + fs::write(target.join("thumbnail.jpg"), b"cover").unwrap(); + let source_root = ProjectRoot::open(&target).unwrap(); + + project.timeline.fps = 48; + let published = project + .publish_complete_replacing_root(&target, source_root) + .expect("same-target publication must release the old root before rename"); + + assert_eq!( + Project::open_from_root(&published).unwrap().timeline.fps, + 48 + ); + assert_eq!(fs::read(target.join("media/clip.bin")).unwrap(), b"media"); + assert_eq!(fs::read(target.join("thumbnail.jpg")).unwrap(), b"cover"); + } + + #[test] + fn complete_publish_streams_generated_media_into_the_new_bundle() { + let tmp = TmpDir::new("complete-generated-media"); + let target = tmp.path().join("Project.opentake"); + let mut project = Project::new(&target); + project.timeline.fps = 24; + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/source.bin"), b"source").unwrap(); + let source_root = ProjectRoot::open(&target).unwrap(); + let mut generated = std::io::Cursor::new(b"generated"); + + project + .publish_complete_replacing_root_with_media( + &target, + source_root, + "output.bin", + 9, + &mut generated, + ) + .expect("generated media must share the bundle publication commit"); + + assert_eq!( + fs::read(target.join("media/source.bin")).unwrap(), + b"source" + ); + assert_eq!( + fs::read(target.join("media/output.bin")).unwrap(), + b"generated" + ); + } + + #[test] + fn generated_media_stream_failure_preserves_the_live_bundle_byte_exact() { + struct FailingReader(bool); + + impl std::io::Read for FailingReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0 { + return Err(std::io::Error::other("injected media read failure")); + } + self.0 = true; + let bytes = b"partial"; + buffer[..bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + } + + let tmp = TmpDir::new("complete-generated-media-failure"); + let target = tmp.path().join("Project.opentake"); + let project = Project::new(&target); + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/source.bin"), b"source").unwrap(); + let before = tree_receipt(&target); + let source_root = ProjectRoot::open(&target).unwrap(); + let mut generated = FailingReader(false); + + project + .publish_complete_replacing_root_with_media( + &target, + source_root, + "output.bin", + 14, + &mut generated, + ) + .expect_err("a failed generated media stream must abort publication"); + + assert_eq!(tree_receipt(&target), before); + assert!(!target.join("media/output.bin").exists()); + } + #[cfg(unix)] #[test] fn media_copy_failure_leaves_an_existing_target_tree_byte_exact() { diff --git a/crates/opentake-project/src/gen_log.rs b/crates/opentake-project/src/gen_log.rs index e5739f94..1ea15e86 100644 --- a/crates/opentake-project/src/gen_log.rs +++ b/crates/opentake-project/src/gen_log.rs @@ -17,6 +17,8 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; +use opentake_domain::GenerationJobStatus; + fn default_version() -> i64 { 1 } @@ -79,6 +81,26 @@ pub struct GenerationLogEntry { /// Apple-reference-date seconds. `None` when unknown. #[serde(skip_serializing_if = "Option::is_none")] pub created_at: Option, + /// Provider-neutral durable job identity. Never a signed URL or credential. + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub asset_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, + /// Fixed application-owned code only; provider diagnostic text is private. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_asset_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_clip_id: Option, } impl GenerationLogEntry { @@ -94,6 +116,49 @@ impl GenerationLogEntry { model: model.into(), cost_credits, created_at, + job_id: None, + provider: None, + provider_job_id: None, + asset_id: None, + status: None, + progress: None, + error_code: None, + source_asset_id: None, + source_clip_id: None, + } + } + + /// Construct one append-only job lifecycle event without provider secrets. + #[allow(clippy::too_many_arguments)] + pub fn job_event( + id: impl Into, + job_id: impl Into, + model: impl Into, + cost_credits: Option, + provider: impl Into, + provider_job_id: Option, + asset_id: impl Into, + status: GenerationJobStatus, + progress: Option, + error_code: Option, + created_at: Option, + source_asset_id: Option, + source_clip_id: Option, + ) -> Self { + Self { + id: id.into(), + job_id: Some(job_id.into()), + model: model.into(), + provider: Some(provider.into()), + provider_job_id, + asset_id: Some(asset_id.into()), + status: Some(status), + progress, + error_code, + cost_credits, + created_at, + source_asset_id, + source_clip_id, } } } @@ -114,6 +179,15 @@ impl<'de> Deserialize<'de> for GenerationLogEntry { created_at: Option, // Legacy: dollars as a float. Only consulted when costCredits is absent. cost: Option, + job_id: Option, + provider: Option, + provider_job_id: Option, + asset_id: Option, + status: Option, + progress: Option, + error_code: Option, + source_asset_id: Option, + source_clip_id: Option, } let raw = Raw::deserialize(deserializer)?; let cost_credits = match raw.cost_credits { @@ -128,6 +202,15 @@ impl<'de> Deserialize<'de> for GenerationLogEntry { model: raw.model, cost_credits, created_at: raw.created_at, + job_id: raw.job_id, + provider: raw.provider, + provider_job_id: raw.provider_job_id, + asset_id: raw.asset_id, + status: raw.status, + progress: raw.progress, + error_code: raw.error_code, + source_asset_id: raw.source_asset_id, + source_clip_id: raw.source_clip_id, }) } } diff --git a/crates/opentake-project/src/project_root.rs b/crates/opentake-project/src/project_root.rs index 42c39ae8..4faa1e9c 100644 --- a/crates/opentake-project/src/project_root.rs +++ b/crates/opentake-project/src/project_root.rs @@ -157,6 +157,57 @@ impl ProjectRoot { self.copy_directory_component_to(destination, crate::layout::MEDIA_DIR, "media-copy") } + /// Write one fresh media leaf into this retained bundle. + /// + /// Complete generation publication uses this only on an unpublished stage, + /// after the existing media tree has been copied. The final leaf is created + /// with `create_new`, streamed without an ambient destination path, checked + /// against the downloader's exact byte count, synced, and kept only after + /// every write succeeds. + pub(crate) fn write_new_media_leaf( + &self, + name: &str, + expected_bytes: u64, + source: &mut dyn Read, + ) -> Result<()> { + validate_leaf(name).map_err(|error| { + ProjectError::io(self.path.join(crate::layout::MEDIA_DIR).join(name), error) + })?; + let media_path = self.path.join(crate::layout::MEDIA_DIR); + match self.dir.create_dir(crate::layout::MEDIA_DIR) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(ProjectError::io(&media_path, error)), + } + let media = self + .dir + .open_dir_nofollow(crate::layout::MEDIA_DIR) + .map_err(|error| ProjectError::io(&media_path, error))?; + let mut leaf = TransactionLeaf::create(&media, name) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + let copied = std::io::copy( + &mut source.take(expected_bytes.saturating_add(1)), + leaf.handle.as_file_mut(), + ) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + if copied != expected_bytes { + return Err(ProjectError::io( + media_path.join(name), + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "generated media size changed before publication", + ), + )); + } + leaf.handle + .as_file_mut() + .flush() + .and_then(|()| leaf.handle.as_file().sync_all()) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + leaf.cleanup_on_drop = false; + Ok(()) + } + /// Copy project-local Agent conversations during complete-bundle /// publication (Save As / archive) through retained no-follow roots. pub fn copy_chat_sessions_to(&self, destination: &ProjectRoot) -> Result<()> { @@ -167,6 +218,14 @@ impl ProjectRoot { ) } + /// Preserve the optional project cover across complete-bundle publication. + pub(crate) fn copy_thumbnail_to(&self, destination: &ProjectRoot) -> Result<()> { + if let Some(bytes) = self.read_optional(crate::layout::THUMBNAIL_FILE)? { + destination.write_atomic(crate::layout::THUMBNAIL_FILE, &bytes)?; + } + Ok(()) + } + fn copy_directory_component_to( &self, destination: &ProjectRoot, diff --git a/docs/architecture/BUGS.md b/docs/architecture/BUGS.md index 83c6d339..4dda349b 100644 --- a/docs/architecture/BUGS.md +++ b/docs/architecture/BUGS.md @@ -39,13 +39,13 @@ --- -### B4. `canGenerate` 硬编码为 `false`（中） +### B4. `canGenerate` 硬编码为 `false`（已修复） | 属性 | 值 | |---|---| -| **位置** | `crates/opentake-agent/src/tools/...` — 具体待定位 | -| **描述** | Agent 工具的 `canGenerate` 返回值硬编码为 `false`，即使 BYOK key 已配置 | -| **影响** | Agent 无法进行任何 AI 生成操作 | +| **位置** | `crates/opentake-agent/src/mcp/{dispatch,generation}.rs`、`src-tauri/src/generation.rs` | +| **修复** | `canGenerate` 由托管凭据或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 凭据动态派生；无可用凭据时四个付费工具不发布，有凭据时 MCP 与 Chat 共用同一异步生产桥 | +| **验证** | `configured_capability_and_cost_authorization_gate_dispatch` 覆盖有/无能力与显式成本授权；生产 mock provider 路径覆盖 image/video/audio/upscale | --- @@ -112,7 +112,7 @@ | **B1** | swapMedia IPC 缺失 | 🔴 关键 | 单功能完全不可用 | 低（~10 行代码） | | **B2** | 前端帧数学截断不一致 | 🟠 高 | 所有媒体导入时长偏差 | 低（3 处 Math.round→floor） | | **B3** | rippleDeleteRanges 忽略 clipId | 🟠 高 | Agent 工具精度下降 | 中 | -| **B4** | canGenerate 硬编码 false | 🟡 中 | AI 生成功能不可用 | 低 | +| **B4** | canGenerate 硬编码 false（已修复） | ✅ 已关闭 | 动态生成能力已恢复 | — | | **D1** | 预览未接入 GPU 合成 | 🟠 高 | 所有编辑效果不可见 | 中 | | **D2** | Agent 工具 30% stub | 🟠 高 | AI 协作核心缺失 | 高 | | **D3** | 缩略图始终 None | 🟡 中 | 媒体库 UX 差 | 中 | diff --git a/docs/architecture/CAPCUT-GAP.md b/docs/architecture/CAPCUT-GAP.md index f2928d69..bca972f6 100644 --- a/docs/architecture/CAPCUT-GAP.md +++ b/docs/architecture/CAPCUT-GAP.md @@ -89,6 +89,7 @@ - **前置依赖**:FFmpeg vidstab 路线仅依赖 ffmpeg-next(已有);自研路线依赖光流(与追踪共用);补偿应用依赖关键帧引擎/wgpu ### 画质超清修复(super-resolution / enhance) — `partial` · 难度 medium · 优先级 p1 +- **OpenTake 当前状态(2026-07-29)**:BYOK 云端 2x 超分竖切已完成。`upscale_media` 经共享 GenerationBridge 创建耐久占位、上传源素材、轮询 Replicate、校验结果尺寸严格为源宽高 2 倍并作为新资产导入；原资产字节与元数据保持不变。UI 已提供进度/取消/失败/重试，重试要求重新确认成本。确定性生产路径测试覆盖成功、取消不导入、恢复、鉴权/限流、源不变与精确 2x。本地 Real-ESRGAN/SeedVR 修复轨仍未实现，因此总体仍标 `partial`。 - **判定依据**:上游有『Upscale』入口但纯云端、且仅放大不修复:AIEditTab『AI Enhance / Upscale / Enhance resolution with AI』(Inspector/AIEditTab.swift:31-36)、UpscaleModelConfig 从 Convex ModelCatalog 拉取(Generation/Catalog/UpscaleModelConfig.swift)、EditSubmitter.submitUpscale 把 sourceURL+durationSeconds 发后端(UpscaleModelConfig.swift:3-15)、需登录订阅否则禁用(ToolExecutor+Generate.swift:320)、video 仅 <2160 可放大(MODULE-PORT-MAP.md:528)。本地零推理。OpenTake 设计稿 Generation 章把它归为 cloud-rebuild,未规划本地超分实现(MODULE-PORT-MAP.md:231、ARCHITECTURE §8 只讲生成代理双模,未列本地 SR)。 - **落点(crate/层)**:opentake-gen(BYOK 云超分,复刻 UpscaleGenerationParams + job 状态机,设计稿已含)+ 新增 opentake-media 本地超分 worker(ort/candle 跑 SR 模型)作为 OpenTake 增强项 - **实现方案**:双轨。① 云端 BYOK(低成本起步、对齐上游):opentake-gen 已规划复刻 GenerationParams 联合类型与 job 抽象(ARCHITECTURE §8、ROADMAP Phase 9),upscale 作为其中一类,用户自带 fal/Replicate key 直连厂商超分模型(Topaz/SeedVR/Real-ESRGAN 端点),零运营成本,这是把上游云能力『去 Convex 化』的自然落点。② 本地推理(OpenTake 反超点,p2):经 ort 跑 Real-ESRGAN / SwinIR(图像)或 SeedVR2(视频,时序一致),复用 SigLIP2/RVM 同套 ONNX 通道与 ModelDownloader;『修复』(去噪/去压缩伪影/人脸增强 GFPGAN/CodeFormer)与『放大』分开提供,补上上游只放大不修复的短板。③ 处理结果作为新媒体资产回填时间线(沿用上游 upscale 回填语义)。 @@ -233,4 +234,3 @@ - **落点(crate/层)**:opentake-gen(扩展 AudioGenerationParams 增加 referenceAudioURL/voiceId 字段 + provider adapter)+ services/opentake-gen-proxy(对接 ElevenLabs voice-clone / fal / MiniMax 等)+ 音色库管理(可存于 .opentake 工程或账户级)+ keyring 存厂商 key。 - **实现方案**:本地无法稳定实现高质量克隆(需说话人编码+声码器大模型),走外部 API 最务实。两步:① 创建音色:用户提供参考音频样本→预签名上传→调 ElevenLabs Instant Voice Cloning / MiniMax voice clone 等→拿回一个 voiceId,存入音色库。② 使用:把该 voiceId 当作 TTS 的 voice 传入现有 generate_audio 流水线即可复用全部落轨逻辑。需扩展 OpenTake 既有的 AudioGenerationParams 联合类型(report03 已设计该 enum)加 referenceAudioURL/clonedVoiceId 字段,catalog 用 caps 标注"该模型支持克隆"。合规上需加"声音授权确认"。若坚持本地路线,可评估 OpenVoice/XTTS(candle/ort 跑)做轻量零样本音色迁移,但质量与多语稳定性远逊云端,定位为可选实验功能。 - **前置依赖**:依赖 opentake-gen 生成后端 + 上传预签名 + keyring 密钥存储先就绪;依赖支持克隆的外部 provider;与 TTS(图文成片配音)、数字人驱动音频可串联复用。 - diff --git a/docs/architecture/FULL_PROJECT_SCAN_REPORT.md b/docs/architecture/FULL_PROJECT_SCAN_REPORT.md index 4d933ed4..0324b889 100644 --- a/docs/architecture/FULL_PROJECT_SCAN_REPORT.md +++ b/docs/architecture/FULL_PROJECT_SCAN_REPORT.md @@ -95,9 +95,9 @@ **问题/风险**: - 38 个发现面工具均已脱离 `not yet implemented` 分支；`inspect_media` 已通过 Tauri `MediaBridge` 接入图片/视频抽帧及本地转写。 -- 生成/超分/Motion 六个兼容线名在生产后端完成前从 MCP、Chat 与系统提示隐藏；Lottie 源检查仍明确不支持。 -- 细节:生成作业/授权/恢复仍未实现，`canGenerate` 仍为 false；这些是未完成能力，不是可发现工具。 -- 严重性: **高** (AI 协作核心缺失 → transcript-driven 编辑等 workflow 不可用)。 +- 生成/超分四个工具已接共享生产 GenerationBridge：仅在托管或兼容 BYOK 凭据可用时动态发布；Motion 两个兼容线名仍隐藏，Lottie 源检查仍明确不支持。 +- 生成作业已具备成本授权、耐久占位/日志、进度、取消、部分成功、失败码、重试与重启恢复；结果下载受协议/地址/大小/重定向约束并在探测后原子导入。 +- 严重性: **中**（生成主竖切已闭合；高级 AI workflow、Motion 与付费真实账号冒烟仍属发布验证项）。 **测试**: mcp_http.rs (传输) 存在；全工具执行弱。 @@ -122,7 +122,7 @@ **问题/风险**: - Import: thumbnails 永远 None (media.rs:79，“placeholder”)、无进度/反馈、扩展静默丢弃、folder 浏览不全。 - Export: H.264 spine 好，但 H265/ProRes 未接、无进度/取消。 -- Gen: 占位部分，但 agent 路径 stub。 +- Gen: image/video/audio/upscale 已从 MCP/Chat 走生产桥并有无付费网络的 provider 合约与应用集成测试；时间线视频源音频生成已预留渲染路径，但当前内置目录没有 `inputs:["video"]` 的音频模型，因此按能力拒绝。 - Bundle 细微差异 (chat 目录名)。 - 严重性: **中高** (工作流断裂)。 diff --git a/docs/architecture/HANDOFF-2026-07.md b/docs/architecture/HANDOFF-2026-07.md index 6c393c17..ba4b2963 100644 --- a/docs/architecture/HANDOFF-2026-07.md +++ b/docs/architecture/HANDOFF-2026-07.md @@ -143,14 +143,14 @@ default-off Rust renderer or to perform the then-missing first device check. 3. 前端:AgentPanel 消息列表 + streaming 渲染 + 工具调用卡片;Context Signal(`signal/`)注入系统提示已有,接上即可。 - **验收**:面板内发"把这段静音剪掉"能走 tighten_silences 工具链并落 EditCommand;无 key 时有引导文案。 -### 3.4 【P1】生成式 UI + `generate_*`/`upscale` 去 stub(#30 切片) +### 3.4 【P1，代码竖切已完成】生成式 UI + `generate_*`/`upscale` 去 stub(#30 切片) -- **现状**:`opentake-gen` client/job/provider/keys 已建成但尚未形成生产调用；`GenerateVideo/Image/Audio | UpscaleMedia` 兼容线名在后端完成前不进入 MCP/Chat 发现面或系统提示；`encode_timeline(..., false)` 仍硬编 canGenerate=false；web 无生成 UI。 +- **现状(2026-07-29)**:`GenerateVideo/Image/Audio | UpscaleMedia` 已通过 MCP/Chat 共用的 Tauri GenerationBridge 接入 fal/Replicate/OpenAI/ElevenLabs；先持久化占位和 generation log，再异步上传/提交/轮询/下载/探测/导入。`canGenerate` 由可用凭据派生，四工具仅在能力可用时动态发布。MediaPanel 卡片显示进度、取消、固定错误码与需重新确认成本的重试。 - **怎么写**: 1. `src-tauri` 命令层:`generate_start(params)->job_id` / `generate_status` / `generate_cancel`,内部 `opentake-gen::client` 异步 job 轮询,产物落项目 `media/` 并走 `import_media` 现有通路(保证出现在素材面板 + 可撤销)。 2. MCP dispatch 同一命令复用,去掉 4 个 stub;canGenerate 改为"有 BYOK key 即 true"(`keys.rs` 可查)。 3. 前端:对照上游 GenerationView 做 MediaPanel「AI 生成」分区(与 #91 的左侧分区合流):模型选择(`catalog/` 已有 list_models)、提示词、参考图、进度卡片。 -- **验收**:配 fal.ai key 后文生图落库并可拖上时间线;无 key 显示引导;取消可用。 +- **验收状态**:无付费网络的配置 provider 生产路径已覆盖 image/video/audio/upscale、N 图顺序、部分/全部失败、取消、恢复、鉴权/限流、重试和精确 2x；完整工作区测试通过。真实账号付费请求保留到 Beta 实机验证清单，未在自动化中消费用户额度。 ### 3.5 【P1】Save as Media(#48 尾巴) @@ -198,7 +198,7 @@ default-off Rust renderer or to perform the then-missing first device check. | 布局常量(#148) | 对照上游 rulerHeight/dropZoneHeight/trackHeight 改 `theme.ts`,纯数值对齐 | | CSP 加固(#161) | null→非 null 白屏高风险,**必须真机逐项验证**,独立小 PR | | Storage/Models 设置页 | `SettingsView.tsx` 加两 pane:模型缓存管理(whisper/SigLIP 已下载模型列表+删除)、存储占用(项目/缓存目录大小) | -| MCP 隐藏能力 | `InspectMedia` 已接 MediaBridge（剩 Lottie）；生成/超分/Motion 六个兼容线名在生产后端完成前不进入发现面 | +| MCP 隐藏能力 | `InspectMedia` 已接 MediaBridge（剩 Lottie）；生成/超分四工具按凭据能力动态发布，Motion 两个兼容线名仍隐藏 | | 技术债 | `fcpxml.rs` 1489 行拆分;`export_fcpxml` 名实不符(产物 XMEML)可改名 `export_xml`;`library.rs:322` remove 静默吞错补 `tracing::warn!` | --- diff --git a/docs/architecture/MODULE-PORT-MAP.md b/docs/architecture/MODULE-PORT-MAP.md index cc467060..58ae1809 100644 --- a/docs/architecture/MODULE-PORT-MAP.md +++ b/docs/architecture/MODULE-PORT-MAP.md @@ -516,6 +516,7 @@ - 【GenerationService.generate 主流程 — 必须一比一复刻的顺序与边界】1) count = clamp(numImages,1,4);baseName = name ?? prompt 前30字符;resolvedFolderId 仅当该 folder 存在才保留。2) 目标目录:有 projectURL 则 projectURL/'media' 并创建,否则系统临时目录;占位文件名 'gen-.'。3) 同步创建 count 个占位 MediaAsset(generationStatus=.generating, folderId 设好, 追加进 editor.mediaAssets),记 primaryId=占位[0].id 并立即 return(异步在后台 Task 继续)。4) 异步:若给了 preUploadedURLs 则直接用;否则:取 references 的 url 列表 urlsToUpload;若 trimmedSourceOverride.hasTrim 且非空则用 VideoTrimExtractor 替换 urlsToUpload[0] 并登记待清理;若有 preprocessRef 则对每个 reference 并发执行(TaskGroup)可能产出改写 URL;计算 cacheKeys(只有既无 preprocessRef 且不是被裁剪的第0项时,才用该 MediaAsset 作缓存键);uploadReferences 并发上传(命中 freshRemoteURL 缓存则跳过实际上传,上传成功则写回缓存 TTL=6天)。5) finalGenInput:若给 snapshotRefs 则调用它写回各类 URL,否则把 uploaded 整体塞进 imageURLs;createdAt 为空则置 now;把 finalGenInput 赋给所有占位的 generationInput。6) params=buildParams(uploaded)→runJob。7) 任意环节抛错:所有占位 generationStatus=.failed('Upload failed: …'),调 onFailure。8) defer 清理所有临时文件。 - 【runJob 提交+订阅状态机】生成 runId(UUID前8)。submit(model,params,projectId) 拿 jobId,失败→占位.failed + onFailure。subscribe(jobId) 拿 Combine publisher(nil→.failed('Backend not configured'))。把 publisher 包成 AsyncStream 在主线程消费:job.status == .queued/.running 时 continue;== .succeeded → finalizeSuccess 后 return;== .failed → 占位.failed(job.errorMessage ?? 'Generation failed') + onFailure + return。Rust 复刻:用一个状态轮询/推送通道(WebSocket 或轮询 REST)驱动同一状态机。 - 【finalizeSuccess 下载落地 — N 图分发规则】resultUrls 为空→全部占位 .failed('No URL in response')+onFailure。若 resultUrls.count < 占位数,多出的占位标记失败(只 log 不报错)。逐 i 配对占位[i]↔resultUrls[i]:URL 非法→.failed('No URL for placeholder');否则 downloadAndFinalize 成功才 onComplete?(占位) 并计入 finalized。最后只要有 finalized 就对 finalized.first 发 generationComplete 通知(count=finalized.count);全失败→onFailure。 +- 【OpenTake 复刻状态(2026-07-29)】`finalize_terminal_outputs` 已按占位下标确定性配对全部 URL，覆盖 0/少于/等于/多于 N、下载失败、部分成功与全失败；每个输出持久化独立终态，重复回调由 lease + 耐久状态幂等化。Tauri 下载器只接收受限 data/HTTPS，逐跳拒绝私网/凭据/非 443、限制重定向与 1 GiB 解码后字节数，取消或失败清理 staging；探测真实容器后才导入，签名 URL 和 provider 正文不落盘。 - 【downloadAndFinalize 落地细节】先置 .downloading。URLSession 下载到临时文件;若远端扩展名非空、与占位 url 扩展名不同、且该扩展名是已知 ClipType,则把目标 url 改成新扩展名(纠正真实类型)。删除旧文件→移动临时文件到目标。清 pendingDownloadURL、status=.none、importMediaAsset(skipAppend:true)→appendGenerationLog→finalizeImportedAsset(异步补全元数据如尺寸/帧率/音轨)。失败:记 pendingDownloadURL=远端 URL、status=.failed(message),供 retryDownload 重试。 - 【uploadReferences 并发+缓存+顺序保持】对每个 url 起 TaskGroup 任务:cacheKey.freshRemoteURL 命中则直接产出该 URL;否则按文件扩展名/回退类型推断 contentType(jpg→image/jpeg, png, webp, heic, gif, mp4/m4v→video/mp4, mov→video/quicktime, mp3→audio/mpeg, wav, m4a→audio/mp4;回退:image→image/jpeg, video→video/mp4, audio→audio/mpeg, text→octet-stream, lottie→application/json),上传后写缓存。最终按原始下标排序返回(顺序对后续 frames/refs 切分至关重要)。 - 【上传引用缓存 freshRemoteURL】MediaAsset.cachedRemoteURL + cachedRemoteURLExpiresAt;freshRemoteURL 仅当存在且 expiresAt>now 才返回。TTL=6*24*60*60 秒。缓存只对『字节纯净』的资产生效(未裁剪、未预处理)。Rust 复刻:按资产内容哈希做上传去重缓存更稳。 @@ -1292,4 +1293,3 @@ MCPService 端口 19789 与工具注册属 Agent/MCP 子系统,App 层只做开 **移植策略**:ToolbarView 是纯 SwiftUI 表现层，需在 React/TS 前端整体重建为一个工具栏组件，不可直接移植。但它本身几乎没有逻辑——真正要忠实复刻的是它调用的 EditorViewModel 编辑算法(split/trim/addText)，那些应放到 Rust core 实现，前端按钮只发命令。具体替换方案：(1) 布局：用 flex 行 + 分隔符/弹簧重建；图标用任意图标库(如 lucide)替换 SF Symbols(cursorarrow→鼠标、scissors→剪刀、square.split.2x1→分割、放大镜→zoom)。(2) 悬停高亮 hoverHighlight→CSS :hover + 圆角背景，命中区用 padding/伪元素扩大。(3) 撤销/重做：在 Rust core 维护撤销栈(对应 NSUndoManager 的 bidirectional swap 模式——每个 mutation 记录 before/after Timeline 或 per-clip 快照，撤销时回写并重新注册逆向 swap)；前端按钮与 Cmd+Z/Shift+Cmd+Z 都调用 core 的 undo()/redo() 命令(invoke)。(4) 工具模式：toolMode 作为前端/或 core 的 UI 状态枚举(Pointer/Razor)，razor 模式下时间线点击发 split 命令。(5) 分割/裁剪/新增文字算法严格按 coreLogic 在 Rust 实现：注意 round() 取整、源帧=时间线帧*speed、fade 在分割时左清淡出右清淡入、关键帧轨切点插边界帧并 rebase、trim 是 overwrite 式(不波纹)。secondsToFrame 用截断(Int(s*fps))而非四舍五入，务必一致。(6) 缩放滑块：前端 input[type=range] 存 ln(scale)，区间 [ln(minZoom), ln(40)]，回写 scale=Math.exp(v)；minZoom 计算(availableWidth/(totalFrames*3)，钳制到[0.0001,40])可放前端或 core。(7) 文字渲染独立于媒体合成(syncTextLayers→在 FFmpeg/前端 overlay 层单独绘制文字)，新增文字时把文字 clip 放到顶层视频轨、按 currentFrame 起点、默认 3 秒。无 Apple 私有框架阻塞，无 blocker。 **关键文件**:Sources/PalmierPro/Toolbar/ToolbarView.swift、Sources/PalmierPro/Editor/ViewModel/EditorViewModel+ClipMutations.swift、Sources/PalmierPro/Editor/ViewModel/EditorViewModel+Ripple.swift、Sources/PalmierPro/Editor/OverwriteEngine.swift、Sources/PalmierPro/Models/Timeline.swift、Sources/PalmierPro/Utilities/Constants.swift - diff --git a/docs/architecture/ROADMAP.md b/docs/architecture/ROADMAP.md index 95d5266c..8f1e8503 100644 --- a/docs/architecture/ROADMAP.md +++ b/docs/architecture/ROADMAP.md @@ -60,7 +60,7 @@ 3. 应用内 chat(reqwest→Anthropic SSE,BYOK;prompt caching)。 4. **OpenTake 增强**:分层可组合系统提示词 + 模型策略配置化;高阶工具 `remove_filler_words`/`tighten_silences`;写工具返回结构化 JSON;新增 `get_capabilities`。 - **验证**:`claude mcp add` 能连;每个工具走通;应用内 chat 能完成多步链式编辑;助手专属 undo 正确。 -- **进度**:`list_models` 已接 `opentake-gen` 本地 catalog；`inspect_media` 已接桌面媒体桥并有图片端到端、视频故事板和转写契约测试。当前发现面发布 38 个真实路径工具；`generate_*`/`upscale_media`/Motion 六个兼容线名在 async ProviderRegistry + BYOK/确定性渲染完成前保持隐藏，仍是 Phase 7 未完成项。 +- **进度**:`list_models` 与 `inspect_media` 已接生产桥。基础发现面为 38 个真实路径工具；存在托管或兼容 BYOK 凭据时动态增加 `generate_*`/`upscale_media` 四个工具，无凭据时保持隐藏；Motion 两个兼容线名仍未发布。生成与 Chat/MCP 复用同一 Dispatcher/GenerationBridge。 ## Phase 8 — 文字/字幕渲染 + 转写 + 语义搜索 - **做**:cosmic-text + tiny-skia/Vello 文字渲染(阴影/描边/背景/对齐/换行,逐帧 opacity)接入合成器;whisper-rs 转写(word/segment 时间戳,`TranscriptionResult` 模型复用);candle/ort 跑 SigLIP2 + tokenizers 做视觉/口语搜索。 @@ -75,6 +75,7 @@ 对应 `opentake-gen` + `services/opentake-gen-proxy`。 - **做**:`GenClient`(复刻 `GenerationParams` 联合类型 + job 状态机);**BYOK 模式**(本地直连 fal/Replicate/OpenAI,keyring 存 key,内置静态 models catalog);**托管模式**(axum 代理 + provider adapters + 对象存储预签名 + 可选积分计费)。 - **验证**:BYOK 下能用自己的 fal key 生图/生视频并落回时间线;模型目录数据驱动 UI;托管代理可自部署。 +- **进度(2026-07-29)**:provider-neutral 耐久作业、BYOK/托管授权、成本确认、fal/Replicate/OpenAI/ElevenLabs dispatch、N 输出终局化、进度/取消/重试/恢复、MediaPanel 状态与安全下载已完成；确定性配置 provider 测试覆盖 image/video/audio/upscale 且不发起付费网络。托管代理自部署与真实账号付费冒烟仍需在 Beta 发布验证阶段完成。 - **进阶扩展 · AIGC 编排(ADVANCED-FEATURES E 层)**:智能剪口播(本地词级转写+静音检测→Rust 内算 ripple,高阶工具 `remove_filler_words`/`tighten_silences`)、图文成片(agent 编排既有工具+SigLIP2 选素材)、音色克隆(ElevenLabs 等)、虚拟数字人(HeyGen/fal,新增 catalog kind)、多语种字幕翻译(MT/LLM,保时码)。 ## Phase 10 —(新)Motion Canvas 动效 / AI Video 插件 diff --git a/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md b/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md index 633120e8..aac026fc 100644 --- a/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md +++ b/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md @@ -448,7 +448,7 @@ - Add mocked-provider and application integration tests for every named authorization, placeholder, progress, cancellation, finalization, persistence, and failure branch; the affected suites must pass without paid network calls. - Exercise the production MCP or UI path with a deterministic local/mock provider and retain exact manifest, job-state, command-result, and runtime evidence before reclassification. -- [ ] **Step 1: Write or extend every reviewed owning test** +- [x] **Step 1: Write or extend every reviewed owning test** - `crates/opentake-agent/tests/generation_dispatch.rs#placeholder_persist_finalize_all_results_and_failures` (reviewed-planned) — Reviewed planned test belongs in this tracked owning runner beside the mapped product boundary. - `crates/opentake-agent/tests/generation_dispatch.rs#placeholder_persists_and_every_terminal_result_finalizes_once` (reviewed-planned) — Reviewed planned test belongs in this tracked owning runner beside the mapped product boundary. @@ -457,20 +457,20 @@ Each assertion must exercise every covered candidate through the mapped product boundary; an existing-owned test may be extended, while a reviewed-planned test must be added at the declared runner path. -- [ ] **Step 2: Run all focused tests and verify RED** +- [x] **Step 2: Record the RED-evidence disposition** - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact` - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact` - Run: `cargo test -p opentake-gen upscale_uses_first_upload_as_source` - Run: `cargo test -p opentake-gen byok_submit_then_watch_to_succeeded` - Expected: FAIL because one or more of the 15 candidate-bound contracts are not yet satisfied. + Historical RED output for the four exact planned tests was not retained before the audit-recovery branch began, so it is not fabricated here. Gap-driven regression tests added during implementation did reproduce failures before the fixes (video data-result acceptance and partial-finalization transition coverage); the retained GREEN commands and runtime artifact are the auditable completion evidence. -- [ ] **Step 3: Implement the minimal vertical slice** +- [x] **Step 3: Implement the minimal vertical slice** Modify only `crates/opentake-agent/src/mcp/dispatch.rs#run_body`, `crates/opentake-agent/src/mcp/generation.rs#GenerationDispatcher`, `crates/opentake-gen/src/build_params.rs#build_image_params`, `crates/opentake-gen/src/build_params.rs#build_upscale_params`, `crates/opentake-gen/src/build_params.rs#build_video_params`, `crates/opentake-gen/src/client.rs#GenClient`, `crates/opentake-gen/src/client.rs#GenClient::submit`, `crates/opentake-gen/src/client.rs#GenClient::submit_byok`, `crates/opentake-gen/src/client.rs#GenClient::watch`, `src-tauri/src/generation.rs#GenerationBridge`, `web/src/components/agent/AgentPanel.tsx#AgentPanel`, `docs/architecture/BUGS.md`, `docs/architecture/CAPCUT-GAP.md`, `docs/architecture/FULL_PROJECT_SCAN_REPORT.md`, `docs/architecture/HANDOFF-2026-07.md`, `docs/architecture/MODULE-PORT-MAP.md`, `docs/architecture/ROADMAP.md`, `docs/modules/opentake-agent/SPEC.md`, `docs/specs/agent/2-tools.md`, `docs/upstream-analysis/04-MCP与Agent工具.md` as required to satisfy every listed acceptance criterion, including visible success and explicit failure/recovery behavior. -- [ ] **Step 4: Run all focused tests and verify GREEN** +- [x] **Step 4: Run all focused tests and verify GREEN** - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact` - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact` @@ -479,12 +479,14 @@ Expected: PASS with every candidate-bound assertion executed. -- [ ] **Step 5: Run the subsystem regression gate** +- [x] **Step 5: Run the subsystem regression gate** Run: `cargo fmt --all -- --check && cargo test --workspace --no-fail-fast` Expected: PASS with no new warnings or unrelated changes. +Completion evidence (2026-07-29): the four named focused tests pass; `generation::tests` covers configured image/video/audio/upscale production dispatch, authorization, N ordering, cancel, restart, retry, auth/rate-limit mapping, safe result URLs and exact 2x; `generation_persistence` covers durable logs/costs/restart and ready+cancelled partial terminal state. Full `cargo fmt`, Clippy `-D warnings`, workspace tests, web production build, 703 web tests, and `git diff --check` pass. Exact commands and asserted artifact state are recorded in `runtime-artifacts/automated/generation-finalization-2026-07-29.md`. + ### Task 3: advanced-ai-workflows (implementation-slice-aec7c23c8d96431e) **Covered records:** diff --git a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md new file mode 100644 index 00000000..ece025f2 --- /dev/null +++ b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md @@ -0,0 +1,45 @@ +# Generation dispatch/finalization runtime evidence — 2026-07-29 + +Scope: `AG-generation-dispatch-finalization + generation-upscale-finalization` (`implementation-slice-f60bdd3e656a7cb0`). All provider responses are deterministic local `MockTransport` fixtures; no paid provider request or user credit was consumed. + +## Production-path artifact assertions + +- `generate_image`: production Dispatcher → shared Tauri GenerationBridge → configured fal adapter; two durable placeholders are returned before background work, paired in output-index order, probed as 2×2 PNG, persisted `ready`, and resolve to real project media files. +- `generate_video`: configured fal adapter returns a valid 16×16 MP4 fixture; the original placeholder identity becomes a durable video asset with probed dimensions. +- `generate_audio`: configured OpenAI adapter returns a valid WAV fixture; the placeholder becomes a durable audio asset with `hasAudio=true` and positive probed duration. +- `upscale_media`: configured Replicate adapter uploads a 2×2 source fixture and returns 4×4 PNG; the output is exactly 2x and the source file bytes remain byte-for-byte identical. +- Cancellation leaves no imported result; ready outputs in a partially finalized N-output job remain ready while non-terminal siblings become cancelled. +- Restart with provider id resumes and finalizes; restart before provider id persists `GENERATION_RESTART_RETRY_REQUIRED` without any resubmission. Retry requires fresh cost authorization and creates a new job. +- Authentication/rate-limit failures persist fixed safe codes. Local/private result targets are rejected. Provider body, credentials, and signed result URLs are absent from project persistence. + +## Verification commands + +```text +cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact +cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact +cargo test -p opentake-gen upscale_uses_first_upload_as_source +cargo test -p opentake-gen byok_submit_then_watch_to_succeeded +cargo test -p opentake-tauri generation::tests +cargo test -p opentake-core --test generation_persistence +cargo test -p opentake-media trim_video_range_materializes_only_visible_window_and_honors_cancel +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace --no-fail-fast -q +pnpm build +pnpm test +git diff --check +``` + +Final result: all commands passed. Workspace tests reported zero failures; repository-declared ignored tests remained ignored. Web result: 70 test files, 703 tests passed. Vite emitted only the pre-existing chunk-size/dynamic-import optimization warnings. + +## Windows CI correction + +The first exact-tree Windows full-product run (`30459985870`, job `90603221524`) failed all three `generation_persistence` tests with Win32 error 32 while replacing `Generation.opentake`. The durable generation transaction had correctly staged a complete sibling bundle, but the live session still retained an open `ProjectRoot` directory handle when the journaled publisher tried to rename the old target. Windows forbids that rename even though macOS permits it. + +The corrected path copies the complete bundle through the retained root, explicitly consumes and closes that root, and only then invokes the existing journal/backup/restore publication commit. Save-As continues to retain its distinct source root. `complete_publish_replaces_the_owned_source_root` pins this same-target Windows requirement, and the three generation persistence tests plus full format/Clippy/workspace gates pass locally after the correction. The replacement keeps whole-bundle atomicity; it does not fall back to two independently visible JSON writes. + +The next exact-tree Windows run (`30462600088`, job `90612253763`) proved that two Tauri-layer handle lifetimes also had to be corrected. Six generation tests failed: four finalization flows held an uncommitted project-media leaf without `FILE_SHARE_DELETE` while complete-bundle publication renamed the project, and two restart tests kept the pre-restart `AppCore` alive while independently opening the same bundle. The import handle now shares deletion on Windows while retaining delete access, identity validation, and handle-relative rollback. The restart fixtures explicitly drop the old core before constructing the new process-equivalent core. + +A later full-product Windows run (`30538580323`, job `90857715826`) narrowed the remaining error-32 failures to five successful-provider finalization paths. Sharing deletion on the leaf was insufficient because `ProjectMediaCapability` also retained root and `media/` directory handles across the live bundle rename. Finalization now streams the downloaded artifact directly into the unpublished sibling stage through `ProjectRoot`; the generated media leaf, `media.json`, and `generation-log.json` therefore share the existing journaled whole-bundle commit. The stream is bounded to the downloader-recorded byte size, and an undersized, oversized, or failed stream aborts without changing the live bundle. `generated_media_stream_failure_preserves_the_live_bundle_byte_exact` pins that rollback guarantee. + +After this correction, local `generation::tests` pass 9/9, the complete workspace test run reports zero failures (only repository-declared hardware probes remain ignored), workspace Clippy with warnings denied passes, and formatting/diff checks pass. The next exact-tree Windows CI rerun is the authoritative remaining platform confirmation. diff --git a/docs/modules/opentake-agent/SPEC.md b/docs/modules/opentake-agent/SPEC.md index 7bef9029..08c040bc 100644 --- a/docs/modules/opentake-agent/SPEC.md +++ b/docs/modules/opentake-agent/SPEC.md @@ -156,6 +156,8 @@ capabilities: { resources: { subscribe:false, listChanged:false }, 枚举顺序严格按 `ToolName`（`ToolDefinitions.swift:4-36`）：`get_timeline, get_media, add_clips, insert_clips, remove_clips, remove_tracks, move_clips, set_clip_properties, set_keyframes, split_clip, ripple_delete_ranges, undo, add_texts, add_captions, generate_video, generate_image, generate_audio, upscale_media, import_media, list_models, inspect_media, get_transcript, inspect_timeline, search_media, list_folders, create_folder, move_to_folder, rename_media, rename_folder, delete_media, delete_folder`。 +> OpenTake 生产发现面按能力动态组成：基础 38 个真实路径工具始终可见；托管凭据或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 可用时再发布四个生成工具。四个工具都要求调用方传 `costAuthorized:true`，MCP 与应用内 Chat 共用同一 GenerationBridge；无凭据时 `get_timeline.canGenerate=false` 且工具不进入目录/系统提示。 + #### A. 读 / 内省（只读，7 个） | # | 工具 | 关键参数（schema 字段；`*`=required） | required 字段 | 背后命令（opentake-core） | Context Signal 附加（§6） | @@ -189,10 +191,10 @@ capabilities: { resources: { subscribe:false, listChanged:false }, | # | 工具 | 关键参数 | required | 背后命令 | |---|---|---|---|---| -| 20 | `generate_video` | `prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交，立即返回 placeholder asset ID；**花钱、不可撤销** | -| 21 | `generate_image` | `prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`referenceMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交 placeholder | -| 22 | `generate_audio` | `prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | （无） | `opentake-gen`：TTS / 文生乐 / 视频配乐；时间线区间结果自动落轨 | -| 23 | `upscale_media` | `mediaRef*`,`model?`,`sourceClipId?` | `mediaRef` | `opentake-gen`：升分辨率 placeholder | +| 20 | `generate_video` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：耐久占位后异步提交；进度/取消/重试/恢复；**花钱、不可撤销** | +| 21 | `generate_image` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`numImages?:1..4`,`referenceMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：异步提交 N 个有序耐久占位并逐项终局化 | +| 22 | `generate_audio` | `costAuthorized*:bool`,`prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | `costAuthorized` | `opentake-gen`：TTS / 文生乐 / 能力允许时的视频配乐；时间线区间先确定性渲染再上传 | +| 23 | `upscale_media` | `costAuthorized*:bool`,`mediaRef*`,`model?`,`sourceClipId?` | `costAuthorized,mediaRef` | `opentake-gen`：异步 2x，占位终局化为新资产且源资产不变 | | 24 | `import_media` | `source*`{三选一 `url?`(HTTPS≤1GB)/`path?`(本地，可目录递归)/`bytes?`(base64≤~15MB)，`mimeType?`},`name?`,`folderId?` | `source` | `opentake-core`/`opentake-project`：url 在阻塞工作线程中逐块下载、path/bytes 同步；扩展名/MIME/探测类型白名单；验证完成后原子发布 manifest | URL 导入禁用自动重定向并逐跳重新校验 HTTPS、host 与 userinfo；`Content-Length` 只做提前拒绝，逐块累计的解码后字节数才是 1 GiB 硬上限。下载先写 retained project capability 下的未提交 leaf，探测、容器/流类型、project epoch/目录、folder 与 leaf identity 全部通过后，才把候选 manifest 原子写入并发事件。任何此前错误或显式 MCP `notifications/cancelled` 都由 guard 擦除/删除 staging，并保持 live manifest 与磁盘 `media.json` 原字节不变。取消是 rmcp 的显式 request-id 通知语义；不承诺仅因原始 TCP 断连而取消。生产 HTTPS 请求与每个响应 chunk 都通过 `tokio::select!` 同时监听取消令牌，不等待长网络超时才清理。 diff --git a/docs/specs/agent/2-tools.md b/docs/specs/agent/2-tools.md index 162130d3..76c4d224 100644 --- a/docs/specs/agent/2-tools.md +++ b/docs/specs/agent/2-tools.md @@ -8,7 +8,7 @@ 枚举顺序严格按 `ToolName`（`ToolDefinitions.swift:4-36`）：`get_timeline, get_media, add_clips, insert_clips, remove_clips, remove_tracks, move_clips, set_clip_properties, set_keyframes, split_clip, ripple_delete_ranges, undo, add_texts, add_captions, generate_video, generate_image, generate_audio, upscale_media, import_media, list_models, inspect_media, get_transcript, inspect_timeline, search_media, list_folders, create_folder, move_to_folder, rename_media, rename_folder, delete_media, delete_folder`。 -> 这是上游 31 工具契约清单，不等于当前发现面。OpenTake 目前在 `ToolName::KNOWN` 保留 44 个兼容线名，仅将 38 个真实路径工具放入 `ToolName::ALL`；`generate_video`、`generate_image`、`generate_audio`、`upscale_media`、`add_motion_graphic`、`edit_motion_graphic` 在生产后端完成前不会出现在 MCP/Chat 目录或系统提示中。 +> 这是上游 31 工具契约清单，不等于当前发现面。OpenTake 在 `ToolName::KNOWN` 保留 44 个兼容线名：基础 38 个真实路径工具始终发布；存在托管或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 凭据时动态增加 `generate_video`、`generate_image`、`generate_audio`、`upscale_media`。Motion 两个兼容线名仍不发布。 ### A. 读 / 内省（只读，7 个） @@ -43,10 +43,12 @@ | # | 工具 | 关键参数 | required | 背后命令 | |---|---|---|---|---| -| 20 | `generate_video` | `prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交，立即返回 placeholder asset ID；**花钱、不可撤销** | -| 21 | `generate_image` | `prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`referenceMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交 placeholder | -| 22 | `generate_audio` | `prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | （无） | `opentake-gen`：TTS / 文生乐 / 视频配乐；时间线区间结果自动落轨 | -| 23 | `upscale_media` | `mediaRef*`,`model?`,`sourceClipId?` | `mediaRef` | `opentake-gen`：升分辨率 placeholder | +| 20 | `generate_video` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：先持久化占位再异步提交；进度/取消/重试/恢复；**花钱、不可撤销** | +| 21 | `generate_image` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`numImages?:1..4`,`referenceMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：N 个有序占位，逐项下载/失败终局化 | +| 22 | `generate_audio` | `costAuthorized*:bool`,`prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | `costAuthorized` | `opentake-gen`：TTS / 文生乐 / 能力允许时的视频配乐；时间线区间先渲染再上传 | +| 23 | `upscale_media` | `costAuthorized*:bool`,`mediaRef*`,`model?`,`sourceClipId?` | `costAuthorized,mediaRef` | `opentake-gen`：严格 2x 新资产，源资产不变 | + +生成请求由 MCP/Chat 共用的 Tauri GenerationBridge 执行。接受请求前必须有显式成本授权；占位与 generation log 先原子持久化，后台再上传/提交/轮询。固定状态为 queued/generating/downloading/finalizing/ready/failed/cancelled；重启时有 provider job id 则恢复轮询，无 id 则以固定错误码要求显式重试，避免重复扣费。结果 URL 不落盘，下载后探测真实容器再导入。 | 24 | `import_media` | `source*`{三选一 `url?`(HTTPS≤1GB)/`path?`(本地，可目录递归)/`bytes?`(base64≤~15MB)，`mimeType?`},`name?`,`folderId?` | `source` | `opentake-core`/`opentake-project`：url 后台下载、path/bytes 同步；扩展名白名单 | ### D. 媒体库组织（写，7 个）——均可撤销，均支持「单条参数 或 `entries[]` 批量」二选一 diff --git "a/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" "b/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" index bffaa0d3..e5e515b3 100644 --- "a/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" +++ "b/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" @@ -112,6 +112,8 @@ require('mcp-remote/dist/proxy.js'); | **upscale_media** | 升分辨率(视频/图) | `mediaRef`,`model?`,`sourceClipId?`(只 upscale trim 段) | `EditSubmitter.submitUpscale(...)`。`+Generate.swift:313` | | **import_media** | 导入外部媒体(其它 MCP / 本地文件)。**异构来源的桥** | `source`{三选一:`url`(HTTPS,≤1GB,后台下载)/`path`(本地,可目录递归)/`bytes`(base64,≤~15MB),`mimeType?`},`name?`,`folderId?` | url 走带 `ImportDownloadDelegate`(超限取消)的后台下载→`editor.importMediaAsset`;path/bytes 同步;目录走 `editor.importFinderItems`。校验 https/无凭据/有 host/扩展名白名单。`+Import.swift:11` | +> **OpenTake 复刻状态(2026-07-29)**：四个生成工具已接 provider-neutral Tauri GenerationBridge，并额外要求 `costAuthorized:true`。fal/Replicate/OpenAI/ElevenLabs BYOK 与托管模式共用耐久占位、日志、进度、取消、重试、重启恢复和安全下载终局化；配置 provider 的确定性生产路径测试已覆盖 image/video/audio/upscale，N 图按下标配对，超分结果必须严格 2x 且不修改源资产。当前内置音频目录没有 `inputs:["video"]` 模型，因此视频配乐输入会按模型能力明确拒绝，时间线区间渲染上传路径保留给未来兼容模型。 + ### D. 媒体库组织(写,7 个) 全部在 `+Folders.swift`,均可撤销,均支持"单条参数 或 `entries[]` 批量"二选一: diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 4a3418b4..f3472553 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -56,7 +56,8 @@ sha2 = "0.10" uuid = { workspace = true } # Optional account scaffold: verify a token only against a user-configured # backend. Rustls keeps the desktop build independent of a system OpenSSL. -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +futures-util = "0.3" # Tokio is also required by the always-built MCP server (URL import cancellation, # timers and the loopback runtime), so it cannot be optional in the minimal diff --git a/src-tauri/src/account.rs b/src-tauri/src/account.rs index 5073b4c5..bea429ea 100644 --- a/src-tauri/src/account.rs +++ b/src-tauri/src/account.rs @@ -100,6 +100,15 @@ struct BoundCredential { backend_url: String, } +/// Stored, backend-bound credential for managed generation. The tuple remains +/// inside the Rust process and is never returned by a Tauri command. +pub(crate) fn generation_credential() -> Result, String> { + let store = keyring_store(); + let backend = load_backend_url(&store)?; + Ok(load_bound_credential(&store, backend.as_deref())? + .map(|credential| (credential.backend_url, credential.token))) +} + fn advance_generation(runtime: &mut AccountRuntime) -> u64 { runtime.generation = runtime.generation.wrapping_add(1); runtime.generation diff --git a/src-tauri/src/chat.rs b/src-tauri/src/chat.rs index e3be6fb0..d33efdf3 100644 --- a/src-tauri/src/chat.rs +++ b/src-tauri/src/chat.rs @@ -23,6 +23,7 @@ use opentake_agent::chat::{ }; use opentake_agent::mcp::core_handle::{AppCoreHandle, CoreHandle}; use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::mcp::generation::GenerationBridge; use opentake_agent::tools::result::ToolResult; use opentake_gen::{KeyStore, KeyringStore}; @@ -89,21 +90,49 @@ impl ChatProjectContext { } impl ChatState { + #[cfg(test)] + pub fn new( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + ) -> Self { + Self::new_inner(core, workflows_dir, cache_root, models_dir, None) + } + /// Build the state in `setup`: a dispatcher over the live core + workflow /// registry + the same media bridge the desktop MCP server uses. - pub fn new( + pub fn new_with_generation( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + generation_bridge: Arc, + ) -> Self { + Self::new_inner( + core, + workflows_dir, + cache_root, + models_dir, + Some(generation_bridge), + ) + } + + fn new_inner( core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_dir: PathBuf, + generation_bridge: Option>, ) -> Self { let handle: Arc = Arc::new(AppCoreHandle::new(core.clone())); let registry = Arc::new(RwLock::new(crate::mcp::build_registry(&workflows_dir))); let bridge = crate::mcp::build_media_bridge(core.clone(), cache_root, models_dir); - let dispatcher = Arc::new(Dispatcher::with_bridge( + let dispatcher = Arc::new(Dispatcher::with_bridges( handle, registry.clone(), Some(bridge), + generation_bridge, )); let store: Arc = Arc::new(KeyringStore::new()); let sessions = Arc::new(Mutex::new(HashMap::new())); diff --git a/src-tauri/src/export.rs b/src-tauri/src/export.rs index 48bb9918..a3892167 100644 --- a/src-tauri/src/export.rs +++ b/src-tauri/src/export.rs @@ -995,6 +995,7 @@ pub fn run_export( #[derive(Default)] pub(crate) struct ExportRunOptions<'a> { pub(crate) control: Option<&'a ExportControl>, + pub(crate) external_cancel: Option, pub(crate) on_progress: Option, pub(crate) frame_range: Option<(i32, i32)>, pub(crate) output_file: Option, @@ -1009,6 +1010,7 @@ pub(crate) fn run_export_with_control( mut options: ExportRunOptions<'_>, ) -> Result { let control = options.control; + let external_cancel = options.external_cancel.clone(); let on_progress = options.on_progress; let defer_completion = options.defer_completion; let reserved_output = options.output_file.is_some(); @@ -1068,7 +1070,11 @@ pub(crate) fn run_export_with_control( let mut last_progress_emit = Instant::now(); for f in start_frame..end_frame { - if control.is_some_and(|c| c.is_cancelled()) { + if control.is_some_and(|c| c.is_cancelled()) + || external_cancel + .as_ref() + .is_some_and(MediaCancelToken::is_cancelled) + { // `abort` kills + waits on the ffmpeg child (unlike a plain `drop`, // which would orphan the process and race the file removal below). encoder.abort(); @@ -1927,6 +1933,7 @@ fn save_range_as_media_workflow( &req, ExportRunOptions { control: Some(control), + external_cancel: None, on_progress: Some(Arc::clone(&on_progress)), frame_range: Some((in_frame, out_frame)), output_file: Some(output_file), diff --git a/src-tauri/src/generation.rs b/src-tauri/src/generation.rs new file mode 100644 index 00000000..33b6c516 --- /dev/null +++ b/src-tauri/src/generation.rs @@ -0,0 +1,2913 @@ +//! Production asynchronous generation bridge shared by MCP and in-app Chat. +//! +//! Durable placeholder state is committed before provider submission. Provider +//! keys stay in the OS keychain; signed result URLs and provider diagnostics are +//! never persisted. Terminal downloads are probed, then streamed into the same +//! complete-bundle publication that makes the original placeholder ready. + +use std::collections::{BTreeSet, HashMap}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use base64::Engine as _; +use futures_util::StreamExt; +use opentake_agent::mcp::generation::{ + finalize_terminal_outputs, DownloadedGenerationArtifact, GenerationArtifactDownloader, + GenerationBridge, GenerationFinalizationStore, GenerationRequest, GenerationSubmission, +}; +use opentake_agent::tools::args::{ + GenerateAudioArgs, GenerateImageArgs, GenerateVideoArgs, UpscaleMediaArgs, +}; +use opentake_core::{ + AppCore, GenerationStateUpdate, PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, +}; +use opentake_domain::{ClipType, GenerationInput, GenerationJobStatus, MediaResolver, Timeline}; +use opentake_gen::catalog::cost::cost_for_input; +use opentake_gen::{ + build_params, Catalog, CatalogEntry, ElevenLabsAdapter, FalAdapter, GenClient, GenError, + JobStatus, KeyStore, KeyringStore, ModelKind, ModelRoute, OpenAiAdapter, ProviderKey, + ProviderRegistry, ReplicateAdapter, ReqwestTransport, StaticToken, UiCapabilities, +}; +use opentake_media::{MediaCancelToken, MediaEngine}; + +const RESULT_BYTES_MAX: u64 = 1024 * 1024 * 1024; +const DATA_URL_ENCODED_MAX: usize = 512 * 1024 * 1024; +const RESULT_REDIRECT_MAX: usize = 5; + +#[derive(Default)] +struct GenerationRuntime { + jobs: Mutex>, + terminal_leases: Mutex>, + completed: Mutex>, +} + +#[derive(Clone)] +pub(crate) struct TauriGenerationBridge { + core: AppCore, + engine: Arc, + staging_root: PathBuf, + runtime: Arc, + clients: Arc, +} + +trait GenerationClientFactory: Send + Sync { + fn configured_byok_prefixes(&self) -> BTreeSet; + fn has_managed_credential(&self) -> bool; + fn build(&self, provider: &str, managed: bool) -> Result; +} + +struct ProductionGenerationClientFactory; + +impl GenerationClientFactory for ProductionGenerationClientFactory { + fn configured_byok_prefixes(&self) -> BTreeSet { + let store = KeyringStore::new(); + [ + ProviderKey::Fal, + ProviderKey::Replicate, + ProviderKey::OpenAI, + ProviderKey::ElevenLabs, + ] + .into_iter() + .filter_map(|key| { + (&store as &dyn KeyStore) + .load_key(key) + .ok() + .flatten() + .map(|_| key.prefix().to_string()) + }) + .collect() + } + + fn has_managed_credential(&self) -> bool { + crate::account::generation_credential() + .ok() + .flatten() + .is_some() + } + + fn build(&self, provider: &str, managed: bool) -> Result { + build_client(provider, managed) + } +} + +struct PreparedDispatch { + plan: PreparedGenerationJob, + references: Vec, + timeline_span: Option, + requires_source_video: bool, + model_kind: ModelKind, + managed: bool, +} + +struct PreparedTimelineSpan { + timeline: Timeline, + manifest: opentake_domain::MediaManifest, + project_dir: Option, + start_frame: i32, + end_frame: i32, +} + +#[derive(Clone)] +struct PreparedReference { + path: PathBuf, + fallback: &'static str, + trim_range: Option<(f64, f64)>, +} + +struct StagedCleanup { + path: PathBuf, + armed: bool, +} + +impl StagedCleanup { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn preserve(mut self) { + self.armed = false; + } +} + +impl Drop for StagedCleanup { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl PreparedReference { + fn whole(path: PathBuf, fallback: &'static str) -> Self { + Self { + path, + fallback, + trim_range: None, + } + } +} + +pub(crate) fn build_bridge( + core: AppCore, + cache_root: PathBuf, + models_dir: PathBuf, +) -> Arc { + Arc::new(TauriGenerationBridge { + core, + engine: Arc::new(MediaEngine::new(cache_root.clone(), models_dir)), + staging_root: cache_root.join("generation-staging"), + runtime: Arc::new(GenerationRuntime::default()), + clients: Arc::new(ProductionGenerationClientFactory), + }) +} + +#[cfg(test)] +fn build_bridge_with_clients( + core: AppCore, + cache_root: PathBuf, + models_dir: PathBuf, + clients: Arc, +) -> Arc { + Arc::new(TauriGenerationBridge { + core, + engine: Arc::new(MediaEngine::new(cache_root.clone(), models_dir)), + staging_root: cache_root.join("generation-staging"), + runtime: Arc::new(GenerationRuntime::default()), + clients, + }) +} + +impl TauriGenerationBridge { + pub(crate) fn cancel(&self, job_id: &str) -> bool { + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(job_id) + .is_some_and(|token| { + token.cancel(); + true + }) + } + + pub(crate) fn retry( + &self, + job_id: &str, + cost_authorized: bool, + ) -> Result { + if !cost_authorized { + return Err("cost authorization is required before retry".to_string()); + } + let snapshot = self.core.runtime_snapshot(); + let outputs = snapshot + .media + .entries + .iter() + .filter(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.job_id.as_deref()) + == Some(job_id) + }) + .collect::>(); + let first = outputs + .first() + .ok_or_else(|| "generation job does not exist".to_string())?; + if outputs.iter().any(|entry| { + !matches!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Failed | GenerationJobStatus::Cancelled) + ) + }) { + return Err("only a failed or cancelled generation can be retried".to_string()); + } + let input = first + .generation_input + .as_ref() + .ok_or_else(|| "generation provenance is missing".to_string())?; + let catalog = Catalog::builtin(); + let model = catalog + .entries() + .iter() + .find(|entry| entry.id == input.model) + .ok_or_else(|| "generation model is no longer available".to_string())?; + let request = match model.kind { + ModelKind::Video => { + let frames = input.image_url_asset_ids.clone().unwrap_or_default(); + GenerationRequest::Video(GenerateVideoArgs { + cost_authorized: Some(true), + prompt: input.prompt.clone(), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + duration: Some(input.duration), + aspect_ratio: Some(input.aspect_ratio.clone()), + resolution: input.resolution.clone(), + start_frame_media_ref: frames.first().cloned(), + end_frame_media_ref: frames.get(1).cloned(), + source_video_media_ref: input.source_asset_id.clone(), + source_clip_id: input.source_clip_id.clone(), + reference_image_media_refs: input.reference_image_asset_ids.clone(), + reference_video_media_refs: input.reference_video_asset_ids.clone(), + reference_audio_media_refs: input.reference_audio_asset_ids.clone(), + folder_id: first.folder_id.clone(), + }) + } + ModelKind::Image => GenerationRequest::Image(GenerateImageArgs { + cost_authorized: Some(true), + prompt: input.prompt.clone(), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + aspect_ratio: Some(input.aspect_ratio.clone()), + resolution: input.resolution.clone(), + quality: input.quality.clone(), + num_images: Some(outputs.len() as i32), + reference_media_refs: input.reference_image_asset_ids.clone(), + folder_id: first.folder_id.clone(), + }), + ModelKind::Audio => GenerationRequest::Audio(GenerateAudioArgs { + cost_authorized: Some(true), + prompt: Some(input.prompt.clone()), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + voice: input.voice.clone(), + lyrics: input.lyrics.clone(), + style_instructions: input.style_instructions.clone(), + instrumental: input.instrumental, + duration: Some(input.duration), + video_source_start_frame: input.source_start_frame, + video_source_end_frame: input.source_end_frame, + video_source_media_ref: input.source_asset_id.clone(), + folder_id: first.folder_id.clone(), + }), + ModelKind::Upscale => GenerationRequest::Upscale(UpscaleMediaArgs { + cost_authorized: Some(true), + media_ref: input + .source_asset_id + .clone() + .ok_or_else(|| "upscale source provenance is missing".to_string())?, + model: Some(input.model.clone()), + source_clip_id: input.source_clip_id.clone(), + }), + }; + self.submit(request, &MediaCancelToken::new()) + } + + /// Resume provider polling for durable non-terminal jobs after a project is + /// opened. A queued record without a provider id is deliberately failed and + /// exposed for explicit retry: resubmitting it automatically could create a + /// second paid job if the process died between provider acceptance and the + /// durable id write. + pub(crate) fn recover_current_project(&self) -> usize { + #[derive(Default)] + struct RecoveryJob { + provider: String, + provider_job_id: Option, + placeholders: Vec<(usize, String)>, + has_active_output: bool, + } + + let snapshot = self.core.runtime_snapshot(); + let Some(project_dir) = snapshot.project_dir.clone() else { + return 0; + }; + let mut recoverable = HashMap::::new(); + for entry in &snapshot.media.entries { + let Some(input) = entry.generation_input.as_ref() else { + continue; + }; + let Some(job_id) = input.job_id.as_ref() else { + continue; + }; + let job = recoverable.entry(job_id.clone()).or_default(); + if job.provider.is_empty() { + job.provider = input.provider.clone().unwrap_or_default(); + } + if job.provider_job_id.is_none() { + job.provider_job_id = input.provider_job_id.clone(); + } + job.placeholders + .push((input.output_index.unwrap_or(usize::MAX), entry.id.clone())); + if matches!( + input.status, + Some( + GenerationJobStatus::Queued + | GenerationJobStatus::Generating + | GenerationJobStatus::Downloading + | GenerationJobStatus::Finalizing + ) + ) { + job.has_active_output = true; + } + } + + let mut resumed = 0; + for (job_id, mut job) in recoverable { + if !job.has_active_output { + continue; + } + let already_running = self + .runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains_key(&job_id); + if already_running { + continue; + } + job.placeholders.sort_by_key(|(index, _)| *index); + let placeholder_ids = job + .placeholders + .into_iter() + .map(|(_, asset_id)| asset_id) + .collect::>(); + let Some(provider_job_id) = job.provider_job_id else { + let _ = self.core.update_generation_job_for_project( + snapshot.project_epoch, + &project_dir, + &job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Failed, + progress: None, + error_code: Some("GENERATION_RESTART_RETRY_REQUIRED".to_string()), + provider_job_id: None, + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ); + continue; + }; + if job.provider.is_empty() { + self.fail_nonterminal_outputs( + snapshot.project_epoch, + &project_dir, + &placeholder_ids, + "GENERATION_RECOVERY_STATE_INVALID", + ); + continue; + } + let cancel = MediaCancelToken::new(); + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.clone(), cancel.clone()); + let bridge = self.clone(); + let recovery_dir = project_dir.clone(); + let managed = !provider_job_id.starts_with(&format!("{}::", job.provider)); + tauri::async_runtime::spawn(async move { + bridge + .run_recovered_job( + snapshot.project_epoch, + recovery_dir, + job_id, + placeholder_ids, + job.provider, + managed, + provider_job_id, + cancel, + ) + .await; + }); + resumed += 1; + } + resumed + } + + fn fail_nonterminal_outputs( + &self, + project_epoch: u64, + project_dir: &Path, + placeholder_ids: &[String], + code: &str, + ) { + let snapshot = self.core.runtime_snapshot(); + for asset_id in placeholder_ids { + let terminal = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == *asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status) + .is_some_and(|status| { + matches!( + status, + GenerationJobStatus::Ready + | GenerationJobStatus::Failed + | GenerationJobStatus::Cancelled + ) + }); + if !terminal { + let _ = self.core.fail_generation_output_for_project( + project_epoch, + project_dir, + asset_id, + code, + Some(now_apple_reference_seconds()), + ); + } + } + } + + fn cancel_nonterminal_outputs( + &self, + project_epoch: u64, + project_dir: &Path, + placeholder_ids: &[String], + ) { + for asset_id in placeholder_ids { + let _ = self.core.cancel_generation_output_for_project( + project_epoch, + project_dir, + asset_id, + Some(now_apple_reference_seconds()), + ); + } + } + + fn configured_byok_prefixes(&self) -> BTreeSet { + self.clients.configured_byok_prefixes() + } + + fn has_managed_credential(&self) -> bool { + self.clients.has_managed_credential() + } + + fn prepare(&self, request: GenerationRequest) -> Result { + let snapshot = self.core.runtime_snapshot(); + snapshot + .project_dir + .as_deref() + .ok_or_else(|| "Save the project before starting generation".to_string())?; + let configured = self.configured_byok_prefixes(); + let managed_available = self.has_managed_credential(); + let catalog = Catalog::builtin(); + + match request { + GenerationRequest::Video(args) => { + let entry = select_model( + &catalog, + ModelKind::Video, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Video(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid video capabilities".to_string()); + }; + let duration = args + .duration + .map(|value| value.max(0) as u32) + .or_else(|| caps.durations.first().copied()) + .unwrap_or(0); + if !caps.durations.is_empty() && !caps.durations.contains(&duration) { + return Err("duration is not supported by the selected model".to_string()); + } + let aspect_ratio = args + .aspect_ratio + .clone() + .or_else(|| caps.aspect_ratios.first().cloned()) + .unwrap_or_default(); + if !caps.aspect_ratios.is_empty() && !caps.aspect_ratios.contains(&aspect_ratio) { + return Err("aspectRatio is not supported by the selected model".to_string()); + } + validate_choice( + "resolution", + args.resolution.as_deref(), + caps.resolutions.as_deref(), + )?; + + let frames = [ + args.start_frame_media_ref.clone(), + args.end_frame_media_ref.clone(), + ] + .into_iter() + .flatten() + .collect::>(); + if args.start_frame_media_ref.is_some() && !caps.supports_first_frame { + return Err("selected model does not support a first frame".to_string()); + } + if args.end_frame_media_ref.is_some() && !caps.supports_last_frame { + return Err("selected model does not support a last frame".to_string()); + } + let image_refs = args.reference_image_media_refs.clone().unwrap_or_default(); + let video_refs = args.reference_video_media_refs.clone().unwrap_or_default(); + let audio_refs = args.reference_audio_media_refs.clone().unwrap_or_default(); + validate_reference_count("image", image_refs.len(), caps.max_reference_images)?; + validate_reference_count("video", video_refs.len(), caps.max_reference_videos)?; + validate_reference_count("audio", audio_refs.len(), caps.max_reference_audios)?; + if let Some(max) = caps.max_total_references { + if image_refs.len() + video_refs.len() + audio_refs.len() > max as usize { + return Err( + "too many combined references for the selected model".to_string() + ); + } + } + if caps.frames_and_references_exclusive + && !frames.is_empty() + && (!image_refs.is_empty() || !video_refs.is_empty() || !audio_refs.is_empty()) + { + return Err( + "frames and references are mutually exclusive for this model".to_string(), + ); + } + if caps.requires_source_video && args.source_video_media_ref.is_none() { + return Err("selected model requires sourceVideoMediaRef".to_string()); + } + if caps.requires_reference_image && image_refs.is_empty() { + return Err("selected model requires an image reference".to_string()); + } + let source_trim = validate_source_clip( + &snapshot.timeline, + args.source_clip_id.as_deref(), + args.source_video_media_ref.as_deref(), + )?; + + let mut references = Vec::new(); + if caps.requires_source_video { + if let Some(source) = args.source_video_media_ref.as_deref() { + references.push(PreparedReference { + path: resolve_media(&snapshot, source, ClipType::Video)?, + fallback: "video", + trim_range: source_trim, + }); + } + } else { + for media_ref in &frames { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Image)?, + "image", + )); + } + } + for media_ref in &image_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Image)?, + "image", + )); + } + for media_ref in &video_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Video)?, + "video", + )); + } + for media_ref in &audio_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Audio)?, + "audio", + )); + } + + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: args.prompt.clone(), + model: entry.id.clone(), + duration: duration as i32, + aspect_ratio, + resolution: args.resolution.clone(), + image_url_asset_ids: (!frames.is_empty()).then_some(frames), + reference_image_asset_ids: (!image_refs.is_empty()).then_some(image_refs), + reference_video_asset_ids: (!video_refs.is_empty()).then_some(video_refs), + reference_audio_asset_ids: (!audio_refs.is_empty()).then_some(audio_refs), + generate_audio: Some(true), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &args.prompt, "Generated video"), + kind: ClipType::Video, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: args.source_video_media_ref, + source_clip_id: args.source_clip_id, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span: None, + requires_source_video: caps.requires_source_video, + model_kind: ModelKind::Video, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Image(args) => { + let entry = select_model( + &catalog, + ModelKind::Image, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Image(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid image capabilities".to_string()); + }; + validate_choice( + "aspectRatio", + args.aspect_ratio.as_deref(), + Some(&caps.aspect_ratios), + )?; + validate_choice( + "resolution", + args.resolution.as_deref(), + caps.resolutions.as_deref(), + )?; + validate_choice( + "quality", + args.quality.as_deref(), + caps.qualities.as_deref(), + )?; + let refs = args.reference_media_refs.clone().unwrap_or_default(); + if !refs.is_empty() && !caps.supports_image_reference { + return Err("selected model does not support image references".to_string()); + } + if refs.len() > caps.max_images as usize { + return Err("too many image references for the selected model".to_string()); + } + let references = refs + .iter() + .map(|media_ref| { + resolve_media(&snapshot, media_ref, ClipType::Image) + .map(|path| PreparedReference::whole(path, "image")) + }) + .collect::, _>>()?; + let output_count = args.num_images.unwrap_or(1).clamp(1, 4) as usize; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: args.prompt.clone(), + model: entry.id.clone(), + duration: 0, + aspect_ratio: args + .aspect_ratio + .clone() + .or_else(|| caps.aspect_ratios.first().cloned()) + .unwrap_or_default(), + resolution: args.resolution.clone(), + quality: args.quality.clone(), + num_images: Some(output_count as i32), + reference_image_asset_ids: (!refs.is_empty()).then_some(refs), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &args.prompt, "Generated image"), + kind: ClipType::Image, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count, + source_asset_id: None, + source_clip_id: None, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span: None, + requires_source_video: false, + model_kind: ModelKind::Image, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Audio(args) => { + let entry = select_model( + &catalog, + ModelKind::Audio, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Audio(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid audio capabilities".to_string()); + }; + let prompt = args.prompt.clone().unwrap_or_default(); + if prompt.chars().count() < caps.min_prompt_length as usize { + return Err("prompt is too short for the selected audio model".to_string()); + } + if args.lyrics.is_some() && !caps.supports_lyrics { + return Err("selected model does not support lyrics".to_string()); + } + if args.instrumental == Some(true) && !caps.supports_instrumental { + return Err("selected model does not support instrumental mode".to_string()); + } + if args.style_instructions.is_some() && !caps.supports_style_instructions { + return Err("selected model does not support style instructions".to_string()); + } + let timeline_span = match ( + args.video_source_start_frame, + args.video_source_end_frame, + ) { + (None, None) => None, + (Some(start), Some(end)) => { + if args.video_source_media_ref.is_some() { + return Err( + "timeline span and videoSourceMediaRef are mutually exclusive" + .to_string(), + ); + } + let total = snapshot.timeline.total_frames(); + if start < 0 || end <= start || end > total { + return Err( + "video source frame range is outside the timeline".to_string() + ); + } + Some(PreparedTimelineSpan { + timeline: snapshot.timeline.clone(), + manifest: snapshot.media.clone(), + project_dir: snapshot.project_dir.clone(), + start_frame: start, + end_frame: end, + }) + } + _ => return Err( + "videoSourceStartFrame and videoSourceEndFrame must be provided together" + .to_string(), + ), + }; + if (timeline_span.is_some() || args.video_source_media_ref.is_some()) + && !caps + .inputs + .as_ref() + .is_some_and(|inputs| inputs.iter().any(|input| input == "video")) + { + return Err("selected audio model does not support a video source".to_string()); + } + let references = match args.video_source_media_ref.as_deref() { + Some(media_ref) => vec![PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Video)?, + "video", + )], + None => Vec::new(), + }; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: prompt.clone(), + model: entry.id.clone(), + duration: args.duration.unwrap_or_else(|| { + timeline_span + .as_ref() + .map(|span| { + ((span.end_frame - span.start_frame) as f64 + / span.timeline.fps.max(1) as f64) + .ceil() as i32 + }) + .unwrap_or(0) + }), + aspect_ratio: String::new(), + voice: args.voice, + lyrics: args.lyrics, + style_instructions: args.style_instructions, + instrumental: args.instrumental, + reference_video_asset_ids: args + .video_source_media_ref + .clone() + .map(|id| vec![id]), + source_start_frame: args.video_source_start_frame, + source_end_frame: args.video_source_end_frame, + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &prompt, "Generated audio"), + kind: ClipType::Audio, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: args.video_source_media_ref, + source_clip_id: None, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span, + requires_source_video: false, + model_kind: ModelKind::Audio, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Upscale(args) => { + let source = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == args.media_ref) + .ok_or_else(|| "upscale source asset does not exist".to_string())?; + if !matches!(source.kind, ClipType::Image | ClipType::Video) { + return Err("upscale source must be image or video".to_string()); + } + let source_trim = validate_source_clip( + &snapshot.timeline, + args.source_clip_id.as_deref(), + Some(&args.media_ref), + )?; + let entry = select_model( + &catalog, + ModelKind::Upscale, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Upscale(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid upscale capabilities".to_string()); + }; + let source_kind = if source.kind == ClipType::Image { + "image" + } else { + "video" + }; + if !caps.supported_types.iter().any(|kind| kind == source_kind) { + return Err("selected upscaler does not support the source type".to_string()); + } + let source_path = resolve_media(&snapshot, &args.media_ref, source.kind)?; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: String::new(), + model: entry.id.clone(), + duration: source.duration.max(0.0).round() as i32, + aspect_ratio: String::new(), + source_asset_id: Some(args.media_ref.clone()), + source_clip_id: args.source_clip_id.clone(), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: format!("{} 2x", source.name), + kind: source.kind, + folder_id: source.folder_id.clone(), + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: Some(args.media_ref), + source_clip_id: args.source_clip_id, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references: vec![PreparedReference { + path: source_path, + fallback: source_kind, + trim_range: source_trim, + }], + timeline_span: None, + requires_source_video: false, + model_kind: ModelKind::Upscale, + managed: !configured.contains(&provider) && managed_available, + }) + } + } + } + + async fn run_job( + self, + project_epoch: u64, + project_dir: PathBuf, + local_job_id: String, + placeholder_ids: Vec, + prepared: PreparedDispatch, + cancel: MediaCancelToken, + ) { + let result = self + .run_job_inner( + project_epoch, + &project_dir, + &local_job_id, + &placeholder_ids, + &prepared, + &cancel, + ) + .await; + if let Err(code) = result { + if cancel.is_cancelled() { + self.cancel_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids); + } else { + self.fail_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids, &code); + } + } + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&local_job_id); + } + + async fn run_job_inner( + &self, + project_epoch: u64, + project_dir: &Path, + local_job_id: &str, + placeholder_ids: &[String], + prepared: &PreparedDispatch, + cancel: &MediaCancelToken, + ) -> Result<(), String> { + cancelled(cancel)?; + let client = self + .clients + .build(&prepared.plan.provider, prepared.managed)?; + let mut references = prepared.references.clone(); + let timeline_cleanup = if let Some(span) = prepared.timeline_span.as_ref() { + std::fs::create_dir_all(&self.staging_root) + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())?; + let destination = self.staging_root.join(format!( + "{local_job_id}-{}.timeline.mp4", + uuid::Uuid::new_v4() + )); + let timeline = span.timeline.clone(); + let manifest = span.manifest.clone(); + let project_dir = span.project_dir.clone(); + let start_frame = span.start_frame; + let end_frame = span.end_frame; + let output = destination.clone(); + let export_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + crate::export::run_export_with_control( + &timeline, + &manifest, + &project_dir, + &crate::export::ExportRequest { + out_path: output.to_string_lossy().into_owned(), + codec: crate::export::ExportCodec::default(), + quality: crate::export::ExportQuality::default(), + }, + crate::export::ExportRunOptions { + external_cancel: Some(export_cancel), + frame_range: Some((start_frame, end_frame)), + ..crate::export::ExportRunOptions::default() + }, + ) + }) + .await + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())? + .map_err(|error| { + if error == crate::export::CANCELLED_SENTINEL { + "GENERATION_CANCELLED".to_string() + } else { + "GENERATION_SOURCE_PREPROCESS_FAILED".to_string() + } + })?; + references.push(PreparedReference::whole(destination.clone(), "video")); + Some(StagedCleanup::new(destination)) + } else { + None + }; + let mut uploaded = Vec::with_capacity(references.len()); + for reference in &references { + cancelled(cancel)?; + let staged_trim = if let Some((start, end)) = reference.trim_range { + let destination = self + .staging_root + .join(format!("{local_job_id}-{}.trim.mp4", uuid::Uuid::new_v4())); + let source = reference.path.clone(); + let output = destination.clone(); + let trim_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + opentake_media::trim_video_range(&source, &output, start, end, &trim_cancel) + }) + .await + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())? + .map_err(|error| { + if matches!(error, opentake_media::MediaError::Cancelled) { + "GENERATION_CANCELLED".to_string() + } else { + "GENERATION_SOURCE_PREPROCESS_FAILED".to_string() + } + })?; + Some(destination) + } else { + None + }; + let upload_path = staged_trim.as_deref().unwrap_or(&reference.path); + let content_type = opentake_gen::content_type_for(upload_path, reference.fallback); + let uploaded_url = if prepared.managed { + client.upload_reference(upload_path, &content_type).await + } else { + client + .upload_reference_via(&prepared.plan.provider, upload_path, &content_type) + .await + }; + if let Some(path) = staged_trim { + let _ = std::fs::remove_file(path); + } + let uploaded_url = uploaded_url.map_err(|error| { + generation_provider_error_code(&error, "GENERATION_REFERENCE_UPLOAD_FAILED") + })?; + uploaded.push(uploaded_url); + } + drop(timeline_cleanup); + cancelled(cancel)?; + let params = build_params( + &prepared.plan.input, + &uploaded, + prepared.model_kind, + prepared.requires_source_video, + ); + let provider_job_id = if prepared.managed { + client + .submit(&prepared.plan.input.model, params, Some(local_job_id)) + .await + } else { + client.submit_byok(&prepared.plan.input.model, params).await + } + .map_err(|error| generation_provider_error_code(&error, "GENERATION_SUBMIT_FAILED"))?; + self.core + .update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.15), + error_code: None, + provider_job_id: Some(provider_job_id.clone()), + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ) + .map_err(|_| "GENERATION_STATE_PERSIST_FAILED".to_string())?; + + self.watch_and_finalize( + project_epoch, + project_dir, + local_job_id, + placeholder_ids, + client, + &provider_job_id, + cancel, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn watch_and_finalize( + &self, + project_epoch: u64, + project_dir: &Path, + local_job_id: &str, + placeholder_ids: &[String], + client: GenClient, + provider_job_id: &str, + cancel: &MediaCancelToken, + ) -> Result<(), String> { + let stream = client.watch(provider_job_id); + futures_util::pin_mut!(stream); + loop { + cancelled(cancel)?; + let next = tokio::select! { + item = stream.next() => item, + () = wait_for_cancel(cancel) => return Err("GENERATION_CANCELLED".to_string()), + }; + let job = next + .ok_or_else(|| "GENERATION_PROVIDER_STREAM_ENDED".to_string())? + .map_err(|error| { + generation_provider_error_code(&error, "GENERATION_PROVIDER_POLL_FAILED") + })?; + match job.status { + JobStatus::Queued => {} + JobStatus::Running => { + let _ = self.core.update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.5), + error_code: None, + provider_job_id: Some(provider_job_id.to_string()), + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ); + } + JobStatus::Failed => return Err("GENERATION_PROVIDER_FAILED".to_string()), + JobStatus::Succeeded => { + self.core + .update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Downloading, + progress: Some(0.8), + error_code: None, + provider_job_id: Some(provider_job_id.to_string()), + cost_credits: job.cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + ) + .map_err(|_| "GENERATION_STATE_PERSIST_FAILED".to_string())?; + let store = TauriFinalizationStore { + bridge: self.clone(), + project_epoch, + project_dir: project_dir.to_path_buf(), + }; + let staging_root = self.staging_root.clone(); + let urls = job.result_urls.unwrap_or_default(); + let terminal_job_id = local_job_id.to_string(); + let terminal_placeholder_ids = placeholder_ids.to_vec(); + let download_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + let downloader = + SecureResultDownloader::new(staging_root, download_cancel)?; + finalize_terminal_outputs( + &store, + &downloader, + &terminal_job_id, + &terminal_placeholder_ids, + &urls, + ) + }) + .await + .map_err(|_| "GENERATION_FINALIZE_TASK_FAILED".to_string())? + .map_err(|error| { + if error == "GENERATION_CANCELLED" { + error + } else { + "GENERATION_FINALIZE_FAILED".to_string() + } + })?; + return Ok(()); + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn run_recovered_job( + self, + project_epoch: u64, + project_dir: PathBuf, + local_job_id: String, + placeholder_ids: Vec, + provider: String, + managed: bool, + provider_job_id: String, + cancel: MediaCancelToken, + ) { + let result = match self.clients.build(&provider, managed) { + Ok(client) => { + self.watch_and_finalize( + project_epoch, + &project_dir, + &local_job_id, + &placeholder_ids, + client, + &provider_job_id, + &cancel, + ) + .await + } + Err(_) => Err("GENERATION_RECOVERY_AUTH_UNAVAILABLE".to_string()), + }; + if let Err(code) = result { + if cancel.is_cancelled() { + self.cancel_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids); + } else { + self.fail_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids, &code); + } + } + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&local_job_id); + } +} + +impl GenerationBridge for TauriGenerationBridge { + fn can_generate(&self) -> bool { + !self.configured_byok_prefixes().is_empty() || self.has_managed_credential() + } + + fn submit( + &self, + request: GenerationRequest, + cancel: &MediaCancelToken, + ) -> Result { + cancelled(cancel)?; + let prepared = self.prepare(request)?; + let snapshot = self.core.runtime_snapshot(); + let project_dir = snapshot + .project_dir + .clone() + .ok_or_else(|| "Save the project before starting generation".to_string())?; + let committed = self + .core + .begin_generation_job_for_project( + snapshot.project_epoch, + &project_dir, + prepared.plan.clone(), + ) + .map_err(|error| error.to_string())?; + let background_cancel = MediaCancelToken::new(); + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(committed.job_id.clone(), background_cancel.clone()); + let bridge = self.clone(); + let job_id = committed.job_id.clone(); + let placeholder_ids = committed.placeholder_asset_ids.clone(); + tauri::async_runtime::spawn(async move { + bridge + .run_job( + snapshot.project_epoch, + project_dir, + job_id, + placeholder_ids, + prepared, + background_cancel, + ) + .await; + }); + Ok(GenerationSubmission { + job_id: committed.job_id, + placeholder_asset_ids: committed.placeholder_asset_ids, + status: "queued".to_string(), + }) + } +} + +#[derive(Clone)] +struct TauriFinalizationStore { + bridge: TauriGenerationBridge, + project_epoch: u64, + project_dir: PathBuf, +} + +impl GenerationFinalizationStore for TauriFinalizationStore { + fn claim_terminal(&self, job_id: &str) -> Result { + if self + .bridge + .runtime + .completed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains(job_id) + { + return Ok(false); + } + Ok(self + .bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.to_string())) + } + + fn release_terminal(&self, job_id: &str) -> Result<(), String> { + self.bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(job_id); + Ok(()) + } + + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String> { + let snapshot = self.bridge.core.runtime_snapshot(); + let entry = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| "generation placeholder disappeared".to_string())?; + let probe = self + .bridge + .engine + .probe(&artifact.path) + .map_err(|_| "downloaded generation result could not be probed".to_string())?; + let actual_kind = if probe.has_video { + if probe.duration_secs > 0.0 { + ClipType::Video + } else { + ClipType::Image + } + } else if probe.has_audio { + ClipType::Audio + } else { + return Err("downloaded generation result has no supported stream".to_string()); + }; + if actual_kind != entry.kind { + return Err("downloaded generation result has the wrong media type".to_string()); + } + if let Some(source_id) = entry + .generation_input + .as_ref() + .and_then(|input| input.source_asset_id.as_deref()) + { + let source = snapshot + .media + .entries + .iter() + .find(|source| source.id == source_id) + .ok_or_else(|| "upscale source disappeared".to_string())?; + if let (Some(source_width), Some(source_height), Some(width), Some(height)) = ( + source.source_width, + source.source_height, + probe.width, + probe.height, + ) { + if width as i32 != source_width.saturating_mul(2) + || height as i32 != source_height.saturating_mul(2) + { + return Err( + "upscale result is not exactly 2x the source dimensions".to_string() + ); + } + } + } + let extension = result_extension( + &artifact.media_type, + actual_kind, + probe.format_name.as_deref(), + )?; + let leaf = format!("{asset_id}.{extension}"); + let mut source = std::fs::File::open(&artifact.path).map_err(|error| error.to_string())?; + self.bridge + .core + .finalize_generation_output_with_media_for_project( + self.project_epoch, + &self.project_dir, + PreparedGenerationOutput { + asset_id: asset_id.to_string(), + relative_path: format!("media/{leaf}"), + probe: ProbedMedia { + duration_secs: probe.duration_secs, + width: probe.width.map(|value| value as i32), + height: probe.height.map(|value| value as i32), + fps: probe.fps, + has_audio: probe.has_audio, + }, + created_at: Some(now_apple_reference_seconds()), + }, + &leaf, + artifact.byte_size, + &mut source, + ) + .map_err(|error| error.to_string())?; + let _ = std::fs::remove_file(&artifact.path); + Ok(()) + } + + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String> { + self.bridge + .core + .fail_generation_output_for_project( + self.project_epoch, + &self.project_dir, + asset_id, + code, + Some(now_apple_reference_seconds()), + ) + .map_err(|error| error.to_string()) + } + + fn complete_job(&self, job_id: &str, _succeeded: usize, _failed: usize) -> Result<(), String> { + self.bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(job_id); + self.bridge + .runtime + .completed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.to_string()); + Ok(()) + } +} + +struct SecureResultDownloader { + client: reqwest::blocking::Client, + staging_root: PathBuf, + cancel: MediaCancelToken, +} + +impl SecureResultDownloader { + fn new(staging_root: PathBuf, cancel: MediaCancelToken) -> Result { + std::fs::create_dir_all(&staging_root) + .map_err(|_| "generation staging directory is unavailable".to_string())?; + let client = reqwest::blocking::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(10 * 60)) + .build() + .map_err(|_| "generation result client initialization failed".to_string())?; + Ok(Self { + client, + staging_root, + cancel, + }) + } + + fn write_staging( + &self, + asset_id: &str, + media_type: String, + bytes: &[u8], + ) -> Result { + if bytes.len() as u64 > RESULT_BYTES_MAX { + return Err("generation result exceeds the download limit".to_string()); + } + let path = self + .staging_root + .join(format!("{asset_id}-{}.download", uuid::Uuid::new_v4())); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + let mut file = options + .open(&path) + .map_err(|_| "generation staging file creation failed".to_string())?; + let cleanup = StagedCleanup::new(path.clone()); + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|_| "generation staging write failed".to_string())?; + cleanup.preserve(); + Ok(DownloadedGenerationArtifact { + path, + media_type, + byte_size: bytes.len() as u64, + }) + } +} + +impl GenerationArtifactDownloader for SecureResultDownloader { + fn download( + &self, + asset_id: &str, + raw_url: &str, + ) -> Result { + cancelled(&self.cancel)?; + if raw_url.starts_with("data:") { + if raw_url.len() > DATA_URL_ENCODED_MAX { + return Err("generation data URL exceeds the download limit".to_string()); + } + let (header, encoded) = raw_url + .split_once(',') + .ok_or_else(|| "generation data URL is malformed".to_string())?; + let media_type = header + .strip_prefix("data:") + .and_then(|value| value.strip_suffix(";base64")) + .filter(|value| { + value.starts_with("image/") + || value.starts_with("audio/") + || value.starts_with("video/") + }) + .ok_or_else(|| "generation data URL media type is unsupported".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| "generation data URL base64 is invalid".to_string())?; + return self.write_staging(asset_id, media_type.to_string(), &bytes); + } + + let mut current = validate_result_https_url(raw_url)?; + for redirect_count in 0..=RESULT_REDIRECT_MAX { + cancelled(&self.cancel)?; + let mut response = self + .client + .get(current.clone()) + .send() + .map_err(|_| "generation result download failed".to_string())?; + if response + .remote_addr() + .is_some_and(|address| !is_public_result_ip(address.ip())) + { + return Err("generation result resolved to a private address".to_string()); + } + if response.status().is_redirection() { + if redirect_count == RESULT_REDIRECT_MAX { + return Err("generation result exceeded redirect limit".to_string()); + } + let location = response + .headers() + .get(reqwest::header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| "generation result redirect is invalid".to_string())?; + current = validate_result_https_url( + current + .join(location) + .map_err(|_| "generation result redirect is invalid".to_string())? + .as_str(), + )?; + continue; + } + if !response.status().is_success() { + return Err("generation result download returned an error".to_string()); + } + if response + .content_length() + .is_some_and(|length| length > RESULT_BYTES_MAX) + { + return Err("generation result exceeds the download limit".to_string()); + } + let media_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .unwrap_or("application/octet-stream") + .trim() + .to_ascii_lowercase(); + let path = self + .staging_root + .join(format!("{asset_id}-{}.download", uuid::Uuid::new_v4())); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|_| "generation staging file creation failed".to_string())?; + let cleanup = StagedCleanup::new(path.clone()); + let mut total = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + loop { + cancelled(&self.cancel)?; + let count = response + .read(&mut buffer) + .map_err(|_| "generation result stream failed".to_string())?; + if count == 0 { + break; + } + total = total.saturating_add(count as u64); + if total > RESULT_BYTES_MAX { + return Err("generation result exceeds the download limit".to_string()); + } + file.write_all(&buffer[..count]) + .map_err(|_| "generation staging write failed".to_string())?; + } + file.sync_all() + .map_err(|_| "generation staging write failed".to_string())?; + cleanup.preserve(); + return Ok(DownloadedGenerationArtifact { + path, + media_type, + byte_size: total, + }); + } + Err("generation result download failed".to_string()) + } +} + +fn build_client(provider: &str, managed: bool) -> Result { + if managed { + let (backend, token) = crate::account::generation_credential()? + .ok_or_else(|| "managed generation credential is unavailable".to_string())?; + let base = reqwest::Url::parse(&(backend + "/")) + .map_err(|_| "managed generation backend is invalid".to_string())?; + return Ok(GenClient::managed(base, Arc::new(StaticToken(token)))); + } + let store = KeyringStore::new(); + let key = match provider { + "fal" => ProviderKey::Fal, + "replicate" => ProviderKey::Replicate, + "openai" => ProviderKey::OpenAI, + "elevenlabs" => ProviderKey::ElevenLabs, + _ => return Err("generation provider is unsupported".to_string()), + }; + let secret = (&store as &dyn KeyStore) + .load_key(key) + .map_err(|_| "generation provider key could not be loaded".to_string())? + .ok_or_else(|| "generation provider key is not configured".to_string())?; + let transport = Arc::new(ReqwestTransport::new()); + let registry = match provider { + "fal" => ProviderRegistry::new().with(Arc::new(FalAdapter::new(transport, secret))), + "replicate" => { + ProviderRegistry::new().with(Arc::new(ReplicateAdapter::new(transport, secret))) + } + "openai" => ProviderRegistry::new().with(Arc::new(OpenAiAdapter::new(transport, secret))), + "elevenlabs" => { + ProviderRegistry::new().with(Arc::new(ElevenLabsAdapter::new(transport, secret))) + } + _ => unreachable!(), + }; + Ok(GenClient::byok(registry, Catalog::builtin())) +} + +fn generation_provider_error_code(error: &GenError, fallback: &str) -> String { + match error { + GenError::Unauthenticated | GenError::NotConfigured => "GENERATION_AUTH_FAILED", + GenError::InsufficientCredits(_) => "GENERATION_INSUFFICIENT_CREDITS", + GenError::Api { status: 429, .. } => "GENERATION_RATE_LIMITED", + _ => fallback, + } + .to_string() +} + +fn select_model<'a>( + catalog: &'a Catalog, + kind: ModelKind, + requested: Option<&str>, + configured: &BTreeSet, + managed: bool, +) -> Result<&'a CatalogEntry, String> { + if let Some(requested) = requested { + let entry = catalog + .by_id(requested) + .ok_or_else(|| "generation model does not exist".to_string())?; + if entry.kind != kind { + return Err("generation model has the wrong media kind".to_string()); + } + let provider = provider_prefix(&entry.id)?; + if !managed && !configured.contains(&provider) { + return Err("selected model provider is not configured".to_string()); + } + return Ok(entry); + } + catalog + .entries() + .iter() + .find(|entry| { + entry.kind == kind + && (managed + || provider_prefix(&entry.id) + .ok() + .is_some_and(|provider| configured.contains(&provider))) + }) + .ok_or_else(|| "no configured provider supports this generation type".to_string()) +} + +fn resolve_media( + snapshot: &opentake_core::ProjectRuntimeSnapshot, + media_ref: &str, + expected_kind: ClipType, +) -> Result { + let entry = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == media_ref) + .ok_or_else(|| format!("referenced media does not exist: {media_ref}"))?; + if entry.kind != expected_kind { + return Err(format!("referenced media has the wrong type: {media_ref}")); + } + let path = MediaResolver::new(&snapshot.media, snapshot.project_dir.as_deref()) + .expected_path(media_ref) + .ok_or_else(|| format!("referenced media cannot be resolved: {media_ref}"))?; + if !path.is_file() { + return Err(format!("referenced media is offline: {media_ref}")); + } + Ok(path) +} + +fn validate_source_clip( + timeline: &Timeline, + clip_id: Option<&str>, + media_ref: Option<&str>, +) -> Result, String> { + let Some(clip_id) = clip_id else { + return Ok(None); + }; + let clip = timeline + .tracks + .iter() + .flat_map(|track| &track.clips) + .find(|clip| clip.id == clip_id) + .ok_or_else(|| "sourceClipId does not exist".to_string())?; + if media_ref.is_some_and(|media_ref| clip.media_ref != media_ref) { + return Err("sourceClipId does not reference the requested media".to_string()); + } + if timeline.fps <= 0 || clip.duration_frames <= 0 { + return Err("sourceClipId has no valid visible source range".to_string()); + } + let start = clip.trim_start_frame.max(0) as f64 / timeline.fps as f64; + let consumed = clip.source_frames_consumed(); + if consumed <= 0 { + return Err("sourceClipId has no valid visible source range".to_string()); + } + Ok(Some((start, start + consumed as f64 / timeline.fps as f64))) +} + +fn validate_choice( + field: &str, + value: Option<&str>, + allowed: Option<&[String]>, +) -> Result<(), String> { + if let (Some(value), Some(allowed)) = (value, allowed) { + if !allowed.is_empty() && !allowed.iter().any(|candidate| candidate == value) { + return Err(format!("{field} is not supported by the selected model")); + } + } + Ok(()) +} + +fn validate_reference_count(label: &str, count: usize, max: u32) -> Result<(), String> { + if count > max as usize { + Err(format!( + "too many {label} references for the selected model" + )) + } else { + Ok(()) + } +} + +fn provider_prefix(model: &str) -> Result { + ModelRoute::parse(model) + .map(|route| route.prefix) + .map_err(|_| "generation model id is invalid".to_string()) +} + +fn display_name(requested: Option<&str>, prompt: &str, fallback: &str) -> String { + requested + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| { + let value = prompt.trim().chars().take(30).collect::(); + (!value.is_empty()).then_some(value) + }) + .unwrap_or_else(|| fallback.to_string()) +} + +fn result_extension( + media_type: &str, + kind: ClipType, + format_name: Option<&str>, +) -> Result<&'static str, String> { + if let Some(format) = format_name { + let formats = format.split(',').collect::>(); + let detected = match kind { + ClipType::Image if formats.contains("png_pipe") => Some("png"), + ClipType::Image if formats.contains("jpeg_pipe") || formats.contains("image2") => { + Some("jpg") + } + ClipType::Image if formats.contains("webp_pipe") => Some("webp"), + ClipType::Video if formats.contains("mov") || formats.contains("mp4") => Some("mp4"), + ClipType::Audio if formats.contains("mp3") => Some("mp3"), + ClipType::Audio if formats.contains("wav") => Some("wav"), + ClipType::Audio if formats.contains("mov") || formats.contains("mp4") => Some("m4a"), + _ => None, + }; + if let Some(extension) = detected { + return Ok(extension); + } + } + match media_type { + "image/png" => Ok("png"), + "image/jpeg" | "image/jpg" => Ok("jpg"), + "image/webp" => Ok("webp"), + "video/mp4" => Ok("mp4"), + "video/quicktime" => Ok("mov"), + "audio/mpeg" => Ok("mp3"), + "audio/wav" | "audio/x-wav" => Ok("wav"), + "audio/mp4" => Ok("m4a"), + "application/octet-stream" => match kind { + ClipType::Image => Ok("png"), + ClipType::Video => Ok("mp4"), + ClipType::Audio => Ok("mp3"), + ClipType::Text | ClipType::Lottie => { + Err("generated media type is unsupported".to_string()) + } + }, + _ => Err("generation result content type is unsupported".to_string()), + } +} + +fn validate_result_https_url(raw: &str) -> Result { + let url = + reqwest::Url::parse(raw).map_err(|_| "generation result URL is invalid".to_string())?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.port().is_some_and(|port| port != 443) + { + return Err("generation result URL is not an accepted HTTPS URL".to_string()); + } + let host = url.host_str().unwrap_or_default().to_ascii_lowercase(); + if host == "localhost" + || host.ends_with(".localhost") + || host.ends_with(".local") + || host.ends_with(".internal") + || host + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok_and(|address| !is_public_result_ip(address)) + { + return Err("generation result URL host is not public".to_string()); + } + Ok(url) +} + +fn is_public_result_ip(address: std::net::IpAddr) -> bool { + match address { + std::net::IpAddr::V4(ip) => { + let [a, b, _, _] = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.is_multicast() + || a == 0 + || (a == 100 && (64..=127).contains(&b)) + || (a == 198 && (18..=19).contains(&b))) + } + std::net::IpAddr::V6(ip) => { + if let Some(mapped) = ip.to_ipv4_mapped() { + return is_public_result_ip(std::net::IpAddr::V4(mapped)); + } + let octets = ip.octets(); + !(ip.is_loopback() + || ip.is_unspecified() + || ip.is_unique_local() + || ip.is_unicast_link_local() + || ip.is_multicast() + || octets[..4] == [0x20, 0x01, 0x0d, 0xb8] + || (octets[0] == 0xfe && octets[1] & 0xc0 == 0xc0)) + } + } +} + +fn cancelled(cancel: &MediaCancelToken) -> Result<(), String> { + if cancel.is_cancelled() { + Err("GENERATION_CANCELLED".to_string()) + } else { + Ok(()) + } +} + +async fn wait_for_cancel(cancel: &MediaCancelToken) { + while !cancel.is_cancelled() { + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn now_apple_reference_seconds() -> f64 { + const APPLE_REFERENCE_UNIX_OFFSET: f64 = 978_307_200.0; + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs_f64() - APPLE_REFERENCE_UNIX_OFFSET) + .unwrap_or(0.0) +} + +#[tauri::command] +pub fn generation_cancel( + bridge: tauri::State<'_, Arc>, + job_id: String, +) -> Result { + Ok(bridge.cancel(&job_id)) +} + +#[tauri::command] +pub fn generation_retry( + bridge: tauri::State<'_, Arc>, + job_id: String, + cost_authorized: bool, +) -> Result { + bridge.retry(&job_id, cost_authorized) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + use image::{DynamicImage, ImageFormat}; + use opentake_agent::mcp::core_handle::AppCoreHandle; + use opentake_agent::mcp::dispatch::Dispatcher; + use opentake_agent::plugin::registry::PluginRegistry; + use opentake_gen::{AuthMode, FalAdapter, HttpResponse, Method, MockTransport}; + use opentake_project::{GenerationLog, Project}; + use serde_json::json; + use std::sync::RwLock; + + #[derive(Clone)] + struct FixtureClients { + client: GenClient, + } + + impl GenerationClientFactory for FixtureClients { + fn configured_byok_prefixes(&self) -> BTreeSet { + BTreeSet::from([ + "fal".to_string(), + "openai".to_string(), + "replicate".to_string(), + ]) + } + + fn has_managed_credential(&self) -> bool { + false + } + + fn build(&self, _provider: &str, _managed: bool) -> Result { + Ok(self.client.clone()) + } + } + + fn fixture_client_with_interval(mock: &MockTransport, interval: Duration) -> GenClient { + let transport = Arc::new(mock.clone()); + let fal = FalAdapter::new(transport.clone(), "fixture-secret").with_base("https://mockfal"); + let openai = + OpenAiAdapter::new(transport.clone(), "fixture-secret").with_base("https://mockoai/v1"); + let replicate = + ReplicateAdapter::new(transport, "fixture-secret").with_base("https://mockrep/v1"); + GenClient::with_transport( + AuthMode::Byok { + registry: ProviderRegistry::new() + .with(Arc::new(fal)) + .with(Arc::new(openai)) + .with(Arc::new(replicate)), + catalog: Catalog::builtin(), + }, + Arc::new(mock.clone()), + ) + .with_poll_interval(interval) + } + + fn fixture_client(mock: &MockTransport) -> GenClient { + fixture_client_with_interval(mock, Duration::ZERO) + } + + fn saved_core() -> (tempfile::TempDir, PathBuf, AppCore) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Generation.opentake"); + let mut project = Project::new(&bundle); + project.generation_log = Some(GenerationLog::new()); + project.save().unwrap(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + (temp, bundle, core) + } + + fn image_plan() -> PreparedGenerationJob { + PreparedGenerationJob { + name: "Recovered image".to_string(), + kind: ClipType::Image, + folder_id: None, + provider: "fal".to_string(), + input: GenerationInput { + prompt: "fixture".to_string(), + model: "fal:flux-pro".to_string(), + duration: 0, + aspect_ratio: "1:1".to_string(), + num_images: Some(1), + ..Default::default() + }, + output_count: 1, + source_asset_id: None, + source_clip_id: None, + estimated_cost_credits: None, + created_at: Some(800_000_000.0), + } + } + + fn runtime_dirs(bundle: &Path) -> (PathBuf, PathBuf) { + let root = bundle.parent().unwrap(); + (root.join("cache"), root.join("models")) + } + + fn png_bytes(width: u32, height: u32) -> Vec { + let mut bytes = Cursor::new(Vec::new()); + DynamicImage::new_rgba8(width, height) + .write_to(&mut bytes, ImageFormat::Png) + .unwrap(); + bytes.into_inner() + } + + fn png_data_url_for(width: u32, height: u32) -> String { + format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png_bytes(width, height)) + ) + } + + fn png_data_url() -> String { + png_data_url_for(2, 2) + } + + fn wav_bytes() -> Vec { + let sample_rate = 8_000_u32; + let sample_count = sample_rate / 10; + let data_size = sample_count * 2; + let mut bytes = Vec::with_capacity((44 + data_size) as usize); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_size).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&sample_rate.to_le_bytes()); + bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + bytes.extend_from_slice(&2_u16.to_le_bytes()); + bytes.extend_from_slice(&16_u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_size.to_le_bytes()); + bytes.resize((44 + data_size) as usize, 0); + bytes + } + + fn mp4_data_url(directory: &Path) -> String { + let path = directory.join("generation-fixture.mp4"); + let status = std::process::Command::new("ffmpeg") + .args([ + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:d=0.1:r=10", + "-pix_fmt", + "yuv420p", + ]) + .arg(&path) + .status() + .unwrap(); + assert!(status.success()); + let encoded = + base64::engine::general_purpose::STANDARD.encode(std::fs::read(path).unwrap()); + format!("data:video/mp4;base64,{encoded}") + } + + fn saved_core_with_source_image() -> (tempfile::TempDir, PathBuf, AppCore) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Upscale.opentake"); + let mut project = Project::new(&bundle); + project.generation_log = Some(GenerationLog::new()); + project + .manifest + .entries + .push(opentake_domain::MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: opentake_domain::MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(2), + source_height: Some(2), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + std::fs::create_dir_all(bundle.join("media")).unwrap(); + std::fs::write(bundle.join("media/source.png"), png_bytes(2, 2)).unwrap(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + (temp, bundle, core) + } + + async fn wait_for_ready_model( + core: &AppCore, + model: &str, + ) -> opentake_domain::MediaManifestEntry { + for _ in 0..150 { + if let Some(entry) = core.media().entries.into_iter().find(|entry| { + entry.generation_input.as_ref().is_some_and(|input| { + input.model == model && input.status == Some(GenerationJobStatus::Ready) + }) + }) { + return entry; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!( + "generation did not become ready for {model}: {:?}", + core.media().entries + ); + } + + #[test] + fn result_url_validation_rejects_local_and_private_network_targets() { + assert!(validate_result_https_url("https://cdn.example.com/result.png").is_ok()); + for url in [ + "https://localhost/result.png", + "https://service.local/result.png", + "https://127.0.0.1/result.png", + "https://10.0.0.1/result.png", + "https://169.254.1.1/result.png", + "https://[::1]/result.png", + "https://[fc00::1]/result.png", + "https://[::ffff:127.0.0.1]/result.png", + "https://[2001:db8::1]/result.png", + ] { + assert!( + validate_result_https_url(url).is_err(), + "private result URL accepted: {url}" + ); + } + } + + #[test] + fn restart_before_provider_id_requires_explicit_retry_without_resubmission() { + let (_temp, bundle, core) = saved_core(); + let committed = core + .begin_generation_job_for_project(1, &bundle, image_plan()) + .unwrap(); + // Model a real process restart. A second independent AppCore retaining + // the old bundle at the same time is not a supported runtime state and + // prevents same-target directory publication on Windows. + drop(core); + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + reopened.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + + assert_eq!(bridge.recover_current_project(), 0); + let persisted = reopened.media(); + let input = persisted + .entries + .iter() + .find(|entry| entry.id == committed.placeholder_asset_ids[0]) + .unwrap() + .generation_input + .as_ref() + .unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Failed)); + assert_eq!( + input.error_code.as_deref(), + Some("GENERATION_RESTART_RETRY_REQUIRED") + ); + assert!(mock.calls().is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn restart_with_provider_id_resumes_and_finalizes_offline_fixture() { + let (_temp, bundle, core) = saved_core(); + let runtime = core.runtime_snapshot(); + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, image_plan()) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.25), + error_code: None, + provider_job_id: Some("fal::flux-pro|recover-1".to_string()), + cost_credits: None, + created_at: Some(800_000_001.0), + }, + ) + .unwrap(); + + // Model a real process restart before opening the same bundle again. + drop(core); + let reopened = AppCore::new(); + let reopened_snapshot = reopened.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/recover-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/recover-1", + 200, + json!({"images": [{"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + reopened.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + + assert_eq!(bridge.recover_current_project(), 1); + let asset_id = committed.placeholder_asset_ids[0].clone(); + for _ in 0..100 { + let status = reopened + .media() + .entries + .iter() + .find(|entry| entry.id == asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status); + if status == Some(GenerationJobStatus::Ready) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let entry = reopened + .media() + .entries + .into_iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Ready) + ); + assert_eq!(entry.source_width, Some(2)); + assert_eq!(entry.source_height, Some(2)); + assert!(MediaResolver::new(&reopened.media(), Some(&bundle)) + .expected_path(&entry.id) + .unwrap() + .is_file()); + assert_eq!( + reopened.project_revision().project_epoch, + reopened_snapshot.project_epoch + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_path_persists_and_finalizes_ordered_mock_results() { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "dispatch-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/dispatch-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/dispatch-1", + 200, + json!({"images": [{"url": png_data_url()}, {"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ); + + let unauthorized = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": false, + "prompt": "ordered fixture", + "model": "fal:flux-pro", + "numImages": 2 + }), + ); + assert!(unauthorized.is_error); + assert!(mock.calls().is_empty()); + + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "ordered fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1", + "numImages": 2 + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + for _ in 0..100 { + let generated = core + .media() + .entries + .into_iter() + .filter(|entry| entry.generation_input.is_some()) + .collect::>(); + if generated.len() == 2 + && generated.iter().all(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.status) + == Some(GenerationJobStatus::Ready) + }) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let mut generated = core + .media() + .entries + .into_iter() + .filter(|entry| entry.generation_input.is_some()) + .collect::>(); + generated.sort_by_key(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.output_index) + }); + assert_eq!(generated.len(), 2); + assert!( + generated.iter().all(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.status) + == Some(GenerationJobStatus::Ready) + && entry.source_width == Some(2) + && entry.source_height == Some(2) + }), + "generated outputs: {generated:?}" + ); + assert_eq!( + generated + .iter() + .filter_map(|entry| entry + .generation_input + .as_ref() + .and_then(|input| input.output_index)) + .collect::>(), + vec![0, 1] + ); + assert!(generated.iter().all(|entry| { + MediaResolver::new(&core.media(), Some(&bundle)) + .expected_path(&entry.id) + .is_some_and(|path| path.is_file()) + })); + } + + #[tokio::test(flavor = "multi_thread")] + async fn configured_provider_smoke_covers_video_audio_and_upscale() { + let (video_temp, video_bundle, video_core) = saved_core(); + let video_mock = MockTransport::new(); + video_mock.on( + Method::Post, + "https://mockfal/kling-video", + 200, + json!({"request_id": "video-1", "status": "IN_QUEUE"}), + ); + video_mock.on( + Method::Get, + "https://mockfal/kling-video/requests/video-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + video_mock.on( + Method::Get, + "https://mockfal/kling-video/requests/video-1", + 200, + json!({"video": {"url": mp4_data_url(video_temp.path())}}), + ); + let (cache, models) = runtime_dirs(&video_bundle); + let video_bridge = build_bridge_with_clients( + video_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&video_mock), + }), + ); + let video_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(video_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(video_bridge), + ); + let video_result = video_dispatcher.dispatch( + "generate_video", + json!({ + "costAuthorized": true, + "prompt": "fixture video", + "model": "fal:kling-video", + "duration": 5, + "aspectRatio": "16:9", + "resolution": "720p" + }), + ); + assert!(!video_result.is_error, "{}", video_result.text_joined()); + let video = wait_for_ready_model(&video_core, "fal:kling-video").await; + assert_eq!(video.kind, ClipType::Video); + assert_eq!(video.source_width, Some(16)); + assert_eq!(video.source_height, Some(16)); + + let (_audio_temp, audio_bundle, audio_core) = saved_core(); + let audio_mock = MockTransport::new(); + let mut audio_response = HttpResponse::new(200, wav_bytes()); + audio_response + .headers + .push(("Content-Type".to_string(), "audio/wav".to_string())); + audio_mock.on_raw( + Method::Post, + "https://mockoai/v1/audio/speech", + audio_response, + ); + let (cache, models) = runtime_dirs(&audio_bundle); + let audio_bridge = build_bridge_with_clients( + audio_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&audio_mock), + }), + ); + let audio_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(audio_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(audio_bridge), + ); + let audio_result = audio_dispatcher.dispatch( + "generate_audio", + json!({ + "costAuthorized": true, + "prompt": "fixture speech", + "model": "openai:tts-1", + "voice": "alloy" + }), + ); + assert!(!audio_result.is_error, "{}", audio_result.text_joined()); + let audio = wait_for_ready_model(&audio_core, "openai:tts-1").await; + assert_eq!(audio.kind, ClipType::Audio); + assert_eq!(audio.has_audio, Some(true)); + assert!(audio.duration > 0.0); + + let (_upscale_temp, upscale_bundle, upscale_core) = saved_core_with_source_image(); + let source_before = std::fs::read(upscale_bundle.join("media/source.png")).unwrap(); + let upscale_mock = MockTransport::new(); + upscale_mock.on( + Method::Post, + "https://mockrep/v1/files", + 200, + json!({"urls": {"get": "https://fixtures.invalid/source.png"}}), + ); + upscale_mock.on( + Method::Post, + "https://mockrep/v1/predictions", + 200, + json!({"id": "upscale-1", "status": "starting"}), + ); + upscale_mock.on( + Method::Get, + "https://mockrep/v1/predictions/upscale-1", + 200, + json!({"id": "upscale-1", "status": "succeeded", "output": png_data_url_for(4, 4)}), + ); + let (cache, models) = runtime_dirs(&upscale_bundle); + let upscale_bridge = build_bridge_with_clients( + upscale_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&upscale_mock), + }), + ); + let upscale_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(upscale_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(upscale_bridge), + ); + let upscale_result = upscale_dispatcher.dispatch( + "upscale_media", + json!({ + "costAuthorized": true, + "mediaRef": "source-image", + "model": "replicate:topaz-upscale" + }), + ); + assert!(!upscale_result.is_error, "{}", upscale_result.text_joined()); + let upscale = wait_for_ready_model(&upscale_core, "replicate:topaz-upscale").await; + assert_eq!(upscale.kind, ClipType::Image); + assert_eq!(upscale.source_width, Some(4)); + assert_eq!(upscale.source_height, Some(4)); + assert_eq!( + std::fs::read(upscale_bundle.join("media/source.png")).unwrap(), + source_before + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_cancel_terminalizes_without_importing_media() { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "cancel-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/cancel-1/status", + 200, + json!({"status": "IN_QUEUE"}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client_with_interval(&mock, Duration::from_secs(2)), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge.clone()), + ); + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "cancel fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1" + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + let (job_id, asset_id) = loop { + if let Some(entry) = core + .media() + .entries + .into_iter() + .find(|entry| entry.generation_input.is_some()) + { + let input = entry.generation_input.as_ref().unwrap(); + if input.provider_job_id.is_some() { + break (input.job_id.clone().unwrap(), entry.id); + } + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + assert!(bridge.cancel(&job_id)); + for _ in 0..100 { + let status = core + .media() + .entries + .iter() + .find(|entry| entry.id == asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status); + if status == Some(GenerationJobStatus::Cancelled) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let entry = core + .media() + .entries + .into_iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Cancelled) + ); + assert!(MediaResolver::new(&core.media(), Some(&bundle)) + .expected_path(&entry.id) + .is_some_and(|path| !path.exists())); + } + + #[test] + fn upscale_finalization_is_exactly_two_x_and_preserves_source_bytes() { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Upscale.opentake"); + let source_bytes = png_bytes(3, 2); + let mut project = Project::new(&bundle); + project + .manifest + .entries + .push(opentake_domain::MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: opentake_domain::MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(3), + source_height: Some(2), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + std::fs::create_dir_all(bundle.join("media")).unwrap(); + std::fs::write(bundle.join("media/source.png"), &source_bytes).unwrap(); + let core = AppCore::new(); + let snapshot = core.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache.clone(), + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let mut plan = image_plan(); + plan.provider = "replicate".to_string(); + plan.input.model = "replicate:topaz-upscale".to_string(); + plan.source_asset_id = Some("source-image".to_string()); + let committed = core + .begin_generation_job_for_project(snapshot.project_epoch, &bundle, plan) + .unwrap(); + core.update_generation_job_for_project( + snapshot.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.5), + error_code: None, + provider_job_id: Some("replicate::fixture".to_string()), + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + core.update_generation_job_for_project( + snapshot.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Downloading, + progress: Some(0.8), + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + std::fs::create_dir_all(&cache).unwrap(); + let staged = cache.join("upscale.png"); + let staged_bytes = png_bytes(6, 4); + std::fs::write(&staged, &staged_bytes).unwrap(); + let store = TauriFinalizationStore { + bridge: bridge.as_ref().clone(), + project_epoch: snapshot.project_epoch, + project_dir: bundle.clone(), + }; + store + .finalize_output( + &committed.placeholder_asset_ids[0], + DownloadedGenerationArtifact { + path: staged, + media_type: "image/png".to_string(), + byte_size: staged_bytes.len() as u64, + }, + ) + .unwrap(); + let media = core.media(); + let source = media + .entries + .iter() + .find(|entry| entry.id == "source-image") + .unwrap(); + let output = media + .entries + .iter() + .find(|entry| entry.id == committed.placeholder_asset_ids[0]) + .unwrap(); + assert_eq!( + (source.source_width, source.source_height), + (Some(3), Some(2)) + ); + assert_eq!( + (output.source_width, output.source_height), + (Some(6), Some(4)) + ); + assert_eq!( + std::fs::read(bundle.join("media/source.png")).unwrap(), + source_bytes + ); + } + + async fn assert_submit_failure(status: u16, body: serde_json::Value, expected_code: &str) { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on(Method::Post, "https://mockfal/flux-pro", status, body); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ); + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "failure fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1" + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + for _ in 0..100 { + let terminal = core + .media() + .entries + .iter() + .find_map(|entry| entry.generation_input.as_ref()) + .is_some_and(|input| input.status == Some(GenerationJobStatus::Failed)); + if terminal { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let input = core + .media() + .entries + .into_iter() + .find_map(|entry| entry.generation_input) + .unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Failed)); + assert_eq!(input.error_code.as_deref(), Some(expected_code)); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_maps_auth_and_rate_limit_to_safe_fixed_codes() { + assert_submit_failure( + 401, + json!({"error": {"code": "unauthenticated", "message": "private"}}), + "GENERATION_AUTH_FAILED", + ) + .await; + assert_submit_failure( + 429, + json!({"error": {"code": "rate_limited", "message": "private"}}), + "GENERATION_RATE_LIMITED", + ) + .await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn retry_requires_fresh_cost_authorization_and_creates_a_new_job() { + let (_temp, bundle, core) = saved_core(); + let runtime = core.runtime_snapshot(); + let failed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, image_plan()) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &failed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Failed, + progress: None, + error_code: Some("GENERATION_PROVIDER_FAILED".to_string()), + provider_job_id: None, + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "retry-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/retry-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/retry-1", + 200, + json!({"images": [{"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + assert!(bridge.retry(&failed.job_id, false).is_err()); + assert!(mock.calls().is_empty()); + let retried = bridge.retry(&failed.job_id, true).unwrap(); + assert_ne!(retried.job_id, failed.job_id); + for _ in 0..100 { + let ready = core.media().entries.iter().any(|entry| { + entry.generation_input.as_ref().is_some_and(|input| { + input.job_id.as_deref() == Some(retried.job_id.as_str()) + && input.status == Some(GenerationJobStatus::Ready) + }) + }); + if ready { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let statuses = core + .media() + .entries + .iter() + .filter_map(|entry| entry.generation_input.as_ref()) + .map(|input| (input.job_id.clone().unwrap(), input.status.unwrap())) + .collect::>(); + assert_eq!( + statuses.get(&failed.job_id), + Some(&GenerationJobStatus::Failed) + ); + assert_eq!( + statuses.get(&retried.job_id), + Some(&GenerationJobStatus::Ready) + ); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 66eb206f..6ba94615 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -14,6 +14,7 @@ mod commands; // drive the export orchestrator (`export::run_export`) against the library // target. The Tauri command itself is registered below like the other modules. pub mod export; +mod generation; mod haptic; mod library; mod mcp; @@ -116,17 +117,21 @@ pub fn run() { .app_data_dir() .unwrap_or_else(|_| std::env::temp_dir()) .join("workflows"); + let generation_bridge = + generation::build_bridge(core.clone(), cache_root.clone(), models_dir.clone()); mcp::spawn( core.clone(), workflows_dir.clone(), cache_root.clone(), models_dir.clone(), + generation_bridge.clone(), ); - let chat_state = chat::ChatState::new( + let chat_state = chat::ChatState::new_with_generation( core.clone(), workflows_dir, cache_root.clone(), models_dir.clone(), + generation_bridge.clone(), ); // A global favorite must never silently become a temporary file. @@ -142,6 +147,7 @@ pub fn run() { }; app.manage(core); + app.manage(generation_bridge); app.manage(chat_state); app.manage(MediaState::new(engine)); app.manage(PrewarmScheduler::new(initial_project_epoch)); @@ -207,6 +213,8 @@ pub fn run() { export::export_video, export::save_range_as_media, export::cancel_export, + generation::generation_cancel, + generation::generation_retry, secret::secret_save, secret::secret_load, secret::secret_delete, @@ -315,6 +323,11 @@ fn forward_event(app: &tauri::AppHandle, event: &CoreEvent) { if let Some(prewarm) = app.try_state::() { prewarm.activate_project(*project_epoch); } + if let Some(generation) = + app.try_state::>() + { + generation.recover_current_project(); + } } #[cfg(feature = "playback-engine")] { diff --git a/src-tauri/src/library.rs b/src-tauri/src/library.rs index 183530fd..4c0cfa53 100644 --- a/src-tauri/src/library.rs +++ b/src-tauri/src/library.rs @@ -621,11 +621,14 @@ impl ProjectMediaCapability { use cap_std::fs::OpenOptionsExt; use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE}; use windows_sys::Win32::Storage::FileSystem::{ - DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + DELETE, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, }; options .access_mode(GENERIC_READ | GENERIC_WRITE | DELETE) - .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE); + // Keep the retained import compatible with a whole-bundle + // rename on Windows while preserving identity validation and + // handle-relative rollback after a namespace move. + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); } let file = self .media diff --git a/src-tauri/src/mcp.rs b/src-tauri/src/mcp.rs index 2b7c2009..95422a0a 100644 --- a/src-tauri/src/mcp.rs +++ b/src-tauri/src/mcp.rs @@ -30,6 +30,7 @@ use std::io::Read; use base64::Engine as _; use opentake_agent::mcp::core_handle::{AppCoreHandle, CoreHandle}; +use opentake_agent::mcp::generation::GenerationBridge; use opentake_agent::mcp::media_bridge::{ BridgeError, ImportOutcome, ImportSource, InspectMediaRequest, InspectMediaResult, InspectResult, InspectedFrame, InspectedMediaFrame, MediaBridge, SearchCandidate, @@ -252,7 +253,13 @@ pub(crate) fn build_registry(workflows_dir: &Path) -> PluginRegistry { /// the same paths the UI's [`MediaEngine`] uses, so the bridge's imports land in /// the same caches. A bind failure (port in use) is logged, not fatal — the app /// keeps running without the agent network face. -pub fn spawn(core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_dir: PathBuf) { +pub fn spawn( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + generation_bridge: Arc, +) { let handle: Arc = Arc::new(AppCoreHandle::new(core.clone())); let bridge = build_media_bridge(core, cache_root, models_dir); let registry = Arc::new(RwLock::new(build_registry(&workflows_dir))); @@ -264,7 +271,15 @@ pub fn spawn(core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_ return; } }; - if let Err(e) = server::serve_with_bridge(addr, handle, registry, Some(bridge)).await { + if let Err(e) = server::serve_with_bridges( + addr, + handle, + registry, + Some(bridge), + Some(generation_bridge), + ) + .await + { eprintln!("[mcp] server stopped: {e}"); } }); diff --git a/src-tauri/src/media.rs b/src-tauri/src/media.rs index dd1eaf95..b47a39e5 100644 --- a/src-tauri/src/media.rs +++ b/src-tauri/src/media.rs @@ -35,7 +35,8 @@ use opentake_core::{ PreparedMediaFolderRef, PreparedMediaImportOp, ProbedMedia, }; use opentake_domain::{ - ClipType, GenerationInput, MediaManifest, MediaManifestEntry, MediaSource, Timeline, + ClipType, GenerationInput, GenerationJobStatus, MediaManifest, MediaManifestEntry, MediaSource, + Timeline, }; use opentake_media::library::{FavoriteRequest, LibraryStore, PreparedFavorite}; #[cfg(test)] @@ -116,6 +117,10 @@ pub struct MediaItemDto { /// always `None` in practice — the Inspector renders those sections only /// when it is present, matching upstream's `if let gen = asset.generationInput`. pub generation_input: Option, + /// Durable async lifecycle projected from `generation_input`. + pub generation_status: String, + pub generation_progress: Option, + pub generation_error_code: Option, /// `true` when the asset's source file is not on disk (moved / deleted / /// offline). Derived from file existence on every read (mirrors upstream /// `MediaResolver.isMissing`), so it clears automatically once a `relink_media` @@ -144,9 +149,31 @@ impl MediaItemDto { // produced by importing (always external) but handled for safety. MediaSource::Project { .. } => None, }; - // Missing = we can resolve a local source path and it doesn't exist. + let generation_input = entry.generation_input.as_ref(); + let generation_status = match generation_input.and_then(|input| input.status) { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => "generating", + Some(GenerationJobStatus::Downloading | GenerationJobStatus::Finalizing) => { + "downloading" + } + Some(GenerationJobStatus::Failed) => "failed", + Some(GenerationJobStatus::Cancelled) => "cancelled", + Some(GenerationJobStatus::Ready) | None => "none", + } + .to_string(); + let generation_pending = matches!( + generation_input.and_then(|input| input.status), + Some( + GenerationJobStatus::Queued + | GenerationJobStatus::Generating + | GenerationJobStatus::Downloading + | GenerationJobStatus::Finalizing + ) + ); + // Missing = a finalized source can resolve and is absent. Pending + // placeholders intentionally have no file yet and are not "offline". // An unresolvable (e.g. remote-only) source is not flagged missing. - let missing = resolved.as_ref().map(|p| !p.exists()).unwrap_or(false); + let missing = + !generation_pending && resolved.as_ref().map(|p| !p.exists()).unwrap_or(false); let thumbnail = if missing { None } else { @@ -177,6 +204,9 @@ impl MediaItemDto { folder_id: entry.folder_id.clone(), file_size, generation_input: entry.generation_input.clone(), + generation_status, + generation_progress: generation_input.and_then(|input| input.progress), + generation_error_code: generation_input.and_then(|input| input.error_code.clone()), missing, favorite, } @@ -4102,6 +4132,9 @@ mod tests { folder_id: None, file_size: Some(2048), generation_input: None, + generation_status: "none".into(), + generation_progress: None, + generation_error_code: None, missing: false, favorite: true, }; diff --git a/src-tauri/src/secret.rs b/src-tauri/src/secret.rs index 061c763e..bdc52366 100644 --- a/src-tauri/src/secret.rs +++ b/src-tauri/src/secret.rs @@ -22,14 +22,15 @@ pub struct SecretStatus { masked: String, } -/// Allowed BYOK chat providers → stable keychain account strings, following the -/// `-api-key` convention from [`opentake_gen::keys`]. Validating the -/// provider here means an unknown value can never address an arbitrary keychain -/// entry — the only writable accounts are the three the UI offers. +/// Allowed chat + generation providers → fixed keychain accounts. Validating +/// here prevents arbitrary keychain account access from the WebView. fn account_for(provider: &str) -> Result<&'static str, String> { match provider { "anthropic" => Ok("anthropic-api-key"), + "fal" => Ok("fal-api-key"), + "replicate" => Ok("replicate-api-key"), "openai" => Ok("openai-api-key"), + "elevenlabs" => Ok("elevenlabs-api-key"), "google" => Ok("google-api-key"), other => Err(format!("unknown provider: {other}")), } @@ -106,7 +107,10 @@ mod tests { #[test] fn account_mapping_is_stable_for_known_providers() { assert_eq!(account_for("anthropic").unwrap(), "anthropic-api-key"); + assert_eq!(account_for("fal").unwrap(), "fal-api-key"); + assert_eq!(account_for("replicate").unwrap(), "replicate-api-key"); assert_eq!(account_for("openai").unwrap(), "openai-api-key"); + assert_eq!(account_for("elevenlabs").unwrap(), "elevenlabs-api-key"); assert_eq!(account_for("google").unwrap(), "google-api-key"); } diff --git a/web/src/components/media/MediaPanel.tsx b/web/src/components/media/MediaPanel.tsx index 33f7af8b..bb470839 100644 --- a/web/src/components/media/MediaPanel.tsx +++ b/web/src/components/media/MediaPanel.tsx @@ -50,7 +50,15 @@ import { BoundedCache } from "../../lib/lru"; import { childFolders, folderTrail, normalizeFolderId } from "../../lib/folderTree"; import { useProjectStore } from "../../store/projectStore"; import { addMediaToTimeline } from "../../store/editActions"; -import { extractAudio, generateThumbnail, getWaveform, preloadMedia, toggleFavorite } from "../../lib/api"; +import { + cancelGeneration, + retryGeneration, + extractAudio, + generateThumbnail, + getWaveform, + preloadMedia, + toggleFavorite, +} from "../../lib/api"; import { saveDialog } from "../../lib/dialog"; import type { MediaFolder, MediaItem } from "../../lib/types"; import { MediaTabBar, MediaSubTabBar, MATERIAL_SUB_TABS, AUDIO_SUB_TABS } from "./MediaTabBar"; @@ -807,6 +815,10 @@ function MediaCard({ item }: { item: MediaItem }) { const durationFrames = Math.round(item.duration * fps); const selected = previewMediaId === item.id; const favorite = item.favorite ?? false; + const generationActive = + item.generationStatus === "generating" || item.generationStatus === "downloading"; + const generationFailed = + item.generationStatus === "failed" || item.generationStatus === "cancelled"; const thumbnailKey = mediaThumbnailKey(item); const [lazyThumbnail, setLazyThumbnail] = useState( item.thumbnail ?? mediaThumbnailCache.get(thumbnailKey) ?? null, @@ -923,20 +935,28 @@ function MediaCard({ item }: { item: MediaItem }) { return (
{ + if (generationActive) return; setPreviewMedia(item.id); // Warm poster/sprite/waveform caches so preview + a later timeline drop // are instant instead of decoding on the interaction path. void preloadMedia(item.id); }} - onDoubleClick={() => void addMediaToTimeline(item)} + onDoubleClick={() => { + if (!generationActive && !generationFailed) void addMediaToTimeline(item); + }} onMouseEnter={() => setHovered(true)} onMouseLeave={() => setHovered(false)} title={item.name} - style={{ display: "flex", flexDirection: "column", gap: 4, cursor: "grab" }} + style={{ + display: "flex", + flexDirection: "column", + gap: 4, + cursor: generationActive ? "progress" : generationFailed ? "default" : "grab", + }} > {/* Thumbnail: generated cache image only. Missing thumbnails are requested lazily as cards enter view, so import/list commands stay cheap. */} @@ -991,6 +1011,98 @@ function MediaCard({ item }: { item: MediaItem }) { {formatTimecode(durationFrames, fps)} )} + {generationActive && ( +
event.stopPropagation()} + style={{ + position: "absolute", + inset: 0, + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: 6, + background: "rgba(15,18,24,0.76)", + color: "#fff", + textAlign: "center", + padding: 8, + }} + > + + + {item.generationStatus === "downloading" ? "正在下载结果" : "正在生成"} + {typeof item.generationProgress === "number" + ? ` ${Math.round(item.generationProgress * 100)}%` + : ""} + + {item.generationInput?.jobId && ( + + )} +
+ )} + {generationFailed && ( +
event.stopPropagation()} + style={{ + position: "absolute", + inset: 0, + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: 4, + background: "rgba(180,35,35,0.55)", + color: "#fff", + textAlign: "center", + padding: 6, + }} + > + + + {item.generationStatus === "cancelled" + ? "生成已取消" + : item.generationErrorCode ?? "GENERATION_FAILED"} + + {item.generationInput?.jobId && ( + + )} +
+ )} {/* Offline overlay: the source file is missing. Relink keeps the asset id, so the timeline clips referencing it recover (no re-import). */} {item.missing && ( diff --git a/web/src/components/settings/SettingsView.tsx b/web/src/components/settings/SettingsView.tsx index 8a6fce17..2942de6a 100644 --- a/web/src/components/settings/SettingsView.tsx +++ b/web/src/components/settings/SettingsView.tsx @@ -488,7 +488,10 @@ function ImportPane() { const PROVIDERS: Array<{ id: ByokProvider; label: string }> = [ { id: "anthropic", label: "Anthropic" }, + { id: "fal", label: "fal.ai" }, + { id: "replicate", label: "Replicate" }, { id: "openai", label: "OpenAI" }, + { id: "elevenlabs", label: "ElevenLabs" }, { id: "google", label: "Google" }, ]; diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 0539205c..9f7c475e 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -312,6 +312,23 @@ export async function cancelExport(operationId: string): Promise { if (invokeImpl) await invokeImpl("cancel_export", { operationId }); } +export async function cancelGeneration(jobId: string): Promise { + await ensureTauri(); + if (invokeImpl) return invokeImpl("generation_cancel", { jobId }); + return false; +} + +export async function retryGeneration( + jobId: string, + costAuthorized: boolean, +): Promise<{ jobId: string; placeholderAssetIds: string[]; status: string }> { + await ensureTauri(); + if (invokeImpl) { + return invokeImpl("generation_retry", { jobId, costAuthorized }); + } + throw new Error("Generation retry is available only in the desktop app"); +} + /** Progress payload for `"export://progress"`: `done` of `total` frames * composited so far. */ export interface ExportProgress { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 3af11306..5f93a599 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -587,6 +587,9 @@ export interface MediaItem { * sections (mirror of `MediaItemDto.generationInput` / upstream * `MediaAsset.generationInput`). */ generationInput?: GenerationInput | null; + generationStatus?: "none" | "generating" | "downloading" | "failed" | "cancelled"; + generationProgress?: number | null; + generationErrorCode?: string | null; /** `true` when the source file is offline (moved/deleted). Derived from file * existence on the backend; clears after a successful relink. */ missing?: boolean; @@ -610,6 +613,18 @@ export interface GenerationInput { referenceImageAssetIds?: string[] | null; referenceVideoAssetIds?: string[] | null; referenceAudioAssetIds?: string[] | null; + jobId?: string | null; + provider?: string | null; + providerJobId?: string | null; + status?: "queued" | "generating" | "downloading" | "finalizing" | "ready" | "failed" | "cancelled"; + progress?: number | null; + errorCode?: string | null; + outputIndex?: number | null; + sourceAssetId?: string | null; + sourceClipId?: string | null; + sourceStartFrame?: number | null; + sourceEndFrame?: number | null; + estimatedCostCredits?: number | null; } /** A media-library folder (flat list; nest via `parentFolderId`). */ diff --git a/web/src/store/settingsStore.ts b/web/src/store/settingsStore.ts index a5f0c36c..a544c832 100644 --- a/web/src/store/settingsStore.ts +++ b/web/src/store/settingsStore.ts @@ -11,7 +11,13 @@ import { create } from "zustand"; import { isTauri } from "../lib/api"; export type Theme = "dark" | "light"; -export type ByokProvider = "anthropic" | "openai" | "google"; +export type ByokProvider = + | "anthropic" + | "fal" + | "replicate" + | "openai" + | "elevenlabs" + | "google"; export type WindowSizeOpt = "standard" | "compact"; const LS = { @@ -31,7 +37,13 @@ function loadString(key: string): string | null { } function loadProvider(): ByokProvider { const v = loadString(LS.byokProvider); - return v === "openai" || v === "google" ? v : "anthropic"; + return v === "fal" || + v === "replicate" || + v === "openai" || + v === "elevenlabs" || + v === "google" + ? v + : "anthropic"; } function loadWindowSize(): WindowSizeOpt { if (typeof localStorage === "undefined") return "standard";