From 637f2715005d97d5c71914d8374f583c0ead28d2 Mon Sep 17 00:00:00 2001 From: baiqing Date: Wed, 29 Jul 2026 22:15:00 +0800 Subject: [PATCH 1/4] feat: finalize generation provider workflows --- Cargo.lock | 3 + crates/opentake-agent/src/chat/loop.rs | 6 +- crates/opentake-agent/src/mcp/dispatch.rs | 188 +- crates/opentake-agent/src/mcp/generation.rs | 223 ++ crates/opentake-agent/src/mcp/mod.rs | 1 + crates/opentake-agent/src/mcp/server.rs | 70 +- crates/opentake-agent/src/tools/args.rs | 12 +- .../opentake-agent/src/tools/descriptions.rs | 13 +- crates/opentake-agent/src/tools/names.rs | 9 + .../tests/generation_dispatch.rs | 285 ++ crates/opentake-core/Cargo.toml | 3 + crates/opentake-core/src/core.rs | 111 +- crates/opentake-core/src/lib.rs | 3 +- crates/opentake-core/src/session.rs | 534 ++- .../tests/generation_persistence.rs | 330 ++ crates/opentake-domain/src/lib.rs | 4 +- crates/opentake-domain/src/media.rs | 73 +- crates/opentake-gen/src/build_params.rs | 37 +- crates/opentake-media/src/lib.rs | 107 + crates/opentake-project/src/bundle.rs | 3 + crates/opentake-project/src/gen_log.rs | 83 + crates/opentake-project/src/project_root.rs | 8 + docs/architecture/BUGS.md | 10 +- docs/architecture/CAPCUT-GAP.md | 2 +- docs/architecture/FULL_PROJECT_SCAN_REPORT.md | 8 +- docs/architecture/HANDOFF-2026-07.md | 8 +- docs/architecture/MODULE-PORT-MAP.md | 2 +- docs/architecture/ROADMAP.md | 3 +- ...gent-settings-generation-implementation.md | 14 +- .../generation-finalization-2026-07-29.md | 33 + docs/modules/opentake-agent/SPEC.md | 10 +- docs/specs/agent/2-tools.md | 12 +- ...4\270\216Agent\345\267\245\345\205\267.md" | 2 + src-tauri/Cargo.toml | 3 +- src-tauri/src/account.rs | 9 + src-tauri/src/chat.rs | 33 +- src-tauri/src/export.rs | 9 +- src-tauri/src/generation.rs | 2922 +++++++++++++++++ src-tauri/src/lib.rs | 15 +- src-tauri/src/mcp.rs | 19 +- src-tauri/src/media.rs | 39 +- src-tauri/src/secret.rs | 12 +- web/src/components/media/MediaPanel.tsx | 120 +- web/src/components/settings/SettingsView.tsx | 3 + web/src/lib/api.ts | 17 + web/src/lib/types.ts | 15 + web/src/store/settingsStore.ts | 16 +- 47 files changed, 5341 insertions(+), 101 deletions(-) create mode 100644 crates/opentake-agent/src/mcp/generation.rs create mode 100644 crates/opentake-agent/tests/generation_dispatch.rs create mode 100644 crates/opentake-core/tests/generation_persistence.rs create mode 100644 docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md create mode 100644 src-tauri/src/generation.rs diff --git a/Cargo.lock b/Cargo.lock index b7c0a0e1..6088042f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3402,6 +3402,7 @@ dependencies = [ "same-file", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", ] @@ -3531,6 +3532,7 @@ dependencies = [ "cap-std", "cpal", "futures", + "futures-util", "image", "libc", "objc2-app-kit", @@ -4180,6 +4182,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", "futures-util", "http", diff --git a/crates/opentake-agent/src/chat/loop.rs b/crates/opentake-agent/src/chat/loop.rs index 8a494ec7..8bb3665a 100644 --- a/crates/opentake-agent/src/chat/loop.rs +++ b/crates/opentake-agent/src/chat/loop.rs @@ -257,9 +257,9 @@ impl ChatLoop { /// When the dispatcher lacks a media bridge, hide the bridge-dependent /// tools instead of advertising tools that would only fail at runtime. fn tool_catalog(&self) -> Vec { - ToolName::ALL - .iter() - .copied() + self.dispatcher + .advertised_tools() + .into_iter() .filter(|tool| { self.dispatcher.has_media_bridge() || !matches!( diff --git a/crates/opentake-agent/src/mcp/dispatch.rs b/crates/opentake-agent/src/mcp/dispatch.rs index 9dd34b06..8b094886 100644 --- a/crates/opentake-agent/src/mcp/dispatch.rs +++ b/crates/opentake-agent/src/mcp/dispatch.rs @@ -22,8 +22,8 @@ use std::sync::{Arc, Mutex, RwLock}; use opentake_domain::{AnimPair, Crop, Interpolation, Keyframe, KeyframeTrack}; use opentake_domain::{ - ChromaKey, ColorGrade, Effect, LiftGammaGain, Mask, MaskShape, MediaManifest, Point2, Rgb, - Rgba, TextStyle, Timeline, Transform, VideoType, + ChromaKey, ColorGrade, Effect, GenerationJobStatus, LiftGammaGain, Mask, MaskShape, + MediaManifest, Point2, Rgb, Rgba, TextStyle, Timeline, Transform, VideoType, }; use opentake_media::analysis::{ detect_beats, detect_silences, BeatDetectionConfig, SilenceDetectionConfig, @@ -37,6 +37,7 @@ use serde_json::Value; use crate::mcp::core_handle::CoreHandle; use crate::mcp::gen_catalog; +use crate::mcp::generation::{GenerationBridge, GenerationRequest}; use crate::mcp::media_bridge::{ frame_to_block, media_frame_to_block, BridgeErrorKind, ImportSource, InspectMediaRequest, InspectMediaResult, InspectResult, MediaBridge, SearchCandidate, TranscriptSource, @@ -63,6 +64,16 @@ const INSPECT_MEDIA_MAX_FRAMES: usize = 12; const INSPECT_MEDIA_MAX_SEGMENTS: usize = 400; const INSPECT_MEDIA_MAX_WORDS: usize = 10_000; +fn is_generation_tool(tool: ToolName) -> bool { + matches!( + tool, + ToolName::GenerateVideo + | ToolName::GenerateImage + | ToolName::GenerateAudio + | ToolName::UpscaleMedia + ) +} + /// The in-process tool dispatcher. Holds the [`CoreHandle`] boundary, the plugin /// registry (read-locked for the active plugin), and a per-dispatcher agent-undo /// stack so `undo` only reverts edits this session made. @@ -74,6 +85,9 @@ pub struct Dispatcher { /// [`CoreHandle`] because those two capabilities reach into crates the agent /// layer does not link (`opentake-render`, the src-tauri import path). bridge: Option>, + /// Paid generation/upscale side-door. The desktop host injects this only + /// when it can persist jobs and run configured providers. + generation_bridge: Option>, /// Action names of agent edits applied through this dispatcher, newest last. /// Guards `undo`: we only revert when this session has pushed an edit. agent_undo: Mutex>, @@ -94,11 +108,22 @@ impl Dispatcher { handle: Arc, registry: Arc>, bridge: Option>, + ) -> Self { + Self::with_bridges(handle, registry, bridge, None) + } + + /// New dispatcher with independent media and generation host capabilities. + pub fn with_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> Self { Dispatcher { handle, registry, bridge, + generation_bridge, agent_undo: Mutex::new(Vec::new()), } } @@ -109,6 +134,20 @@ impl Dispatcher { self.bridge.is_some() } + pub fn can_generate(&self) -> bool { + self.generation_bridge + .as_ref() + .is_some_and(|bridge| bridge.can_generate()) + } + + pub fn advertised_tools(&self) -> Vec { + let mut tools = ToolName::ALL.to_vec(); + if self.can_generate() { + tools.extend(ToolName::GENERATION); + } + tools + } + /// Snapshot the current timeline from the bound core handle. pub fn timeline(&self) -> Timeline { self.handle.timeline() @@ -145,7 +184,9 @@ impl Dispatcher { error.message, ); } - if !ToolName::ALL.contains(&tool) { + if !(ToolName::ALL.contains(&tool) + || is_generation_tool(tool) && self.generation_bridge.is_some()) + { return ToolResult::public_error( PublicErrorKind::UnknownTool, format!("Tool is not advertised: {}", tool.as_str()), @@ -208,16 +249,15 @@ impl Dispatcher { ToolName::GetTimeline => { let a: GetTimelineArgs = decode_tool_args(args, "")?; let tl = self.handle.timeline(); - // canGenerate is gated by the (not-yet-wired) generation backend; - // false until that lands so the model never proposes generation. - let json = encode_timeline(&tl, a.start_frame, a.end_frame, false); + let json = encode_timeline(&tl, a.start_frame, a.end_frame, self.can_generate()); Ok(ToolResult::ok(json.to_string())) } ToolName::GetMedia => { let manifest = self.handle.media(); - let json = serde_json::to_value(&manifest) + let mut json = serde_json::to_value(&manifest) .map(round_floats_3dp) .map_err(|e| ToolError::new(format!("get_media: {e}")))?; + inject_generation_statuses(&mut json, &manifest); Ok(ToolResult::ok(json.to_string())) } ToolName::ListFolders => { @@ -278,17 +318,54 @@ impl Dispatcher { ToolName::GenerateVideo | ToolName::GenerateImage | ToolName::GenerateAudio - | ToolName::UpscaleMedia - | ToolName::AddMotionGraphic - | ToolName::EditMotionGraphic => Ok(ToolResult::error(format!( - "{}: capability is not advertised", - tool.as_str() - ))), + | ToolName::UpscaleMedia => self.submit_generation(tool, args, cancel), + ToolName::AddMotionGraphic | ToolName::EditMotionGraphic => Ok(ToolResult::error( + format!("{}: capability is not advertised", tool.as_str()), + )), } } // MARK: - Generative read bodies + fn submit_generation( + &self, + tool: ToolName, + args: &Value, + cancel: &opentake_media::MediaCancelToken, + ) -> Result { + let request = match tool { + ToolName::GenerateVideo => GenerationRequest::Video(decode_tool_args(args, "")?), + ToolName::GenerateImage => GenerationRequest::Image(decode_tool_args(args, "")?), + ToolName::GenerateAudio => GenerationRequest::Audio(decode_tool_args(args, "")?), + ToolName::UpscaleMedia => GenerationRequest::Upscale(decode_tool_args(args, "")?), + _ => return Err(ToolError::new("not a generation tool")), + }; + if !request.cost_authorized() { + return Ok(ToolResult::public_error( + PublicErrorKind::InvalidArguments(tool), + "costAuthorized must be true after explicit user approval", + )); + } + let Some(bridge) = self.generation_bridge.as_ref() else { + return Ok(ToolResult::public_error( + PublicErrorKind::CapabilityUnavailable(tool), + "Generation is not available in this build.", + )); + }; + if !bridge.can_generate() { + return Ok(ToolResult::public_error( + PublicErrorKind::CapabilityUnavailable(tool), + "Configure a compatible generation provider before submitting.", + )); + } + let submission = bridge + .submit(request, cancel) + .map_err(|_| ToolError::new("generation submission failed"))?; + let payload = serde_json::to_string(&submission) + .map_err(|_| ToolError::new("generation submission response failed"))?; + Ok(ToolResult::ok(payload)) + } + /// `list_models`: project the built-in static catalog from `opentake-gen` /// into the `{ models, loaded }` payload, optionally filtered by `?type=`. /// Fully local — no network, no BYOK key — so it runs synchronously here and @@ -323,6 +400,7 @@ impl Dispatcher { format!("Media not found: {}", a.media_ref), )); }; + ensure_generation_output_ready(entry, "inspect_media")?; if entry.kind == opentake_domain::ClipType::Text { return Ok(ToolResult::public_error( PublicErrorKind::CapabilityUnavailable(ToolName::InspectMedia), @@ -1029,6 +1107,13 @@ impl Dispatcher { let mut explicit_count = 0usize; for (i, raw) in a.entries.iter().enumerate() { let e: AddClipEntry = decode_tool_args(raw, &format!("entries[{i}]"))?; + if let Some(entry) = manifest + .entries + .iter() + .find(|entry| entry.id == e.media_ref) + { + ensure_generation_output_ready(entry, &format!("entries[{i}]"))?; + } let (media_type, has_audio) = resolve_media_kind(manifest, &e.media_ref); if e.track_index.is_some() { explicit_count += 1; @@ -1082,6 +1167,13 @@ impl Dispatcher { let mut entries = Vec::with_capacity(a.entries.len()); for (i, raw) in a.entries.iter().enumerate() { let e: InsertClipEntry = decode_tool_args(raw, &format!("entries[{i}]"))?; + if let Some(entry) = manifest + .entries + .iter() + .find(|entry| entry.id == e.media_ref) + { + ensure_generation_output_ready(entry, &format!("entries[{i}]"))?; + } let (media_type, has_audio) = resolve_media_kind(manifest, &e.media_ref); let duration_frames = match e.duration_frames { Some(d) => d, @@ -2274,6 +2366,63 @@ fn resolve_media_kind( .unwrap_or((opentake_domain::ClipType::Video, false)) } +fn generation_status_label(status: Option) -> &'static str { + match status { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => "generating", + Some(GenerationJobStatus::Downloading | GenerationJobStatus::Finalizing) => "downloading", + Some(GenerationJobStatus::Failed) => "failed", + Some(GenerationJobStatus::Cancelled) => "cancelled", + Some(GenerationJobStatus::Ready) | None => "none", + } +} + +fn ensure_generation_output_ready( + entry: &opentake_domain::MediaManifestEntry, + path: &str, +) -> Result<(), ToolError> { + let status = entry + .generation_input + .as_ref() + .and_then(|input| input.status); + if matches!(status, Some(GenerationJobStatus::Ready) | None) { + return Ok(()); + } + Err(ToolError::new(format!( + "{path}: generated media '{}' is not ready (status {})", + entry.id, + generation_status_label(status) + ))) +} + +fn inject_generation_statuses(payload: &mut Value, manifest: &MediaManifest) { + let Some(entries) = payload + .as_object_mut() + .and_then(|object| object.get_mut("entries")) + .and_then(Value::as_array_mut) + else { + return; + }; + for (payload_entry, manifest_entry) in entries.iter_mut().zip(&manifest.entries) { + let Some(object) = payload_entry.as_object_mut() else { + continue; + }; + let input = manifest_entry.generation_input.as_ref(); + object.insert( + "generationStatus".into(), + Value::String(generation_status_label(input.and_then(|value| value.status)).into()), + ); + if let Some(progress) = input.and_then(|value| value.progress) { + object.insert("generationProgress".into(), json_number(progress, 3)); + } + if let Some(error_code) = input.and_then(|value| value.error_code.as_ref()) { + object.insert( + "generationErrorCode".into(), + Value::String(error_code.clone()), + ); + } + } +} + /// Current `(track_index, start_frame)` of a clip on the timeline, or `(None, /// None)` if absent. Used to fill optional `move_clips` fields. fn clip_location(timeline: &Timeline, clip_id: &str) -> (Option, Option) { @@ -3026,7 +3175,18 @@ fn inspect_media_result( "duration".into(), json_number(inspected.duration_seconds, 3), ); - meta.insert("generationStatus".into(), Value::String("none".into())); + meta.insert( + "generationStatus".into(), + Value::String( + generation_status_label( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + ) + .into(), + ), + ); meta.insert("byteSize".into(), Value::from(inspected.byte_size)); if let Some(file_name) = manifest_file_name(entry) { meta.insert("fileName".into(), Value::String(file_name)); diff --git a/crates/opentake-agent/src/mcp/generation.rs b/crates/opentake-agent/src/mcp/generation.rs new file mode 100644 index 00000000..e17f91a7 --- /dev/null +++ b/crates/opentake-agent/src/mcp/generation.rs @@ -0,0 +1,223 @@ +//! Provider-neutral generation lifecycle coordination. +//! +//! The Agent dispatcher creates durable placeholders synchronously, while the +//! desktop bridge submits and watches the paid provider job off-thread. This +//! module owns the deterministic terminal pairing contract so every returned +//! result is applied to at most one placeholder and every placeholder reaches +//! one persisted terminal state. + +use std::path::PathBuf; + +use serde::Serialize; + +use crate::tools::args::{ + GenerateAudioArgs, GenerateImageArgs, GenerateVideoArgs, UpscaleMediaArgs, +}; + +/// Typed paid-generation request passed from the tool dispatcher to the +/// desktop runtime. The bridge owns model resolution, reference validation, +/// durable placeholder creation, provider submission, and background watch. +#[derive(Debug, Clone, PartialEq)] +pub enum GenerationRequest { + Video(GenerateVideoArgs), + Image(GenerateImageArgs), + Audio(GenerateAudioArgs), + Upscale(UpscaleMediaArgs), +} + +impl GenerationRequest { + pub fn cost_authorized(&self) -> bool { + match self { + Self::Video(args) => args.cost_authorized == Some(true), + Self::Image(args) => args.cost_authorized == Some(true), + Self::Audio(args) => args.cost_authorized == Some(true), + Self::Upscale(args) => args.cost_authorized == Some(true), + } + } +} + +/// Immediate response from an accepted asynchronous generation submission. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GenerationSubmission { + pub job_id: String, + pub placeholder_asset_ids: Vec, + pub status: String, +} + +/// Host boundary for production generation. Implementations must return only +/// after the placeholder/job record is durably committed, and must continue +/// provider work off the synchronous MCP dispatch thread. +pub trait GenerationBridge: Send + Sync { + /// True only when managed authorization or at least one compatible BYOK + /// credential is currently usable. + fn can_generate(&self) -> bool; + + fn submit( + &self, + request: GenerationRequest, + cancel: &opentake_media::MediaCancelToken, + ) -> Result; +} + +/// A provider result downloaded into a private staging location. The store +/// validates and commits it into the project before reporting success. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DownloadedGenerationArtifact { + pub path: PathBuf, + pub media_type: String, + pub byte_size: u64, +} + +/// Downloads one provider result without exposing credentials or signed URL +/// details to the persistence layer's public error contract. +pub trait GenerationArtifactDownloader { + fn download(&self, asset_id: &str, url: &str) -> Result; +} + +/// Durable state boundary implemented by the desktop project runtime. +pub trait GenerationFinalizationStore { + /// Atomically claim a terminal-finalization lease. `false` means the job is + /// already complete or another callback currently owns the lease. + fn claim_terminal(&self, job_id: &str) -> Result; + + /// Release a failed terminal-finalization lease so restart recovery or a + /// duplicate provider callback can retry. Output operations below must be + /// idempotent because a prior attempt may already have committed a prefix. + fn release_terminal(&self, job_id: &str) -> Result<(), String>; + + /// Commit one staged artifact to the matching placeholder identity. + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String>; + + /// Persist one fixed, non-sensitive terminal failure code. + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String>; + + /// Persist the aggregate job terminal state after every placeholder has a + /// terminal record. + fn complete_job(&self, job_id: &str, succeeded: usize, failed: usize) -> Result<(), String>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GenerationFinalizationSummary { + pub claimed: bool, + pub succeeded: usize, + pub failed: usize, + pub ignored_result_urls: usize, +} + +/// Pair provider URLs to placeholders in order and terminalize every +/// placeholder exactly once. Missing, malformed, download-failed, and +/// commit-failed results become fixed failure codes; extra URLs are ignored. +pub fn finalize_terminal_outputs( + store: &dyn GenerationFinalizationStore, + downloader: &dyn GenerationArtifactDownloader, + job_id: &str, + placeholder_ids: &[String], + result_urls: &[String], +) -> Result { + if !store.claim_terminal(job_id)? { + return Ok(GenerationFinalizationSummary { + claimed: false, + succeeded: 0, + failed: 0, + ignored_result_urls: 0, + }); + } + + let attempt = (|| { + let mut succeeded = 0; + let mut failed = 0; + for (index, asset_id) in placeholder_ids.iter().enumerate() { + let Some(url) = result_urls.get(index) else { + store.fail_output(asset_id, "GENERATION_RESULT_MISSING")?; + failed += 1; + continue; + }; + if !is_accepted_result_url(url) { + store.fail_output(asset_id, "GENERATION_RESULT_URL_INVALID")?; + failed += 1; + continue; + } + let artifact = match downloader.download(asset_id, url) { + Ok(artifact) => artifact, + Err(error) if error == "GENERATION_CANCELLED" => return Err(error), + Err(_) => { + store.fail_output(asset_id, "GENERATION_DOWNLOAD_FAILED")?; + failed += 1; + continue; + } + }; + if store.finalize_output(asset_id, artifact).is_err() { + store.fail_output(asset_id, "GENERATION_FINALIZE_FAILED")?; + failed += 1; + continue; + } + succeeded += 1; + } + store.complete_job(job_id, succeeded, failed)?; + Ok::<_, String>((succeeded, failed)) + })(); + + let (succeeded, failed) = match attempt { + Ok(summary) => summary, + Err(error) => { + if let Err(release_error) = store.release_terminal(job_id) { + return Err(format!( + "generation finalization failed and lease release failed: {release_error}" + )); + } + return Err(error); + } + }; + + Ok(GenerationFinalizationSummary { + claimed: true, + succeeded, + failed, + ignored_result_urls: result_urls.len().saturating_sub(placeholder_ids.len()), + }) +} + +fn is_accepted_result_url(raw: &str) -> bool { + let Ok(url) = reqwest::Url::parse(raw) else { + return false; + }; + match url.scheme() { + "https" => { + url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.port().is_none_or(|port| port == 443) + } + "data" => { + let value = url.as_str(); + value.starts_with("data:image/") + || value.starts_with("data:audio/") + || value.starts_with("data:video/") + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::is_accepted_result_url; + + #[test] + fn provider_result_urls_are_https_or_bounded_media_data_urls() { + assert!(is_accepted_result_url("https://cdn.test/result.png")); + assert!(is_accepted_result_url("data:image/png;base64,AAAA")); + assert!(is_accepted_result_url("data:audio/mpeg;base64,AAAA")); + assert!(is_accepted_result_url("data:video/mp4;base64,AAAA")); + assert!(!is_accepted_result_url("http://cdn.test/result.png")); + assert!(!is_accepted_result_url( + "https://user:secret@cdn.test/result.png" + )); + assert!(!is_accepted_result_url("file:///tmp/result.png")); + assert!(!is_accepted_result_url("not-a-url")); + } +} diff --git a/crates/opentake-agent/src/mcp/mod.rs b/crates/opentake-agent/src/mcp/mod.rs index 23362344..98c01322 100644 --- a/crates/opentake-agent/src/mcp/mod.rs +++ b/crates/opentake-agent/src/mcp/mod.rs @@ -13,5 +13,6 @@ pub mod convert; pub mod core_handle; pub mod dispatch; pub mod gen_catalog; +pub mod generation; pub mod media_bridge; pub mod server; diff --git a/crates/opentake-agent/src/mcp/server.rs b/crates/opentake-agent/src/mcp/server.rs index d95af475..3afc2b06 100644 --- a/crates/opentake-agent/src/mcp/server.rs +++ b/crates/opentake-agent/src/mcp/server.rs @@ -27,11 +27,13 @@ use serde_json::{Map, Value}; use crate::mcp::convert::to_call_tool_result; use crate::mcp::core_handle::CoreHandle; use crate::mcp::dispatch::Dispatcher; +use crate::mcp::generation::GenerationBridge; use crate::mcp::media_bridge::{MediaBridge, MCP_REQUEST_BODY_MAX}; use crate::plugin::registry::PluginRegistry; use crate::prompt::assemble::assemble_system_prompt; use crate::tools::descriptions::{description, input_schema}; use crate::tools::errors::first_non_finite_json_number_path; +#[cfg(test)] use crate::tools::names::ToolName; use crate::tools::panic_boundary::with_redacted_dispatch_panic; @@ -69,22 +71,37 @@ impl McpServer { handle: Arc, registry: Arc>, bridge: Option>, + ) -> Self { + Self::with_bridges(handle, registry, bridge, None) + } + + pub fn with_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> Self { let instructions = registry .read() .map(|r| assemble_system_prompt(&r, "default")) .unwrap_or_default(); McpServer { - dispatcher: Arc::new(Dispatcher::with_bridge(handle, registry, bridge)), + dispatcher: Arc::new(Dispatcher::with_bridges( + handle, + registry, + bridge, + generation_bridge, + )), instructions, } } - /// All tool schemas (1:1 with [`ToolName::ALL`]). - fn tools() -> Vec { - ToolName::ALL - .iter() - .map(|&t| { + /// Tool schemas for capabilities live in this exact host session. + fn tools(&self) -> Vec { + self.dispatcher + .advertised_tools() + .into_iter() + .map(|t| { let obj = input_schema(t) .as_object() .cloned() @@ -121,7 +138,7 @@ impl ServerHandler for McpServer { _context: RequestContext, ) -> Result { Ok(ListToolsResult { - tools: Self::tools(), + tools: self.tools(), next_cursor: None, meta: None, }) @@ -453,6 +470,16 @@ pub fn build_router_with_bridge_for_port( registry: Arc>, bridge: Option>, expected_port: u16, +) -> axum::Router { + build_router_with_bridges_for_port(handle, registry, bridge, None, expected_port) +} + +pub fn build_router_with_bridges_for_port( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + expected_port: u16, ) -> axum::Router { use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use rmcp::transport::streamable_http_server::{ @@ -463,10 +490,11 @@ pub fn build_router_with_bridge_for_port( let service = StreamableHttpService::new( move || { - Ok(McpServer::with_bridge( + Ok(McpServer::with_bridges( handle.clone(), registry.clone(), bridge.clone(), + generation_bridge.clone(), )) }, Arc::new(LocalSessionManager::default()), @@ -508,6 +536,16 @@ pub async fn serve_with_bridge( handle: Arc, registry: Arc>, bridge: Option>, +) -> std::io::Result<()> { + serve_with_bridges(addr, handle, registry, bridge, None).await +} + +pub async fn serve_with_bridges( + addr: SocketAddr, + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, ) -> std::io::Result<()> { if !addr.ip().is_loopback() { return Err(std::io::Error::new( @@ -517,7 +555,13 @@ pub async fn serve_with_bridge( } let listener = tokio::net::TcpListener::bind(addr).await?; let bound_addr = listener.local_addr()?; - let router = build_router_with_bridge_for_port(handle, registry, bridge, bound_addr.port()); + let router = build_router_with_bridges_for_port( + handle, + registry, + bridge, + generation_bridge, + bound_addr.port(), + ); tracing::info!("MCP server listening on http://{bound_addr}/mcp"); axum::serve(listener, router).await } @@ -567,12 +611,10 @@ mod tests { #[test] fn lists_every_advertised_tool() { - assert_eq!(McpServer::tools().len(), ToolName::ALL.len()); + let server = server(); + assert_eq!(server.tools().len(), ToolName::ALL.len()); // Names round-trip to the wire names. - let names: Vec = McpServer::tools() - .iter() - .map(|t| t.name.to_string()) - .collect(); + let names: Vec = server.tools().iter().map(|t| t.name.to_string()).collect(); assert!(names.contains(&"add_clips".to_string())); assert!(names.contains(&"detect_beats".to_string())); assert!(names.contains(&"activate_workflow".to_string())); diff --git a/crates/opentake-agent/src/tools/args.rs b/crates/opentake-agent/src/tools/args.rs index 1a40d730..cc67a9f2 100644 --- a/crates/opentake-agent/src/tools/args.rs +++ b/crates/opentake-agent/src/tools/args.rs @@ -682,6 +682,7 @@ impl ToolArgs for TightenSilencesArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateVideoArgs { + pub cost_authorized: Option, pub prompt: String, pub name: Option, pub model: Option, @@ -700,6 +701,7 @@ pub struct GenerateVideoArgs { impl ToolArgs for GenerateVideoArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "duration", @@ -720,23 +722,27 @@ impl ToolArgs for GenerateVideoArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateImageArgs { + pub cost_authorized: Option, pub prompt: String, pub name: Option, pub model: Option, pub aspect_ratio: Option, pub resolution: Option, pub quality: Option, + pub num_images: Option, pub reference_media_refs: Option>, pub folder_id: Option, } impl ToolArgs for GenerateImageArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "aspectRatio", "resolution", "quality", + "numImages", "referenceMediaRefs", "folderId", ]; @@ -746,6 +752,7 @@ impl ToolArgs for GenerateImageArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GenerateAudioArgs { + pub cost_authorized: Option, pub prompt: Option, pub name: Option, pub model: Option, @@ -762,6 +769,7 @@ pub struct GenerateAudioArgs { impl ToolArgs for GenerateAudioArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ "prompt", + "costAuthorized", "name", "model", "voice", @@ -780,12 +788,14 @@ impl ToolArgs for GenerateAudioArgs { #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct UpscaleMediaArgs { + pub cost_authorized: Option, pub media_ref: String, pub model: Option, pub source_clip_id: Option, } impl ToolArgs for UpscaleMediaArgs { - const ALLOWED_KEYS: &'static [&'static str] = &["mediaRef", "model", "sourceClipId"]; + const ALLOWED_KEYS: &'static [&'static str] = + &["costAuthorized", "mediaRef", "model", "sourceClipId"]; } // --- import_media --- diff --git a/crates/opentake-agent/src/tools/descriptions.rs b/crates/opentake-agent/src/tools/descriptions.rs index 3a5e1a5a..74042e7f 100644 --- a/crates/opentake-agent/src/tools/descriptions.rs +++ b/crates/opentake-agent/src/tools/descriptions.rs @@ -402,6 +402,7 @@ pub fn input_schema(tool: ToolName) -> Value { ToolName::GenerateVideo => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Text description of the video to generate"}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID (e.g. 'veo3.1-fast'). Use list_models to see options. Defaults to first available model."}, @@ -417,25 +418,28 @@ pub fn input_schema(tool: ToolName) -> Value { "referenceAudioMediaRefs": {"type": "array", "items": {"type": "string"}, "description": "Media asset IDs of audio references (Seedance only). Refer to them as @Audio1, @Audio2. See maxReferenceAudios and maxCombinedAudioRefSeconds."}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &["prompt"], + &["costAuthorized", "prompt"], ), ToolName::GenerateImage => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Text description of the image to generate"}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID (e.g. 'nano-banana-pro'). Use list_models to see options. Defaults to first available model."}, "aspectRatio": {"type": "string", "description": "Aspect ratio (e.g. '16:9', '9:16')"}, "resolution": {"type": "string", "description": "Resolution (e.g. '2K', '4K')"}, "quality": {"type": "string", "description": "Image quality (e.g. 'low', 'medium', 'high'). Only supported by some models — see list_models."}, + "numImages": {"type": "integer", "minimum": 1, "maximum": 4, "description": "Number of ordered image results to generate. Defaults to 1 and is capped at 4."}, "referenceMediaRefs": {"type": "array", "items": {"type": "string"}, "description": "Media asset IDs to use as reference images"}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &["prompt"], + &["costAuthorized", "prompt"], ), ToolName::GenerateAudio => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, "prompt": {"type": "string", "description": "Required for TTS (the text to speak) and text-to-music (style/mood/genre; MiniMax needs ≥10 chars). For Lyria 3 Pro, include lyrics, tempo, language, and vocal style directly in the prompt. Optional style guide for video-to-music models."}, "name": {"type": "string", "description": "Display name for the asset in the media library. Defaults to first 30 chars of prompt."}, "model": {"type": "string", "description": "Model ID. Use list_models with type='audio' to see options and their 'inputs'. Defaults to the first model."}, @@ -449,16 +453,17 @@ pub fn input_schema(tool: ToolName) -> Value { "videoSourceMediaRef": {"type": "string", "description": "Video-to-audio models only. Score this existing video asset instead of a timeline span. Mutually exclusive with the videoSource frames."}, "folderId": {"type": "string", "description": "Optional. Folder id (from list_folders or create_folder) to place the result in. Omit for the project root."} }), - &[], + &["costAuthorized"], ), ToolName::UpscaleMedia => object( json!({ + "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid upscale request in this conversation."}, "mediaRef": {"type": "string", "description": "ID of the video or image asset to upscale"}, "model": {"type": "string", "description": "Upscaler model ID (e.g. 'bytedance-upscaler', 'seedvr-image-upscaler'). Defaults to the first model that supports the asset's type."}, "sourceClipId": {"type": "string", "description": "Optional. Video clip id (from get_timeline) referencing mediaRef. When set and the clip is trimmed, only the clip's visible range is upscaled, not the full source."} }), - &["mediaRef"], + &["costAuthorized", "mediaRef"], ), ToolName::ImportMedia => object( diff --git a/crates/opentake-agent/src/tools/names.rs b/crates/opentake-agent/src/tools/names.rs index 5fee8fbf..52c48d39 100644 --- a/crates/opentake-agent/src/tools/names.rs +++ b/crates/opentake-agent/src/tools/names.rs @@ -157,6 +157,15 @@ impl ToolName { ToolName::ApplyEffect, ]; + /// Provider-backed tools appended to a host catalog only while its live + /// generation bridge reports usable authorization. + pub const GENERATION: [ToolName; 4] = [ + ToolName::GenerateVideo, + ToolName::GenerateImage, + ToolName::GenerateAudio, + ToolName::UpscaleMedia, + ]; + /// Every recognized schema/wire name, including capabilities deliberately /// hidden from discovery until a real backend exists. Keeping this set lets /// strict argument validation and compatibility tests cover future tools diff --git a/crates/opentake-agent/tests/generation_dispatch.rs b/crates/opentake-agent/tests/generation_dispatch.rs new file mode 100644 index 00000000..46c9e9af --- /dev/null +++ b/crates/opentake-agent/tests/generation_dispatch.rs @@ -0,0 +1,285 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use opentake_agent::mcp::core_handle::AppCoreHandle; +use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::mcp::generation::{ + finalize_terminal_outputs, DownloadedGenerationArtifact, GenerationArtifactDownloader, + GenerationBridge, GenerationFinalizationStore, GenerationRequest, GenerationSubmission, +}; +use opentake_agent::plugin::registry::PluginRegistry; +use opentake_core::AppCore; +use serde_json::json; + +#[derive(Default)] +struct RecordingStore { + claimed: Mutex>, + completed: Mutex>, + finalized: Mutex>, + failed: Mutex>, + completions: Mutex>, + output_write_failures_remaining: Mutex, +} + +impl GenerationFinalizationStore for RecordingStore { + fn claim_terminal(&self, job_id: &str) -> Result { + if self.completed.lock().unwrap().contains(job_id) { + return Ok(false); + } + Ok(self.claimed.lock().unwrap().insert(job_id.to_string())) + } + + fn release_terminal(&self, job_id: &str) -> Result<(), String> { + self.claimed.lock().unwrap().remove(job_id); + Ok(()) + } + + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String> { + let mut failures = self.output_write_failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("transient manifest write failure".to_string()); + } + self.finalized + .lock() + .unwrap() + .insert(asset_id.to_string(), artifact.path); + Ok(()) + } + + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String> { + let mut failures = self.output_write_failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("transient manifest write failure".to_string()); + } + self.failed + .lock() + .unwrap() + .insert(asset_id.to_string(), code.to_string()); + Ok(()) + } + + fn complete_job(&self, job_id: &str, succeeded: usize, failed: usize) -> Result<(), String> { + if !self.completed.lock().unwrap().insert(job_id.to_string()) { + return Ok(()); + } + self.completions + .lock() + .unwrap() + .push((job_id.to_string(), succeeded, failed)); + Ok(()) + } +} + +struct FixtureDownloader; + +impl GenerationArtifactDownloader for FixtureDownloader { + fn download(&self, asset_id: &str, url: &str) -> Result { + if url.contains("download-fails") { + return Err("provider download failed with private detail".to_string()); + } + Ok(DownloadedGenerationArtifact { + path: PathBuf::from(format!("/fixture/{asset_id}.bin")), + media_type: "application/octet-stream".to_string(), + byte_size: 7, + }) + } +} + +struct RecordingGenerationBridge { + available: AtomicBool, + submissions: AtomicUsize, +} + +impl RecordingGenerationBridge { + fn new(available: bool) -> Self { + Self { + available: AtomicBool::new(available), + submissions: AtomicUsize::new(0), + } + } +} + +impl GenerationBridge for RecordingGenerationBridge { + fn can_generate(&self) -> bool { + self.available.load(Ordering::Acquire) + } + + fn submit( + &self, + _request: GenerationRequest, + _cancel: &opentake_media::MediaCancelToken, + ) -> Result { + self.submissions.fetch_add(1, Ordering::AcqRel); + Ok(GenerationSubmission { + job_id: "job-dispatch".to_string(), + placeholder_asset_ids: vec!["asset-placeholder".to_string()], + status: "queued".to_string(), + }) + } +} + +fn dispatcher_with_generation_bridge(bridge: Arc) -> Dispatcher { + Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(AppCore::new())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ) +} + +#[test] +fn placeholder_persist_finalize_all_results_and_failures() { + let store = RecordingStore::default(); + let summary = finalize_terminal_outputs( + &store, + &FixtureDownloader, + "job-1", + &[ + "asset-a".into(), + "asset-b".into(), + "asset-c".into(), + "asset-d".into(), + ], + &[ + "https://results.test/a.png".into(), + "https://results.test/download-fails.png".into(), + "not-a-result-url".into(), + "https://results.test/extra.png".into(), + "https://results.test/ignored-extra.png".into(), + ], + ) + .expect("terminal outputs are recorded"); + + assert!(summary.claimed); + assert_eq!(summary.succeeded, 2); + assert_eq!(summary.failed, 2); + assert_eq!(summary.ignored_result_urls, 1); + assert_eq!( + store + .finalized + .lock() + .unwrap() + .keys() + .cloned() + .collect::>(), + vec!["asset-a", "asset-d"] + ); + assert_eq!( + store.failed.lock().unwrap().clone(), + BTreeMap::from([ + ( + "asset-b".to_string(), + "GENERATION_DOWNLOAD_FAILED".to_string() + ), + ( + "asset-c".to_string(), + "GENERATION_RESULT_URL_INVALID".to_string() + ), + ]) + ); + assert_eq!( + store.completions.lock().unwrap().as_slice(), + &[("job-1".to_string(), 2, 2)] + ); +} + +#[test] +fn placeholder_persists_and_every_terminal_result_finalizes_once() { + let store = RecordingStore::default(); + let placeholders = vec!["asset-a".to_string(), "asset-b".to_string()]; + let urls = vec!["https://results.test/a.png".to_string()]; + + let first = + finalize_terminal_outputs(&store, &FixtureDownloader, "job-once", &placeholders, &urls) + .expect("first terminal callback succeeds"); + let duplicate = + finalize_terminal_outputs(&store, &FixtureDownloader, "job-once", &placeholders, &urls) + .expect("duplicate terminal callback is idempotent"); + + assert!(first.claimed); + assert_eq!((first.succeeded, first.failed), (1, 1)); + assert!(!duplicate.claimed); + assert_eq!((duplicate.succeeded, duplicate.failed), (0, 0)); + assert_eq!(store.finalized.lock().unwrap().len(), 1); + assert_eq!( + store + .failed + .lock() + .unwrap() + .get("asset-b") + .map(String::as_str), + Some("GENERATION_RESULT_MISSING") + ); + assert_eq!(store.completions.lock().unwrap().len(), 1); + + let retryable = RecordingStore { + output_write_failures_remaining: Mutex::new(2), + ..Default::default() + }; + let first_attempt = finalize_terminal_outputs( + &retryable, + &FixtureDownloader, + "job-retry", + &["asset-retry".to_string()], + &["https://results.test/retry.png".to_string()], + ); + assert!(first_attempt.is_err()); + assert!(retryable.claimed.lock().unwrap().is_empty()); + + let recovered = finalize_terminal_outputs( + &retryable, + &FixtureDownloader, + "job-retry", + &["asset-retry".to_string()], + &["https://results.test/retry.png".to_string()], + ) + .expect("restart recovery can reacquire the released terminal lease"); + assert!(recovered.claimed); + assert_eq!((recovered.succeeded, recovered.failed), (1, 0)); + assert_eq!(retryable.completions.lock().unwrap().len(), 1); +} + +#[test] +fn configured_capability_and_cost_authorization_gate_dispatch() { + let bridge = Arc::new(RecordingGenerationBridge::new(false)); + let dispatcher = dispatcher_with_generation_bridge(bridge.clone()); + + let timeline = dispatcher.dispatch("get_timeline", json!({})); + assert!(!timeline.is_error); + assert!(timeline.text_joined().contains("\"canGenerate\":false")); + + let unavailable = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": true, "prompt": "fixture"}), + ); + assert!(unavailable.is_error); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 0); + + bridge.available.store(true, Ordering::Release); + let timeline = dispatcher.dispatch("get_timeline", json!({})); + assert!(timeline.text_joined().contains("\"canGenerate\":true")); + + let unauthorized = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": false, "prompt": "fixture"}), + ); + assert!(unauthorized.is_error); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 0); + + let accepted = dispatcher.dispatch( + "generate_image", + json!({"costAuthorized": true, "prompt": "fixture"}), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + assert!(accepted.text_joined().contains("job-dispatch")); + assert!(accepted.text_joined().contains("asset-placeholder")); + assert_eq!(bridge.submissions.load(Ordering::Acquire), 1); +} diff --git a/crates/opentake-core/Cargo.toml b/crates/opentake-core/Cargo.toml index b8eee2ca..446fd06a 100644 --- a/crates/opentake-core/Cargo.toml +++ b/crates/opentake-core/Cargo.toml @@ -14,3 +14,6 @@ opentake-ops = { workspace = true } opentake-project = { workspace = true } thiserror = "2" same-file = "1.0.6" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/opentake-core/src/core.rs b/crates/opentake-core/src/core.rs index fc3fc772..4c5e170a 100644 --- a/crates/opentake-core/src/core.rs +++ b/crates/opentake-core/src/core.rs @@ -40,7 +40,10 @@ use same_file::Handle; use crate::deps::CoreDeps; use crate::error::{CoreError, Result}; use crate::events::{CoreEvent, EventBus, SubscriptionId}; -use crate::session::{EditorSession, ProbedMedia}; +use crate::session::{ + EditorSession, GenerationJobCommit, GenerationStateUpdate, PreparedGenerationJob, + PreparedGenerationOutput, ProbedMedia, +}; type ProjectIdentityTransitionListener = Arc; @@ -676,6 +679,112 @@ impl AppCore { self.lock().editor.generation_log().clone() } + /// Persist placeholder assets and the queued audit event before a paid + /// provider request is submitted. No ids are returned unless the project + /// snapshot is durable. + pub fn begin_generation_job_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + plan: PreparedGenerationJob, + ) -> Result { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.begin_generation_job(plan, ids), + ) + } + + pub fn update_generation_job_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + job_id: &str, + update: GenerationStateUpdate, + ) -> Result { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.update_generation_job(job_id, update, ids), + ) + } + + pub fn finalize_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + output: PreparedGenerationOutput, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.finalize_generation_output(output, ids), + ) + } + + pub fn fail_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + asset_id: &str, + error_code: &str, + created_at: Option, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.fail_generation_output(asset_id, error_code, created_at, ids), + ) + } + + pub fn cancel_generation_output_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + asset_id: &str, + created_at: Option, + ) -> Result<()> { + self.persist_generation_mutation( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.cancel_generation_output(asset_id, created_at, ids), + ) + } + + fn persist_generation_mutation( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + mutate: impl FnOnce(&mut EditorSession, &dyn IdGen) -> Result, + ) -> Result { + let (value, count, written) = { + let mut session = self.lock(); + ensure_project_identity(&session, expected_project_epoch, expected_project_dir)?; + let checkpoint = session.editor.checkpoint_generation_state(); + let result = (|| { + let value = mutate(&mut session.editor, self.ids.as_ref())?; + let written = session.editor.save_generation_state()?; + Ok((value, written)) + })(); + match result { + Ok((value, written)) => (value, session.editor.media().entries.len(), written), + Err(error) => { + session.editor.restore_generation_state(checkpoint); + return Err(error); + } + } + }; + self.events.emit(&CoreEvent::MediaChanged { + project_epoch: expected_project_epoch, + count, + }); + self.events.emit(&CoreEvent::ProjectSaved { + path: written.to_string_lossy().into_owned(), + project_epoch: expected_project_epoch, + }); + Ok(value) + } + /// The open project's `.opentake` bundle directory, or `None` for an unsaved /// project. Needed to resolve [`MediaSource::Project`](opentake_domain::MediaSource) /// relative paths to on-disk files (preview/composite read the original media). diff --git a/crates/opentake-core/src/lib.rs b/crates/opentake-core/src/lib.rs index ad4632b9..070c06eb 100644 --- a/crates/opentake-core/src/lib.rs +++ b/crates/opentake-core/src/lib.rs @@ -49,7 +49,8 @@ pub use crate::core::{ ProjectRevision, ProjectRuntimeSnapshot, TimelineSnapshot, }; pub use session::{ - importable_clip_type, EditorSession, ProbedMedia, SUPPORTED_AUDIO_EXTENSIONS, + importable_clip_type, EditorSession, GenerationJobCommit, GenerationStateUpdate, + PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, SUPPORTED_AUDIO_EXTENSIONS, SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, }; diff --git a/crates/opentake-core/src/session.rs b/crates/opentake-core/src/session.rs index 7bcc2113..f4bbf19a 100644 --- a/crates/opentake-core/src/session.rs +++ b/crates/opentake-core/src/session.rs @@ -34,10 +34,15 @@ use std::path::{Path, PathBuf}; -use opentake_domain::{ClipType, MediaAsset, MediaManifest, MediaManifestEntry, Timeline}; +use opentake_domain::{ + ClipType, GenerationInput, GenerationJobStatus, MediaAsset, MediaManifest, MediaManifestEntry, + MediaSource, Timeline, +}; use opentake_ops::command::{self, EditCommand, EditResult}; use opentake_ops::{EditorState, IdGen}; -use opentake_project::{GenerationLog, Project, ProjectCompatibility, ProjectRoot}; +use opentake_project::{ + GenerationLog, GenerationLogEntry, Project, ProjectCompatibility, ProjectRoot, +}; use same_file::Handle; use crate::error::{CoreError, Result}; @@ -65,6 +70,55 @@ pub struct ProbedMedia { pub has_audio: bool, } +/// Validated provider-neutral generation job prepared by the Agent/Tauri host. +/// Credentials, signed URLs, and provider diagnostics are deliberately absent. +#[derive(Clone, Debug)] +pub struct PreparedGenerationJob { + pub name: String, + pub kind: ClipType, + pub folder_id: Option, + pub provider: String, + pub input: GenerationInput, + pub output_count: usize, + pub source_asset_id: Option, + pub source_clip_id: Option, + pub estimated_cost_credits: Option, + pub created_at: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GenerationJobCommit { + pub job_id: String, + pub placeholder_asset_ids: Vec, +} + +/// One durable lifecycle update. Provider messages are reduced to a fixed +/// application-owned `error_code` before reaching this boundary. +#[derive(Clone, Debug)] +pub struct GenerationStateUpdate { + pub status: GenerationJobStatus, + pub progress: Option, + pub error_code: Option, + pub provider_job_id: Option, + pub cost_credits: Option, + pub created_at: Option, +} + +#[derive(Clone, Debug)] +pub struct PreparedGenerationOutput { + pub asset_id: String, + pub relative_path: String, + pub probe: ProbedMedia, + pub created_at: Option, +} + +#[derive(Clone)] +pub(crate) struct GenerationStateCheckpoint { + manifest: MediaManifest, + log: GenerationLog, + component_present: bool, +} + /// File extensions the importer accepts, grouped by the [`ClipType`] they map to. /// /// Upstream's picker (`MediaTab.swift:754` — `allowedContentTypes = [.movie, @@ -105,6 +159,104 @@ pub fn importable_clip_type(path: &Path) -> Option { } } +fn safe_provider_prefix(value: &str) -> bool { + !value.is_empty() + && value.len() <= 32 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +fn safe_generation_error_code(value: &str) -> bool { + !value.is_empty() + && value.len() <= 80 + && value + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn validate_generation_update(update: &GenerationStateUpdate) -> Result<()> { + if update.cost_credits.is_some_and(|credits| credits < 0) { + return Err(CoreError::Media( + "generation cost must not be negative".to_string(), + )); + } + if let Some(progress) = update.progress { + if !progress.is_finite() || !(0.0..=1.0).contains(&progress) { + return Err(CoreError::Media( + "generation progress must be finite and between 0 and 1".to_string(), + )); + } + } + if let Some(error_code) = update.error_code.as_deref() { + if !safe_generation_error_code(error_code) { + return Err(CoreError::Media( + "generation failure code is invalid".to_string(), + )); + } + } + if update.status == GenerationJobStatus::Failed && update.error_code.is_none() { + return Err(CoreError::Media( + "failed generation status requires an error code".to_string(), + )); + } + if update.status != GenerationJobStatus::Failed && update.error_code.is_some() { + return Err(CoreError::Media( + "generation error code is only valid for failed status".to_string(), + )); + } + if let Some(provider_job_id) = update.provider_job_id.as_deref() { + if provider_job_id.is_empty() + || provider_job_id.len() > 512 + || provider_job_id.chars().any(char::is_control) + || provider_job_id.contains("://") + { + return Err(CoreError::Media( + "provider job identity is invalid".to_string(), + )); + } + } + Ok(()) +} + +fn valid_generation_transition( + current: Option, + next: GenerationJobStatus, +) -> bool { + use GenerationJobStatus as Status; + match (current, next) { + (None, Status::Queued) => true, + (Some(current), next) if current == next => true, + (Some(Status::Queued), Status::Generating | Status::Failed | Status::Cancelled) => true, + (Some(Status::Generating), Status::Downloading | Status::Failed | Status::Cancelled) => { + true + } + ( + Some(Status::Downloading), + Status::Finalizing | Status::Ready | Status::Failed | Status::Cancelled, + ) => true, + (Some(Status::Finalizing), Status::Ready | Status::Failed | Status::Cancelled) => true, + (Some(Status::Failed | Status::Cancelled), Status::Queued) => true, + (Some(Status::Ready), Status::Ready) => true, + _ => false, + } +} + +fn validate_project_media_relative_path(value: &str) -> Result<()> { + let path = Path::new(value); + let mut components = path.components(); + if components.next() != Some(std::path::Component::Normal("media".as_ref())) + || components.clone().next().is_none() + || path.is_absolute() + || components.any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err(CoreError::Media( + "generated output path must be a safe media-relative path".to_string(), + )); + } + Ok(()) +} + /// The open document plus its project-level metadata. pub struct EditorSession { /// Authoritative editable state: timeline, manifest, undo/redo, version. @@ -534,6 +686,384 @@ impl EditorSession { &self.generation_log } + /// Create durable manifest placeholders and append the corresponding + /// queued audit events. The caller persists the returned in-memory state in + /// the same critical section before exposing the placeholder ids. + pub(crate) fn begin_generation_job( + &mut self, + mut plan: PreparedGenerationJob, + ids: &dyn IdGen, + ) -> Result { + self.ensure_mutable()?; + if self.project_dir.is_none() { + return Err(CoreError::NoProjectOpen); + } + if !(1..=4).contains(&plan.output_count) { + return Err(CoreError::Media( + "generation output count must be between 1 and 4".to_string(), + )); + } + if plan.input.model.trim().is_empty() || plan.provider.trim().is_empty() { + return Err(CoreError::Media( + "generation model and provider are required".to_string(), + )); + } + if !safe_provider_prefix(&plan.provider) { + return Err(CoreError::Media( + "generation provider prefix is invalid".to_string(), + )); + } + if let Some(folder_id) = plan.folder_id.as_deref() { + if !self + .state + .manifest + .folders + .iter() + .any(|folder| folder.id == folder_id) + { + return Err(CoreError::Media(format!( + "generation folder does not exist: {folder_id}" + ))); + } + } + if let Some(source_asset_id) = plan.source_asset_id.as_deref() { + if !self + .state + .manifest + .entries + .iter() + .any(|entry| entry.id == source_asset_id) + { + return Err(CoreError::Media(format!( + "generation source asset does not exist: {source_asset_id}" + ))); + } + } + + let job_id = ids.next_id(); + plan.input.job_id = Some(job_id.clone()); + plan.input.provider = Some(plan.provider.clone()); + plan.input.provider_job_id = None; + plan.input.status = Some(GenerationJobStatus::Queued); + plan.input.progress = Some(0.0); + plan.input.error_code = None; + plan.input.source_asset_id = plan.source_asset_id.clone(); + plan.input.source_clip_id = plan.source_clip_id.clone(); + plan.input.estimated_cost_credits = plan.estimated_cost_credits; + plan.input.created_at = plan.created_at; + + let mut placeholder_asset_ids = Vec::with_capacity(plan.output_count); + for output_index in 0..plan.output_count { + let asset_id = ids.next_id(); + let mut input = plan.input.clone(); + input.output_index = Some(output_index); + let display_name = if plan.output_count == 1 { + plan.name.clone() + } else { + format!("{} {}", plan.name, output_index + 1) + }; + self.state.manifest.entries.push(MediaManifestEntry { + id: asset_id.clone(), + name: display_name, + kind: plan.kind, + source: MediaSource::Project { + relative_path: format!("media/{asset_id}.pending"), + }, + duration: (input.duration.max(0)) as f64, + generation_input: Some(input.clone()), + source_width: None, + source_height: None, + source_fps: None, + has_audio: Some( + plan.kind == ClipType::Audio + || (plan.kind == ClipType::Video && input.generate_audio.unwrap_or(true)), + ), + folder_id: plan.folder_id.clone(), + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + self.generation_log + .entries + .push(GenerationLogEntry::job_event( + ids.next_id(), + job_id.clone(), + input.model.clone(), + None, + plan.provider.clone(), + None, + asset_id.clone(), + GenerationJobStatus::Queued, + Some(0.0), + None, + plan.created_at, + plan.source_asset_id.clone(), + plan.source_clip_id.clone(), + )); + placeholder_asset_ids.push(asset_id); + } + self.generation_log_component_present = true; + Ok(GenerationJobCommit { + job_id, + placeholder_asset_ids, + }) + } + + pub(crate) fn update_generation_job( + &mut self, + job_id: &str, + update: GenerationStateUpdate, + ids: &dyn IdGen, + ) -> Result { + self.ensure_mutable()?; + validate_generation_update(&update)?; + let mut events = Vec::new(); + for entry in &mut self.state.manifest.entries { + let Some(input) = entry.generation_input.as_mut() else { + continue; + }; + if input.job_id.as_deref() != Some(job_id) { + continue; + } + if !valid_generation_transition(input.status, update.status) { + return Err(CoreError::Media(format!( + "invalid generation transition from {:?} to {:?}", + input.status, update.status + ))); + } + input.status = Some(update.status); + input.progress = update.progress; + input.error_code = update.error_code.clone(); + if update.provider_job_id.is_some() { + input.provider_job_id = update.provider_job_id.clone(); + } + events.push((entry.id.clone(), input.clone())); + } + if events.is_empty() { + return Err(CoreError::Media(format!( + "generation job does not exist: {job_id}" + ))); + } + for (event_index, (asset_id, input)) in events.iter().enumerate() { + self.append_generation_event( + ids, + asset_id, + input, + update.status, + update.progress, + update.error_code.clone(), + (event_index == 0).then_some(update.cost_credits).flatten(), + update.created_at, + ); + } + Ok(events.len()) + } + + pub(crate) fn finalize_generation_output( + &mut self, + output: PreparedGenerationOutput, + ids: &dyn IdGen, + ) -> Result<()> { + self.ensure_mutable()?; + validate_project_media_relative_path(&output.relative_path)?; + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == output.asset_id) + .ok_or_else(|| { + CoreError::Media(format!( + "generation placeholder does not exist: {}", + output.asset_id + )) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if !valid_generation_transition(input.status, GenerationJobStatus::Ready) { + return Err(CoreError::Media(format!( + "generation placeholder cannot finalize from {:?}", + input.status + ))); + } + entry.source = MediaSource::Project { + relative_path: output.relative_path, + }; + entry.duration = output.probe.duration_secs; + entry.source_width = output.probe.width; + entry.source_height = output.probe.height; + entry.source_fps = output.probe.fps; + entry.has_audio = Some(output.probe.has_audio); + input.status = Some(GenerationJobStatus::Ready); + input.progress = Some(1.0); + input.error_code = None; + let asset_id = entry.id.clone(); + let input = input.clone(); + self.append_generation_event( + ids, + &asset_id, + &input, + GenerationJobStatus::Ready, + Some(1.0), + None, + None, + output.created_at, + ); + Ok(()) + } + + pub(crate) fn fail_generation_output( + &mut self, + asset_id: &str, + error_code: &str, + created_at: Option, + ids: &dyn IdGen, + ) -> Result<()> { + if !safe_generation_error_code(error_code) { + return Err(CoreError::Media( + "generation failure code is invalid".to_string(), + )); + } + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| { + CoreError::Media(format!("generation placeholder does not exist: {asset_id}")) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if matches!(input.status, Some(GenerationJobStatus::Ready)) { + return Ok(()); + } + input.status = Some(GenerationJobStatus::Failed); + input.progress = None; + input.error_code = Some(error_code.to_string()); + let input = input.clone(); + self.append_generation_event( + ids, + asset_id, + &input, + GenerationJobStatus::Failed, + None, + Some(error_code.to_string()), + None, + created_at, + ); + Ok(()) + } + + pub(crate) fn cancel_generation_output( + &mut self, + asset_id: &str, + created_at: Option, + ids: &dyn IdGen, + ) -> Result<()> { + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| { + CoreError::Media(format!("generation placeholder does not exist: {asset_id}")) + })?; + let input = entry.generation_input.as_mut().ok_or_else(|| { + CoreError::Media("generation placeholder has no provenance".to_string()) + })?; + if matches!( + input.status, + Some( + GenerationJobStatus::Ready + | GenerationJobStatus::Failed + | GenerationJobStatus::Cancelled + ) + ) { + return Ok(()); + } + input.status = Some(GenerationJobStatus::Cancelled); + input.progress = None; + input.error_code = None; + let input = input.clone(); + self.append_generation_event( + ids, + asset_id, + &input, + GenerationJobStatus::Cancelled, + None, + None, + None, + created_at, + ); + Ok(()) + } + + pub(crate) fn save_generation_state(&mut self) -> Result { + self.ensure_mutable()?; + let target = self.project_dir.clone().ok_or(CoreError::NoProjectOpen)?; + let mut project = + Project::new_with_compatibility(target.clone(), self.compatibility.clone()); + project.timeline = self.state.timeline.clone(); + project.manifest = self.state.manifest.clone(); + project.generation_log = Some(self.generation_log.clone()); + // Generation spans media.json + generation-log.json. Publish a complete + // sibling bundle so both become visible at one rename commit point. + // The source root carries media/chat/thumbnail into the fresh stage. + let new_root = project.publish_complete_to(&target, self.project_root.as_ref())?; + self.project_root = Some(new_root); + self.generation_log_component_present = true; + Ok(target) + } + + pub(crate) fn checkpoint_generation_state(&self) -> GenerationStateCheckpoint { + GenerationStateCheckpoint { + manifest: self.state.manifest.clone(), + log: self.generation_log.clone(), + component_present: self.generation_log_component_present, + } + } + + pub(crate) fn restore_generation_state(&mut self, checkpoint: GenerationStateCheckpoint) { + self.state.manifest = checkpoint.manifest; + self.generation_log = checkpoint.log; + self.generation_log_component_present = checkpoint.component_present; + } + + #[allow(clippy::too_many_arguments)] + fn append_generation_event( + &mut self, + ids: &dyn IdGen, + asset_id: &str, + input: &GenerationInput, + status: GenerationJobStatus, + progress: Option, + error_code: Option, + cost_credits: Option, + created_at: Option, + ) { + self.generation_log + .entries + .push(GenerationLogEntry::job_event( + ids.next_id(), + input.job_id.clone().unwrap_or_default(), + input.model.clone(), + cost_credits, + input.provider.clone().unwrap_or_default(), + input.provider_job_id.clone(), + asset_id.to_string(), + status, + progress, + error_code, + created_at, + input.source_asset_id.clone(), + input.source_clip_id.clone(), + )); + self.generation_log_component_present = true; + } + /// Compatibility state inherited from the opened project. pub fn compatibility(&self) -> &ProjectCompatibility { &self.compatibility diff --git a/crates/opentake-core/tests/generation_persistence.rs b/crates/opentake-core/tests/generation_persistence.rs new file mode 100644 index 00000000..e9350db2 --- /dev/null +++ b/crates/opentake-core/tests/generation_persistence.rs @@ -0,0 +1,330 @@ +use std::fs; + +use opentake_core::{ + AppCore, GenerationStateUpdate, PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, +}; +use opentake_domain::{ + ClipType, GenerationInput, GenerationJobStatus, MediaManifestEntry, MediaSource, +}; +use opentake_project::Project; + +fn saved_project() -> (tempfile::TempDir, std::path::PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Generation.opentake"); + let mut project = Project::new(&bundle); + project.manifest.entries.push(MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(4), + source_height: Some(3), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + fs::create_dir_all(bundle.join("media")).unwrap(); + fs::write(bundle.join("media/source.png"), b"source-bytes").unwrap(); + fs::write(bundle.join("thumbnail.jpg"), b"cover").unwrap(); + (temp, bundle) +} + +fn upscale_plan() -> PreparedGenerationJob { + PreparedGenerationJob { + name: "Upscaled source".to_string(), + kind: ClipType::Image, + folder_id: None, + provider: "fal".to_string(), + input: GenerationInput { + prompt: String::new(), + model: "fal:fixture-upscaler".to_string(), + duration: 0, + aspect_ratio: String::new(), + ..Default::default() + }, + output_count: 1, + source_asset_id: Some("source-image".to_string()), + source_clip_id: Some("source-clip".to_string()), + estimated_cost_credits: Some(12), + created_at: Some(800_000_000.0), + } +} + +fn update(status: GenerationJobStatus, progress: Option) -> GenerationStateUpdate { + GenerationStateUpdate { + status, + progress, + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: Some(800_000_001.0), + } +} + +#[test] +fn placeholders_job_events_and_finalized_output_survive_restart() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, upscale_plan()) + .unwrap(); + assert_eq!(committed.placeholder_asset_ids.len(), 1); + let asset_id = committed.placeholder_asset_ids[0].clone(); + + let queued = Project::open(&bundle).unwrap(); + let placeholder = queued + .manifest + .entries + .iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + let input = placeholder.generation_input.as_ref().unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Queued)); + assert_eq!(input.source_asset_id.as_deref(), Some("source-image")); + assert_eq!(input.source_clip_id.as_deref(), Some("source-clip")); + assert_eq!(input.estimated_cost_credits, Some(12)); + assert_eq!(fs::read(bundle.join("thumbnail.jpg")).unwrap(), b"cover"); + assert_eq!( + queued.generation_log.as_ref().unwrap().entries[0].status, + Some(GenerationJobStatus::Queued) + ); + + let mut running = update(GenerationJobStatus::Generating, Some(0.2)); + running.provider_job_id = Some("fal::fixture-job".to_string()); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + running, + ) + .unwrap(); + let mut downloading = update(GenerationJobStatus::Downloading, Some(0.8)); + downloading.cost_credits = Some(11); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + downloading, + ) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + update(GenerationJobStatus::Finalizing, Some(0.9)), + ) + .unwrap(); + + let relative_path = format!("media/{asset_id}.png"); + fs::write(bundle.join(&relative_path), b"upscaled-bytes").unwrap(); + core.finalize_generation_output_for_project( + runtime.project_epoch, + &bundle, + PreparedGenerationOutput { + asset_id: asset_id.clone(), + relative_path: relative_path.clone(), + probe: ProbedMedia { + duration_secs: 0.0, + width: Some(8), + height: Some(6), + fps: None, + has_audio: false, + }, + created_at: Some(800_000_002.0), + }, + ) + .unwrap(); + + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + let media = reopened.media(); + let source = media + .entries + .iter() + .find(|entry| entry.id == "source-image") + .unwrap(); + assert_eq!(source.source_width, Some(4)); + assert_eq!(source.source_height, Some(3)); + assert_eq!( + fs::read(bundle.join("media/source.png")).unwrap(), + b"source-bytes" + ); + + let output = media + .entries + .iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!(output.source_width, Some(8)); + assert_eq!(output.source_height, Some(6)); + assert_eq!(output.source, MediaSource::Project { relative_path }); + assert_eq!( + output.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Ready) + ); + assert_eq!(fs::read(bundle.join("thumbnail.jpg")).unwrap(), b"cover"); + + let log = reopened.generation_log(); + assert_eq!(log.entries.len(), 5); + assert_eq!(log.total_credits(), 11); + assert_eq!( + log.entries.last().and_then(|entry| entry.status), + Some(GenerationJobStatus::Ready) + ); + assert!(log.entries.iter().all(|entry| { + let json = serde_json::to_string(entry).unwrap(); + !json.contains("source-bytes") && !json.contains("https://") + })); +} + +#[test] +fn invalid_progress_and_error_codes_do_not_mutate_the_durable_job() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, upscale_plan()) + .unwrap(); + let before = fs::read(bundle.join("media.json")).unwrap(); + + let invalid = GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(f64::NAN), + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: None, + }; + assert!(core + .update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + invalid, + ) + .is_err()); + assert_eq!(fs::read(bundle.join("media.json")).unwrap(), before); + + assert!(core + .fail_generation_output_for_project( + runtime.project_epoch, + &bundle, + &committed.placeholder_asset_ids[0], + "provider leaked /private/path", + None, + ) + .is_err()); + assert_eq!(fs::read(bundle.join("media.json")).unwrap(), before); +} + +#[test] +fn cancelling_a_partially_finalized_job_preserves_ready_outputs() { + let (_temp, bundle) = saved_project(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let runtime = core.runtime_snapshot(); + let mut plan = upscale_plan(); + plan.output_count = 2; + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, plan) + .unwrap(); + let ready_id = committed.placeholder_asset_ids[0].clone(); + let cancelled_id = committed.placeholder_asset_ids[1].clone(); + + for (status, progress) in [ + (GenerationJobStatus::Generating, Some(0.2)), + (GenerationJobStatus::Downloading, Some(0.8)), + (GenerationJobStatus::Finalizing, Some(0.9)), + ] { + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + update(status, progress), + ) + .unwrap(); + } + + let relative_path = format!("media/{ready_id}.png"); + fs::write(bundle.join(&relative_path), b"ready-output").unwrap(); + core.finalize_generation_output_for_project( + runtime.project_epoch, + &bundle, + PreparedGenerationOutput { + asset_id: ready_id.clone(), + relative_path: relative_path.clone(), + probe: ProbedMedia { + duration_secs: 0.0, + width: Some(8), + height: Some(6), + fps: None, + has_audio: false, + }, + created_at: Some(800_000_002.0), + }, + ) + .unwrap(); + core.cancel_generation_output_for_project( + runtime.project_epoch, + &bundle, + &ready_id, + Some(800_000_003.0), + ) + .unwrap(); + core.cancel_generation_output_for_project( + runtime.project_epoch, + &bundle, + &cancelled_id, + Some(800_000_003.0), + ) + .unwrap(); + + let reopened = Project::open(&bundle).unwrap(); + let ready = reopened + .manifest + .entries + .iter() + .find(|entry| entry.id == ready_id) + .unwrap(); + assert_eq!( + ready.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Ready) + ); + assert_eq!( + ready.source, + MediaSource::Project { + relative_path: relative_path.clone(), + } + ); + assert_eq!( + fs::read(bundle.join(relative_path)).unwrap(), + b"ready-output" + ); + + let cancelled = reopened + .manifest + .entries + .iter() + .find(|entry| entry.id == cancelled_id) + .unwrap(); + assert_eq!( + cancelled.generation_input.as_ref().unwrap().status, + Some(GenerationJobStatus::Cancelled) + ); + assert!(matches!( + cancelled.source, + MediaSource::Project { ref relative_path } if relative_path.ends_with(".pending") + )); + assert!(!bundle.join(format!("media/{cancelled_id}.png")).exists()); +} diff --git a/crates/opentake-domain/src/lib.rs b/crates/opentake-domain/src/lib.rs index c7efa6ac..18bb40c8 100644 --- a/crates/opentake-domain/src/lib.rs +++ b/crates/opentake-domain/src/lib.rs @@ -48,8 +48,8 @@ pub use keyframe::{ KeyframeInterpolatable, KeyframeTrack, }; pub use media::{ - GenerationInput, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, MediaManifestEntry, - MediaResolver, MediaSource, + GenerationInput, GenerationJobStatus, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, + MediaManifestEntry, MediaResolver, MediaSource, }; pub use signal::{ ContextSignal, EditingSkeleton, EditingStage, StageGuidance, TrackHint, TrackRole, diff --git a/crates/opentake-domain/src/media.rs b/crates/opentake-domain/src/media.rs index 3ae9ac47..36c27d15 100644 --- a/crates/opentake-domain/src/media.rs +++ b/crates/opentake-domain/src/media.rs @@ -23,6 +23,21 @@ use serde::{Deserialize, Serialize}; use crate::clip_type::ClipType; +/// Durable provider-neutral lifecycle for an asynchronous generated output. +/// Stored with `GenerationInput` so the manifest is the recovery source of +/// truth and never needs provider credentials or signed result URLs. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum GenerationJobStatus { + Queued, + Generating, + Downloading, + Finalizing, + Ready, + Failed, + Cancelled, +} + /// Where a media file lives. Encoded externally-tagged to match Swift's /// synthesized `Codable` for an enum with associated values: /// `{"external":{"absolutePath":"..."}}` / `{"project":{"relativePath":"..."}}`. @@ -100,6 +115,41 @@ pub struct GenerationInput { /// Apple-reference-date seconds (see module note on dates). #[serde(default, skip_serializing_if = "Option::is_none")] pub created_at: Option, + /// Local durable job identity. Provider job ids remain private to the host. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_id: Option, + /// Non-secret provider routing prefix (`fal`, `replicate`, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Provider job identity required for restart recovery. This is not a + /// credential and must never contain a result URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_job_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, + /// Normalized 0..1 progress. Providers without progress report phase-only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub progress: Option, + /// Fixed application-owned failure code; provider messages are never stored. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_code: Option, + /// Stable ordered output index for N-result generation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_index: Option, + /// Source asset provenance for upscale/edit flows. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_asset_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_clip_id: Option, + /// Timeline span provenance for video-to-audio generation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_start_frame: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_end_frame: Option, + /// Client-side estimate shown before submission; actual settled cost is + /// recorded once in the generation log when a provider supplies it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_cost_credits: Option, } /// Serializable manifest entry. 1:1 port of `MediaManifestEntry`. @@ -460,7 +510,28 @@ impl MediaAsset { source_fps: entry.source_fps, has_audio: entry.has_audio.unwrap_or(false), generation_input: entry.generation_input.clone(), - generation_status: GenerationStatus::None, + generation_status: match entry + .generation_input + .as_ref() + .and_then(|input| input.status) + { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => { + GenerationStatus::Generating + } + Some(GenerationJobStatus::Downloading) => GenerationStatus::Downloading, + Some(GenerationJobStatus::Finalizing) => GenerationStatus::Rendering, + Some(GenerationJobStatus::Failed) => GenerationStatus::Failed( + entry + .generation_input + .as_ref() + .and_then(|input| input.error_code.clone()) + .unwrap_or_else(|| "GENERATION_FAILED".to_string()), + ), + Some(GenerationJobStatus::Cancelled) => { + GenerationStatus::Failed("GENERATION_CANCELLED".to_string()) + } + Some(GenerationJobStatus::Ready) | None => GenerationStatus::None, + }, folder_id: entry.folder_id.clone(), pending_download_url: None, cached_remote_url: entry.cached_remote_url.clone(), diff --git a/crates/opentake-gen/src/build_params.rs b/crates/opentake-gen/src/build_params.rs index b6f6cef7..fb80cbd2 100644 --- a/crates/opentake-gen/src/build_params.rs +++ b/crates/opentake-gen/src/build_params.rs @@ -168,21 +168,44 @@ pub fn build_params( GenerationParams::Video(build_video_edit_params(input, uploaded)) } else { // Frame count: derive from start/end presence via image_urls slot. - let frame_count = input.image_urls.as_ref().map(|v| v.len()).unwrap_or(0); + let frame_count = input + .image_url_asset_ids + .as_ref() + .map(|values| values.len()) + .or_else(|| input.image_urls.as_ref().map(|values| values.len())) + .unwrap_or(0); let image_ref_count = input - .reference_image_urls + .reference_image_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_image_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); let video_ref_count = input - .reference_video_urls + .reference_video_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_video_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); let audio_ref_count = input - .reference_audio_urls + .reference_audio_asset_ids .as_ref() - .map(|v| v.len()) + .map(|values| values.len()) + .or_else(|| { + input + .reference_audio_urls + .as_ref() + .map(|values| values.len()) + }) .unwrap_or(0); GenerationParams::Video(build_video_params( input, diff --git a/crates/opentake-media/src/lib.rs b/crates/opentake-media/src/lib.rs index 762bcd73..3d42d1b5 100644 --- a/crates/opentake-media/src/lib.rs +++ b/crates/opentake-media/src/lib.rs @@ -45,6 +45,78 @@ pub mod waveform; use std::path::{Path, PathBuf}; +/// Materialize an exact visible source range as an uploadable MP4. This is used +/// by generation/upscale when `sourceClipId` is supplied, so provider uploads +/// receive the clip's trimmed source window instead of the complete asset. +pub fn trim_video_range( + source: &Path, + destination: &Path, + start_seconds: f64, + end_seconds: f64, + cancel: &MediaCancelToken, +) -> Result<()> { + if !start_seconds.is_finite() + || !end_seconds.is_finite() + || start_seconds < 0.0 + || end_seconds <= start_seconds + { + return Err(MediaError::Ffmpeg( + "invalid trimmed generation source range".to_string(), + )); + } + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent)?; + } + let duration = end_seconds - start_seconds; + let mut child = std::process::Command::new(ff::ffmpeg_path()) + .args(["-hide_banner", "-loglevel", "error", "-nostdin", "-y"]) + .arg("-ss") + .arg(format!("{start_seconds:.6}")) + .arg("-i") + .arg(source) + .arg("-t") + .arg(format!("{duration:.6}")) + .args([ + "-map", + "0:v:0", + "-map", + "0:a?", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "18", + "-c:a", + "aac", + "-movflags", + "+faststart", + ]) + .arg(destination) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|error| MediaError::Ffmpeg(format!("trim source spawn: {error}")))?; + loop { + if cancel.is_cancelled() { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_file(destination); + return Err(MediaError::Cancelled); + } + if let Some(status) = child.try_wait()? { + if !status.success() || !destination.is_file() { + let _ = std::fs::remove_file(destination); + return Err(MediaError::Ffmpeg( + "trimmed generation source could not be materialized".to_string(), + )); + } + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + // --- flat re-exports of the public API --- pub use cancel::MediaCancelToken; @@ -457,4 +529,39 @@ mod tests { v_streams.trim() ); } + + #[test] + fn trim_video_range_materializes_only_visible_window_and_honors_cancel() { + use std::process::Command; + if !ff::ffmpeg_available() || !ff::ffprobe_available() { + eprintln!("skipping: ffmpeg/ffprobe unavailable"); + return; + } + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("source.mp4"); + let trimmed = temp.path().join("trimmed.mp4"); + let generated = Command::new(ff::ffmpeg_path()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "color=size=64x48:rate=24:duration=3"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&source) + .output() + .unwrap(); + assert!(generated.status.success()); + let source_before = std::fs::read(&source).unwrap(); + + trim_video_range(&source, &trimmed, 0.75, 1.75, &MediaCancelToken::new()).unwrap(); + let trimmed_probe = probe(&trimmed).unwrap(); + assert!((trimmed_probe.duration_secs - 1.0).abs() < 0.2); + assert_eq!(std::fs::read(&source).unwrap(), source_before); + + let cancelled_path = temp.path().join("cancelled.mp4"); + let cancel = MediaCancelToken::new(); + cancel.cancel(); + assert!(matches!( + trim_video_range(&source, &cancelled_path, 0.0, 2.0, &cancel), + Err(MediaError::Cancelled) + )); + assert!(!cancelled_path.exists()); + } } diff --git a/crates/opentake-project/src/bundle.rs b/crates/opentake-project/src/bundle.rs index 6278abf7..fcd51111 100644 --- a/crates/opentake-project/src/bundle.rs +++ b/crates/opentake-project/src/bundle.rs @@ -312,6 +312,9 @@ impl Project { if let Some(source) = media_source { source.copy_media_to(publisher.stage())?; source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + source.copy_thumbnail_to(publisher.stage())?; + } } publisher.publish() } diff --git a/crates/opentake-project/src/gen_log.rs b/crates/opentake-project/src/gen_log.rs index e5739f94..1ea15e86 100644 --- a/crates/opentake-project/src/gen_log.rs +++ b/crates/opentake-project/src/gen_log.rs @@ -17,6 +17,8 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; +use opentake_domain::GenerationJobStatus; + fn default_version() -> i64 { 1 } @@ -79,6 +81,26 @@ pub struct GenerationLogEntry { /// Apple-reference-date seconds. `None` when unknown. #[serde(skip_serializing_if = "Option::is_none")] pub created_at: Option, + /// Provider-neutral durable job identity. Never a signed URL or credential. + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub asset_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, + /// Fixed application-owned code only; provider diagnostic text is private. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_asset_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_clip_id: Option, } impl GenerationLogEntry { @@ -94,6 +116,49 @@ impl GenerationLogEntry { model: model.into(), cost_credits, created_at, + job_id: None, + provider: None, + provider_job_id: None, + asset_id: None, + status: None, + progress: None, + error_code: None, + source_asset_id: None, + source_clip_id: None, + } + } + + /// Construct one append-only job lifecycle event without provider secrets. + #[allow(clippy::too_many_arguments)] + pub fn job_event( + id: impl Into, + job_id: impl Into, + model: impl Into, + cost_credits: Option, + provider: impl Into, + provider_job_id: Option, + asset_id: impl Into, + status: GenerationJobStatus, + progress: Option, + error_code: Option, + created_at: Option, + source_asset_id: Option, + source_clip_id: Option, + ) -> Self { + Self { + id: id.into(), + job_id: Some(job_id.into()), + model: model.into(), + provider: Some(provider.into()), + provider_job_id, + asset_id: Some(asset_id.into()), + status: Some(status), + progress, + error_code, + cost_credits, + created_at, + source_asset_id, + source_clip_id, } } } @@ -114,6 +179,15 @@ impl<'de> Deserialize<'de> for GenerationLogEntry { created_at: Option, // Legacy: dollars as a float. Only consulted when costCredits is absent. cost: Option, + job_id: Option, + provider: Option, + provider_job_id: Option, + asset_id: Option, + status: Option, + progress: Option, + error_code: Option, + source_asset_id: Option, + source_clip_id: Option, } let raw = Raw::deserialize(deserializer)?; let cost_credits = match raw.cost_credits { @@ -128,6 +202,15 @@ impl<'de> Deserialize<'de> for GenerationLogEntry { model: raw.model, cost_credits, created_at: raw.created_at, + job_id: raw.job_id, + provider: raw.provider, + provider_job_id: raw.provider_job_id, + asset_id: raw.asset_id, + status: raw.status, + progress: raw.progress, + error_code: raw.error_code, + source_asset_id: raw.source_asset_id, + source_clip_id: raw.source_clip_id, }) } } diff --git a/crates/opentake-project/src/project_root.rs b/crates/opentake-project/src/project_root.rs index 42c39ae8..d3bc2347 100644 --- a/crates/opentake-project/src/project_root.rs +++ b/crates/opentake-project/src/project_root.rs @@ -167,6 +167,14 @@ impl ProjectRoot { ) } + /// Preserve the optional project cover across complete-bundle publication. + pub(crate) fn copy_thumbnail_to(&self, destination: &ProjectRoot) -> Result<()> { + if let Some(bytes) = self.read_optional(crate::layout::THUMBNAIL_FILE)? { + destination.write_atomic(crate::layout::THUMBNAIL_FILE, &bytes)?; + } + Ok(()) + } + fn copy_directory_component_to( &self, destination: &ProjectRoot, diff --git a/docs/architecture/BUGS.md b/docs/architecture/BUGS.md index 83c6d339..4dda349b 100644 --- a/docs/architecture/BUGS.md +++ b/docs/architecture/BUGS.md @@ -39,13 +39,13 @@ --- -### B4. `canGenerate` 硬编码为 `false`（中） +### B4. `canGenerate` 硬编码为 `false`（已修复） | 属性 | 值 | |---|---| -| **位置** | `crates/opentake-agent/src/tools/...` — 具体待定位 | -| **描述** | Agent 工具的 `canGenerate` 返回值硬编码为 `false`，即使 BYOK key 已配置 | -| **影响** | Agent 无法进行任何 AI 生成操作 | +| **位置** | `crates/opentake-agent/src/mcp/{dispatch,generation}.rs`、`src-tauri/src/generation.rs` | +| **修复** | `canGenerate` 由托管凭据或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 凭据动态派生；无可用凭据时四个付费工具不发布，有凭据时 MCP 与 Chat 共用同一异步生产桥 | +| **验证** | `configured_capability_and_cost_authorization_gate_dispatch` 覆盖有/无能力与显式成本授权；生产 mock provider 路径覆盖 image/video/audio/upscale | --- @@ -112,7 +112,7 @@ | **B1** | swapMedia IPC 缺失 | 🔴 关键 | 单功能完全不可用 | 低（~10 行代码） | | **B2** | 前端帧数学截断不一致 | 🟠 高 | 所有媒体导入时长偏差 | 低（3 处 Math.round→floor） | | **B3** | rippleDeleteRanges 忽略 clipId | 🟠 高 | Agent 工具精度下降 | 中 | -| **B4** | canGenerate 硬编码 false | 🟡 中 | AI 生成功能不可用 | 低 | +| **B4** | canGenerate 硬编码 false（已修复） | ✅ 已关闭 | 动态生成能力已恢复 | — | | **D1** | 预览未接入 GPU 合成 | 🟠 高 | 所有编辑效果不可见 | 中 | | **D2** | Agent 工具 30% stub | 🟠 高 | AI 协作核心缺失 | 高 | | **D3** | 缩略图始终 None | 🟡 中 | 媒体库 UX 差 | 中 | diff --git a/docs/architecture/CAPCUT-GAP.md b/docs/architecture/CAPCUT-GAP.md index f2928d69..bca972f6 100644 --- a/docs/architecture/CAPCUT-GAP.md +++ b/docs/architecture/CAPCUT-GAP.md @@ -89,6 +89,7 @@ - **前置依赖**:FFmpeg vidstab 路线仅依赖 ffmpeg-next(已有);自研路线依赖光流(与追踪共用);补偿应用依赖关键帧引擎/wgpu ### 画质超清修复(super-resolution / enhance) — `partial` · 难度 medium · 优先级 p1 +- **OpenTake 当前状态(2026-07-29)**:BYOK 云端 2x 超分竖切已完成。`upscale_media` 经共享 GenerationBridge 创建耐久占位、上传源素材、轮询 Replicate、校验结果尺寸严格为源宽高 2 倍并作为新资产导入；原资产字节与元数据保持不变。UI 已提供进度/取消/失败/重试，重试要求重新确认成本。确定性生产路径测试覆盖成功、取消不导入、恢复、鉴权/限流、源不变与精确 2x。本地 Real-ESRGAN/SeedVR 修复轨仍未实现，因此总体仍标 `partial`。 - **判定依据**:上游有『Upscale』入口但纯云端、且仅放大不修复:AIEditTab『AI Enhance / Upscale / Enhance resolution with AI』(Inspector/AIEditTab.swift:31-36)、UpscaleModelConfig 从 Convex ModelCatalog 拉取(Generation/Catalog/UpscaleModelConfig.swift)、EditSubmitter.submitUpscale 把 sourceURL+durationSeconds 发后端(UpscaleModelConfig.swift:3-15)、需登录订阅否则禁用(ToolExecutor+Generate.swift:320)、video 仅 <2160 可放大(MODULE-PORT-MAP.md:528)。本地零推理。OpenTake 设计稿 Generation 章把它归为 cloud-rebuild,未规划本地超分实现(MODULE-PORT-MAP.md:231、ARCHITECTURE §8 只讲生成代理双模,未列本地 SR)。 - **落点(crate/层)**:opentake-gen(BYOK 云超分,复刻 UpscaleGenerationParams + job 状态机,设计稿已含)+ 新增 opentake-media 本地超分 worker(ort/candle 跑 SR 模型)作为 OpenTake 增强项 - **实现方案**:双轨。① 云端 BYOK(低成本起步、对齐上游):opentake-gen 已规划复刻 GenerationParams 联合类型与 job 抽象(ARCHITECTURE §8、ROADMAP Phase 9),upscale 作为其中一类,用户自带 fal/Replicate key 直连厂商超分模型(Topaz/SeedVR/Real-ESRGAN 端点),零运营成本,这是把上游云能力『去 Convex 化』的自然落点。② 本地推理(OpenTake 反超点,p2):经 ort 跑 Real-ESRGAN / SwinIR(图像)或 SeedVR2(视频,时序一致),复用 SigLIP2/RVM 同套 ONNX 通道与 ModelDownloader;『修复』(去噪/去压缩伪影/人脸增强 GFPGAN/CodeFormer)与『放大』分开提供,补上上游只放大不修复的短板。③ 处理结果作为新媒体资产回填时间线(沿用上游 upscale 回填语义)。 @@ -233,4 +234,3 @@ - **落点(crate/层)**:opentake-gen(扩展 AudioGenerationParams 增加 referenceAudioURL/voiceId 字段 + provider adapter)+ services/opentake-gen-proxy(对接 ElevenLabs voice-clone / fal / MiniMax 等)+ 音色库管理(可存于 .opentake 工程或账户级)+ keyring 存厂商 key。 - **实现方案**:本地无法稳定实现高质量克隆(需说话人编码+声码器大模型),走外部 API 最务实。两步:① 创建音色:用户提供参考音频样本→预签名上传→调 ElevenLabs Instant Voice Cloning / MiniMax voice clone 等→拿回一个 voiceId,存入音色库。② 使用:把该 voiceId 当作 TTS 的 voice 传入现有 generate_audio 流水线即可复用全部落轨逻辑。需扩展 OpenTake 既有的 AudioGenerationParams 联合类型(report03 已设计该 enum)加 referenceAudioURL/clonedVoiceId 字段,catalog 用 caps 标注"该模型支持克隆"。合规上需加"声音授权确认"。若坚持本地路线,可评估 OpenVoice/XTTS(candle/ort 跑)做轻量零样本音色迁移,但质量与多语稳定性远逊云端,定位为可选实验功能。 - **前置依赖**:依赖 opentake-gen 生成后端 + 上传预签名 + keyring 密钥存储先就绪;依赖支持克隆的外部 provider;与 TTS(图文成片配音)、数字人驱动音频可串联复用。 - diff --git a/docs/architecture/FULL_PROJECT_SCAN_REPORT.md b/docs/architecture/FULL_PROJECT_SCAN_REPORT.md index 4d933ed4..0324b889 100644 --- a/docs/architecture/FULL_PROJECT_SCAN_REPORT.md +++ b/docs/architecture/FULL_PROJECT_SCAN_REPORT.md @@ -95,9 +95,9 @@ **问题/风险**: - 38 个发现面工具均已脱离 `not yet implemented` 分支；`inspect_media` 已通过 Tauri `MediaBridge` 接入图片/视频抽帧及本地转写。 -- 生成/超分/Motion 六个兼容线名在生产后端完成前从 MCP、Chat 与系统提示隐藏；Lottie 源检查仍明确不支持。 -- 细节:生成作业/授权/恢复仍未实现，`canGenerate` 仍为 false；这些是未完成能力，不是可发现工具。 -- 严重性: **高** (AI 协作核心缺失 → transcript-driven 编辑等 workflow 不可用)。 +- 生成/超分四个工具已接共享生产 GenerationBridge：仅在托管或兼容 BYOK 凭据可用时动态发布；Motion 两个兼容线名仍隐藏，Lottie 源检查仍明确不支持。 +- 生成作业已具备成本授权、耐久占位/日志、进度、取消、部分成功、失败码、重试与重启恢复；结果下载受协议/地址/大小/重定向约束并在探测后原子导入。 +- 严重性: **中**（生成主竖切已闭合；高级 AI workflow、Motion 与付费真实账号冒烟仍属发布验证项）。 **测试**: mcp_http.rs (传输) 存在；全工具执行弱。 @@ -122,7 +122,7 @@ **问题/风险**: - Import: thumbnails 永远 None (media.rs:79，“placeholder”)、无进度/反馈、扩展静默丢弃、folder 浏览不全。 - Export: H.264 spine 好，但 H265/ProRes 未接、无进度/取消。 -- Gen: 占位部分，但 agent 路径 stub。 +- Gen: image/video/audio/upscale 已从 MCP/Chat 走生产桥并有无付费网络的 provider 合约与应用集成测试；时间线视频源音频生成已预留渲染路径，但当前内置目录没有 `inputs:["video"]` 的音频模型，因此按能力拒绝。 - Bundle 细微差异 (chat 目录名)。 - 严重性: **中高** (工作流断裂)。 diff --git a/docs/architecture/HANDOFF-2026-07.md b/docs/architecture/HANDOFF-2026-07.md index 6c393c17..ba4b2963 100644 --- a/docs/architecture/HANDOFF-2026-07.md +++ b/docs/architecture/HANDOFF-2026-07.md @@ -143,14 +143,14 @@ default-off Rust renderer or to perform the then-missing first device check. 3. 前端:AgentPanel 消息列表 + streaming 渲染 + 工具调用卡片;Context Signal(`signal/`)注入系统提示已有,接上即可。 - **验收**:面板内发"把这段静音剪掉"能走 tighten_silences 工具链并落 EditCommand;无 key 时有引导文案。 -### 3.4 【P1】生成式 UI + `generate_*`/`upscale` 去 stub(#30 切片) +### 3.4 【P1，代码竖切已完成】生成式 UI + `generate_*`/`upscale` 去 stub(#30 切片) -- **现状**:`opentake-gen` client/job/provider/keys 已建成但尚未形成生产调用；`GenerateVideo/Image/Audio | UpscaleMedia` 兼容线名在后端完成前不进入 MCP/Chat 发现面或系统提示；`encode_timeline(..., false)` 仍硬编 canGenerate=false；web 无生成 UI。 +- **现状(2026-07-29)**:`GenerateVideo/Image/Audio | UpscaleMedia` 已通过 MCP/Chat 共用的 Tauri GenerationBridge 接入 fal/Replicate/OpenAI/ElevenLabs；先持久化占位和 generation log，再异步上传/提交/轮询/下载/探测/导入。`canGenerate` 由可用凭据派生，四工具仅在能力可用时动态发布。MediaPanel 卡片显示进度、取消、固定错误码与需重新确认成本的重试。 - **怎么写**: 1. `src-tauri` 命令层:`generate_start(params)->job_id` / `generate_status` / `generate_cancel`,内部 `opentake-gen::client` 异步 job 轮询,产物落项目 `media/` 并走 `import_media` 现有通路(保证出现在素材面板 + 可撤销)。 2. MCP dispatch 同一命令复用,去掉 4 个 stub;canGenerate 改为"有 BYOK key 即 true"(`keys.rs` 可查)。 3. 前端:对照上游 GenerationView 做 MediaPanel「AI 生成」分区(与 #91 的左侧分区合流):模型选择(`catalog/` 已有 list_models)、提示词、参考图、进度卡片。 -- **验收**:配 fal.ai key 后文生图落库并可拖上时间线;无 key 显示引导;取消可用。 +- **验收状态**:无付费网络的配置 provider 生产路径已覆盖 image/video/audio/upscale、N 图顺序、部分/全部失败、取消、恢复、鉴权/限流、重试和精确 2x；完整工作区测试通过。真实账号付费请求保留到 Beta 实机验证清单，未在自动化中消费用户额度。 ### 3.5 【P1】Save as Media(#48 尾巴) @@ -198,7 +198,7 @@ default-off Rust renderer or to perform the then-missing first device check. | 布局常量(#148) | 对照上游 rulerHeight/dropZoneHeight/trackHeight 改 `theme.ts`,纯数值对齐 | | CSP 加固(#161) | null→非 null 白屏高风险,**必须真机逐项验证**,独立小 PR | | Storage/Models 设置页 | `SettingsView.tsx` 加两 pane:模型缓存管理(whisper/SigLIP 已下载模型列表+删除)、存储占用(项目/缓存目录大小) | -| MCP 隐藏能力 | `InspectMedia` 已接 MediaBridge（剩 Lottie）；生成/超分/Motion 六个兼容线名在生产后端完成前不进入发现面 | +| MCP 隐藏能力 | `InspectMedia` 已接 MediaBridge（剩 Lottie）；生成/超分四工具按凭据能力动态发布，Motion 两个兼容线名仍隐藏 | | 技术债 | `fcpxml.rs` 1489 行拆分;`export_fcpxml` 名实不符(产物 XMEML)可改名 `export_xml`;`library.rs:322` remove 静默吞错补 `tracing::warn!` | --- diff --git a/docs/architecture/MODULE-PORT-MAP.md b/docs/architecture/MODULE-PORT-MAP.md index cc467060..58ae1809 100644 --- a/docs/architecture/MODULE-PORT-MAP.md +++ b/docs/architecture/MODULE-PORT-MAP.md @@ -516,6 +516,7 @@ - 【GenerationService.generate 主流程 — 必须一比一复刻的顺序与边界】1) count = clamp(numImages,1,4);baseName = name ?? prompt 前30字符;resolvedFolderId 仅当该 folder 存在才保留。2) 目标目录:有 projectURL 则 projectURL/'media' 并创建,否则系统临时目录;占位文件名 'gen-.'。3) 同步创建 count 个占位 MediaAsset(generationStatus=.generating, folderId 设好, 追加进 editor.mediaAssets),记 primaryId=占位[0].id 并立即 return(异步在后台 Task 继续)。4) 异步:若给了 preUploadedURLs 则直接用;否则:取 references 的 url 列表 urlsToUpload;若 trimmedSourceOverride.hasTrim 且非空则用 VideoTrimExtractor 替换 urlsToUpload[0] 并登记待清理;若有 preprocessRef 则对每个 reference 并发执行(TaskGroup)可能产出改写 URL;计算 cacheKeys(只有既无 preprocessRef 且不是被裁剪的第0项时,才用该 MediaAsset 作缓存键);uploadReferences 并发上传(命中 freshRemoteURL 缓存则跳过实际上传,上传成功则写回缓存 TTL=6天)。5) finalGenInput:若给 snapshotRefs 则调用它写回各类 URL,否则把 uploaded 整体塞进 imageURLs;createdAt 为空则置 now;把 finalGenInput 赋给所有占位的 generationInput。6) params=buildParams(uploaded)→runJob。7) 任意环节抛错:所有占位 generationStatus=.failed('Upload failed: …'),调 onFailure。8) defer 清理所有临时文件。 - 【runJob 提交+订阅状态机】生成 runId(UUID前8)。submit(model,params,projectId) 拿 jobId,失败→占位.failed + onFailure。subscribe(jobId) 拿 Combine publisher(nil→.failed('Backend not configured'))。把 publisher 包成 AsyncStream 在主线程消费:job.status == .queued/.running 时 continue;== .succeeded → finalizeSuccess 后 return;== .failed → 占位.failed(job.errorMessage ?? 'Generation failed') + onFailure + return。Rust 复刻:用一个状态轮询/推送通道(WebSocket 或轮询 REST)驱动同一状态机。 - 【finalizeSuccess 下载落地 — N 图分发规则】resultUrls 为空→全部占位 .failed('No URL in response')+onFailure。若 resultUrls.count < 占位数,多出的占位标记失败(只 log 不报错)。逐 i 配对占位[i]↔resultUrls[i]:URL 非法→.failed('No URL for placeholder');否则 downloadAndFinalize 成功才 onComplete?(占位) 并计入 finalized。最后只要有 finalized 就对 finalized.first 发 generationComplete 通知(count=finalized.count);全失败→onFailure。 +- 【OpenTake 复刻状态(2026-07-29)】`finalize_terminal_outputs` 已按占位下标确定性配对全部 URL，覆盖 0/少于/等于/多于 N、下载失败、部分成功与全失败；每个输出持久化独立终态，重复回调由 lease + 耐久状态幂等化。Tauri 下载器只接收受限 data/HTTPS，逐跳拒绝私网/凭据/非 443、限制重定向与 1 GiB 解码后字节数，取消或失败清理 staging；探测真实容器后才导入，签名 URL 和 provider 正文不落盘。 - 【downloadAndFinalize 落地细节】先置 .downloading。URLSession 下载到临时文件;若远端扩展名非空、与占位 url 扩展名不同、且该扩展名是已知 ClipType,则把目标 url 改成新扩展名(纠正真实类型)。删除旧文件→移动临时文件到目标。清 pendingDownloadURL、status=.none、importMediaAsset(skipAppend:true)→appendGenerationLog→finalizeImportedAsset(异步补全元数据如尺寸/帧率/音轨)。失败:记 pendingDownloadURL=远端 URL、status=.failed(message),供 retryDownload 重试。 - 【uploadReferences 并发+缓存+顺序保持】对每个 url 起 TaskGroup 任务:cacheKey.freshRemoteURL 命中则直接产出该 URL;否则按文件扩展名/回退类型推断 contentType(jpg→image/jpeg, png, webp, heic, gif, mp4/m4v→video/mp4, mov→video/quicktime, mp3→audio/mpeg, wav, m4a→audio/mp4;回退:image→image/jpeg, video→video/mp4, audio→audio/mpeg, text→octet-stream, lottie→application/json),上传后写缓存。最终按原始下标排序返回(顺序对后续 frames/refs 切分至关重要)。 - 【上传引用缓存 freshRemoteURL】MediaAsset.cachedRemoteURL + cachedRemoteURLExpiresAt;freshRemoteURL 仅当存在且 expiresAt>now 才返回。TTL=6*24*60*60 秒。缓存只对『字节纯净』的资产生效(未裁剪、未预处理)。Rust 复刻:按资产内容哈希做上传去重缓存更稳。 @@ -1292,4 +1293,3 @@ MCPService 端口 19789 与工具注册属 Agent/MCP 子系统,App 层只做开 **移植策略**:ToolbarView 是纯 SwiftUI 表现层，需在 React/TS 前端整体重建为一个工具栏组件，不可直接移植。但它本身几乎没有逻辑——真正要忠实复刻的是它调用的 EditorViewModel 编辑算法(split/trim/addText)，那些应放到 Rust core 实现，前端按钮只发命令。具体替换方案：(1) 布局：用 flex 行 + 分隔符/弹簧重建；图标用任意图标库(如 lucide)替换 SF Symbols(cursorarrow→鼠标、scissors→剪刀、square.split.2x1→分割、放大镜→zoom)。(2) 悬停高亮 hoverHighlight→CSS :hover + 圆角背景，命中区用 padding/伪元素扩大。(3) 撤销/重做：在 Rust core 维护撤销栈(对应 NSUndoManager 的 bidirectional swap 模式——每个 mutation 记录 before/after Timeline 或 per-clip 快照，撤销时回写并重新注册逆向 swap)；前端按钮与 Cmd+Z/Shift+Cmd+Z 都调用 core 的 undo()/redo() 命令(invoke)。(4) 工具模式：toolMode 作为前端/或 core 的 UI 状态枚举(Pointer/Razor)，razor 模式下时间线点击发 split 命令。(5) 分割/裁剪/新增文字算法严格按 coreLogic 在 Rust 实现：注意 round() 取整、源帧=时间线帧*speed、fade 在分割时左清淡出右清淡入、关键帧轨切点插边界帧并 rebase、trim 是 overwrite 式(不波纹)。secondsToFrame 用截断(Int(s*fps))而非四舍五入，务必一致。(6) 缩放滑块：前端 input[type=range] 存 ln(scale)，区间 [ln(minZoom), ln(40)]，回写 scale=Math.exp(v)；minZoom 计算(availableWidth/(totalFrames*3)，钳制到[0.0001,40])可放前端或 core。(7) 文字渲染独立于媒体合成(syncTextLayers→在 FFmpeg/前端 overlay 层单独绘制文字)，新增文字时把文字 clip 放到顶层视频轨、按 currentFrame 起点、默认 3 秒。无 Apple 私有框架阻塞，无 blocker。 **关键文件**:Sources/PalmierPro/Toolbar/ToolbarView.swift、Sources/PalmierPro/Editor/ViewModel/EditorViewModel+ClipMutations.swift、Sources/PalmierPro/Editor/ViewModel/EditorViewModel+Ripple.swift、Sources/PalmierPro/Editor/OverwriteEngine.swift、Sources/PalmierPro/Models/Timeline.swift、Sources/PalmierPro/Utilities/Constants.swift - diff --git a/docs/architecture/ROADMAP.md b/docs/architecture/ROADMAP.md index 95d5266c..8f1e8503 100644 --- a/docs/architecture/ROADMAP.md +++ b/docs/architecture/ROADMAP.md @@ -60,7 +60,7 @@ 3. 应用内 chat(reqwest→Anthropic SSE,BYOK;prompt caching)。 4. **OpenTake 增强**:分层可组合系统提示词 + 模型策略配置化;高阶工具 `remove_filler_words`/`tighten_silences`;写工具返回结构化 JSON;新增 `get_capabilities`。 - **验证**:`claude mcp add` 能连;每个工具走通;应用内 chat 能完成多步链式编辑;助手专属 undo 正确。 -- **进度**:`list_models` 已接 `opentake-gen` 本地 catalog；`inspect_media` 已接桌面媒体桥并有图片端到端、视频故事板和转写契约测试。当前发现面发布 38 个真实路径工具；`generate_*`/`upscale_media`/Motion 六个兼容线名在 async ProviderRegistry + BYOK/确定性渲染完成前保持隐藏，仍是 Phase 7 未完成项。 +- **进度**:`list_models` 与 `inspect_media` 已接生产桥。基础发现面为 38 个真实路径工具；存在托管或兼容 BYOK 凭据时动态增加 `generate_*`/`upscale_media` 四个工具，无凭据时保持隐藏；Motion 两个兼容线名仍未发布。生成与 Chat/MCP 复用同一 Dispatcher/GenerationBridge。 ## Phase 8 — 文字/字幕渲染 + 转写 + 语义搜索 - **做**:cosmic-text + tiny-skia/Vello 文字渲染(阴影/描边/背景/对齐/换行,逐帧 opacity)接入合成器;whisper-rs 转写(word/segment 时间戳,`TranscriptionResult` 模型复用);candle/ort 跑 SigLIP2 + tokenizers 做视觉/口语搜索。 @@ -75,6 +75,7 @@ 对应 `opentake-gen` + `services/opentake-gen-proxy`。 - **做**:`GenClient`(复刻 `GenerationParams` 联合类型 + job 状态机);**BYOK 模式**(本地直连 fal/Replicate/OpenAI,keyring 存 key,内置静态 models catalog);**托管模式**(axum 代理 + provider adapters + 对象存储预签名 + 可选积分计费)。 - **验证**:BYOK 下能用自己的 fal key 生图/生视频并落回时间线;模型目录数据驱动 UI;托管代理可自部署。 +- **进度(2026-07-29)**:provider-neutral 耐久作业、BYOK/托管授权、成本确认、fal/Replicate/OpenAI/ElevenLabs dispatch、N 输出终局化、进度/取消/重试/恢复、MediaPanel 状态与安全下载已完成；确定性配置 provider 测试覆盖 image/video/audio/upscale 且不发起付费网络。托管代理自部署与真实账号付费冒烟仍需在 Beta 发布验证阶段完成。 - **进阶扩展 · AIGC 编排(ADVANCED-FEATURES E 层)**:智能剪口播(本地词级转写+静音检测→Rust 内算 ripple,高阶工具 `remove_filler_words`/`tighten_silences`)、图文成片(agent 编排既有工具+SigLIP2 选素材)、音色克隆(ElevenLabs 等)、虚拟数字人(HeyGen/fal,新增 catalog kind)、多语种字幕翻译(MT/LLM,保时码)。 ## Phase 10 —(新)Motion Canvas 动效 / AI Video 插件 diff --git a/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md b/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md index 633120e8..aac026fc 100644 --- a/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md +++ b/docs/audit/2026-07-14/implementation-plans/agent-settings-generation-implementation.md @@ -448,7 +448,7 @@ - Add mocked-provider and application integration tests for every named authorization, placeholder, progress, cancellation, finalization, persistence, and failure branch; the affected suites must pass without paid network calls. - Exercise the production MCP or UI path with a deterministic local/mock provider and retain exact manifest, job-state, command-result, and runtime evidence before reclassification. -- [ ] **Step 1: Write or extend every reviewed owning test** +- [x] **Step 1: Write or extend every reviewed owning test** - `crates/opentake-agent/tests/generation_dispatch.rs#placeholder_persist_finalize_all_results_and_failures` (reviewed-planned) — Reviewed planned test belongs in this tracked owning runner beside the mapped product boundary. - `crates/opentake-agent/tests/generation_dispatch.rs#placeholder_persists_and_every_terminal_result_finalizes_once` (reviewed-planned) — Reviewed planned test belongs in this tracked owning runner beside the mapped product boundary. @@ -457,20 +457,20 @@ Each assertion must exercise every covered candidate through the mapped product boundary; an existing-owned test may be extended, while a reviewed-planned test must be added at the declared runner path. -- [ ] **Step 2: Run all focused tests and verify RED** +- [x] **Step 2: Record the RED-evidence disposition** - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact` - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact` - Run: `cargo test -p opentake-gen upscale_uses_first_upload_as_source` - Run: `cargo test -p opentake-gen byok_submit_then_watch_to_succeeded` - Expected: FAIL because one or more of the 15 candidate-bound contracts are not yet satisfied. + Historical RED output for the four exact planned tests was not retained before the audit-recovery branch began, so it is not fabricated here. Gap-driven regression tests added during implementation did reproduce failures before the fixes (video data-result acceptance and partial-finalization transition coverage); the retained GREEN commands and runtime artifact are the auditable completion evidence. -- [ ] **Step 3: Implement the minimal vertical slice** +- [x] **Step 3: Implement the minimal vertical slice** Modify only `crates/opentake-agent/src/mcp/dispatch.rs#run_body`, `crates/opentake-agent/src/mcp/generation.rs#GenerationDispatcher`, `crates/opentake-gen/src/build_params.rs#build_image_params`, `crates/opentake-gen/src/build_params.rs#build_upscale_params`, `crates/opentake-gen/src/build_params.rs#build_video_params`, `crates/opentake-gen/src/client.rs#GenClient`, `crates/opentake-gen/src/client.rs#GenClient::submit`, `crates/opentake-gen/src/client.rs#GenClient::submit_byok`, `crates/opentake-gen/src/client.rs#GenClient::watch`, `src-tauri/src/generation.rs#GenerationBridge`, `web/src/components/agent/AgentPanel.tsx#AgentPanel`, `docs/architecture/BUGS.md`, `docs/architecture/CAPCUT-GAP.md`, `docs/architecture/FULL_PROJECT_SCAN_REPORT.md`, `docs/architecture/HANDOFF-2026-07.md`, `docs/architecture/MODULE-PORT-MAP.md`, `docs/architecture/ROADMAP.md`, `docs/modules/opentake-agent/SPEC.md`, `docs/specs/agent/2-tools.md`, `docs/upstream-analysis/04-MCP与Agent工具.md` as required to satisfy every listed acceptance criterion, including visible success and explicit failure/recovery behavior. -- [ ] **Step 4: Run all focused tests and verify GREEN** +- [x] **Step 4: Run all focused tests and verify GREEN** - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact` - Run: `cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact` @@ -479,12 +479,14 @@ Expected: PASS with every candidate-bound assertion executed. -- [ ] **Step 5: Run the subsystem regression gate** +- [x] **Step 5: Run the subsystem regression gate** Run: `cargo fmt --all -- --check && cargo test --workspace --no-fail-fast` Expected: PASS with no new warnings or unrelated changes. +Completion evidence (2026-07-29): the four named focused tests pass; `generation::tests` covers configured image/video/audio/upscale production dispatch, authorization, N ordering, cancel, restart, retry, auth/rate-limit mapping, safe result URLs and exact 2x; `generation_persistence` covers durable logs/costs/restart and ready+cancelled partial terminal state. Full `cargo fmt`, Clippy `-D warnings`, workspace tests, web production build, 703 web tests, and `git diff --check` pass. Exact commands and asserted artifact state are recorded in `runtime-artifacts/automated/generation-finalization-2026-07-29.md`. + ### Task 3: advanced-ai-workflows (implementation-slice-aec7c23c8d96431e) **Covered records:** diff --git a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md new file mode 100644 index 00000000..1739d9c5 --- /dev/null +++ b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md @@ -0,0 +1,33 @@ +# Generation dispatch/finalization runtime evidence — 2026-07-29 + +Scope: `AG-generation-dispatch-finalization + generation-upscale-finalization` (`implementation-slice-f60bdd3e656a7cb0`). All provider responses are deterministic local `MockTransport` fixtures; no paid provider request or user credit was consumed. + +## Production-path artifact assertions + +- `generate_image`: production Dispatcher → shared Tauri GenerationBridge → configured fal adapter; two durable placeholders are returned before background work, paired in output-index order, probed as 2×2 PNG, persisted `ready`, and resolve to real project media files. +- `generate_video`: configured fal adapter returns a valid 16×16 MP4 fixture; the original placeholder identity becomes a durable video asset with probed dimensions. +- `generate_audio`: configured OpenAI adapter returns a valid WAV fixture; the placeholder becomes a durable audio asset with `hasAudio=true` and positive probed duration. +- `upscale_media`: configured Replicate adapter uploads a 2×2 source fixture and returns 4×4 PNG; the output is exactly 2x and the source file bytes remain byte-for-byte identical. +- Cancellation leaves no imported result; ready outputs in a partially finalized N-output job remain ready while non-terminal siblings become cancelled. +- Restart with provider id resumes and finalizes; restart before provider id persists `GENERATION_RESTART_RETRY_REQUIRED` without any resubmission. Retry requires fresh cost authorization and creates a new job. +- Authentication/rate-limit failures persist fixed safe codes. Local/private result targets are rejected. Provider body, credentials, and signed result URLs are absent from project persistence. + +## Verification commands + +```text +cargo test -p opentake-agent --test generation_dispatch placeholder_persist_finalize_all_results_and_failures -- --exact +cargo test -p opentake-agent --test generation_dispatch placeholder_persists_and_every_terminal_result_finalizes_once -- --exact +cargo test -p opentake-gen upscale_uses_first_upload_as_source +cargo test -p opentake-gen byok_submit_then_watch_to_succeeded +cargo test -p opentake-tauri generation::tests +cargo test -p opentake-core --test generation_persistence +cargo test -p opentake-media trim_video_range_materializes_only_visible_window_and_honors_cancel +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace --no-fail-fast -q +pnpm build +pnpm test +git diff --check +``` + +Final result: all commands passed. Workspace tests reported zero failures; repository-declared ignored tests remained ignored. Web result: 70 test files, 703 tests passed. Vite emitted only the pre-existing chunk-size/dynamic-import optimization warnings. diff --git a/docs/modules/opentake-agent/SPEC.md b/docs/modules/opentake-agent/SPEC.md index 7bef9029..08c040bc 100644 --- a/docs/modules/opentake-agent/SPEC.md +++ b/docs/modules/opentake-agent/SPEC.md @@ -156,6 +156,8 @@ capabilities: { resources: { subscribe:false, listChanged:false }, 枚举顺序严格按 `ToolName`（`ToolDefinitions.swift:4-36`）：`get_timeline, get_media, add_clips, insert_clips, remove_clips, remove_tracks, move_clips, set_clip_properties, set_keyframes, split_clip, ripple_delete_ranges, undo, add_texts, add_captions, generate_video, generate_image, generate_audio, upscale_media, import_media, list_models, inspect_media, get_transcript, inspect_timeline, search_media, list_folders, create_folder, move_to_folder, rename_media, rename_folder, delete_media, delete_folder`。 +> OpenTake 生产发现面按能力动态组成：基础 38 个真实路径工具始终可见；托管凭据或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 可用时再发布四个生成工具。四个工具都要求调用方传 `costAuthorized:true`，MCP 与应用内 Chat 共用同一 GenerationBridge；无凭据时 `get_timeline.canGenerate=false` 且工具不进入目录/系统提示。 + #### A. 读 / 内省（只读，7 个） | # | 工具 | 关键参数（schema 字段；`*`=required） | required 字段 | 背后命令（opentake-core） | Context Signal 附加（§6） | @@ -189,10 +191,10 @@ capabilities: { resources: { subscribe:false, listChanged:false }, | # | 工具 | 关键参数 | required | 背后命令 | |---|---|---|---|---| -| 20 | `generate_video` | `prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交，立即返回 placeholder asset ID；**花钱、不可撤销** | -| 21 | `generate_image` | `prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`referenceMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交 placeholder | -| 22 | `generate_audio` | `prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | （无） | `opentake-gen`：TTS / 文生乐 / 视频配乐；时间线区间结果自动落轨 | -| 23 | `upscale_media` | `mediaRef*`,`model?`,`sourceClipId?` | `mediaRef` | `opentake-gen`：升分辨率 placeholder | +| 20 | `generate_video` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：耐久占位后异步提交；进度/取消/重试/恢复；**花钱、不可撤销** | +| 21 | `generate_image` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`numImages?:1..4`,`referenceMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：异步提交 N 个有序耐久占位并逐项终局化 | +| 22 | `generate_audio` | `costAuthorized*:bool`,`prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | `costAuthorized` | `opentake-gen`：TTS / 文生乐 / 能力允许时的视频配乐；时间线区间先确定性渲染再上传 | +| 23 | `upscale_media` | `costAuthorized*:bool`,`mediaRef*`,`model?`,`sourceClipId?` | `costAuthorized,mediaRef` | `opentake-gen`：异步 2x，占位终局化为新资产且源资产不变 | | 24 | `import_media` | `source*`{三选一 `url?`(HTTPS≤1GB)/`path?`(本地，可目录递归)/`bytes?`(base64≤~15MB)，`mimeType?`},`name?`,`folderId?` | `source` | `opentake-core`/`opentake-project`：url 在阻塞工作线程中逐块下载、path/bytes 同步；扩展名/MIME/探测类型白名单；验证完成后原子发布 manifest | URL 导入禁用自动重定向并逐跳重新校验 HTTPS、host 与 userinfo；`Content-Length` 只做提前拒绝，逐块累计的解码后字节数才是 1 GiB 硬上限。下载先写 retained project capability 下的未提交 leaf，探测、容器/流类型、project epoch/目录、folder 与 leaf identity 全部通过后，才把候选 manifest 原子写入并发事件。任何此前错误或显式 MCP `notifications/cancelled` 都由 guard 擦除/删除 staging，并保持 live manifest 与磁盘 `media.json` 原字节不变。取消是 rmcp 的显式 request-id 通知语义；不承诺仅因原始 TCP 断连而取消。生产 HTTPS 请求与每个响应 chunk 都通过 `tokio::select!` 同时监听取消令牌，不等待长网络超时才清理。 diff --git a/docs/specs/agent/2-tools.md b/docs/specs/agent/2-tools.md index 162130d3..76c4d224 100644 --- a/docs/specs/agent/2-tools.md +++ b/docs/specs/agent/2-tools.md @@ -8,7 +8,7 @@ 枚举顺序严格按 `ToolName`（`ToolDefinitions.swift:4-36`）：`get_timeline, get_media, add_clips, insert_clips, remove_clips, remove_tracks, move_clips, set_clip_properties, set_keyframes, split_clip, ripple_delete_ranges, undo, add_texts, add_captions, generate_video, generate_image, generate_audio, upscale_media, import_media, list_models, inspect_media, get_transcript, inspect_timeline, search_media, list_folders, create_folder, move_to_folder, rename_media, rename_folder, delete_media, delete_folder`。 -> 这是上游 31 工具契约清单，不等于当前发现面。OpenTake 目前在 `ToolName::KNOWN` 保留 44 个兼容线名，仅将 38 个真实路径工具放入 `ToolName::ALL`；`generate_video`、`generate_image`、`generate_audio`、`upscale_media`、`add_motion_graphic`、`edit_motion_graphic` 在生产后端完成前不会出现在 MCP/Chat 目录或系统提示中。 +> 这是上游 31 工具契约清单，不等于当前发现面。OpenTake 在 `ToolName::KNOWN` 保留 44 个兼容线名：基础 38 个真实路径工具始终发布；存在托管或 fal/Replicate/OpenAI/ElevenLabs 兼容 BYOK 凭据时动态增加 `generate_video`、`generate_image`、`generate_audio`、`upscale_media`。Motion 两个兼容线名仍不发布。 ### A. 读 / 内省（只读，7 个） @@ -43,10 +43,12 @@ | # | 工具 | 关键参数 | required | 背后命令 | |---|---|---|---|---| -| 20 | `generate_video` | `prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交，立即返回 placeholder asset ID；**花钱、不可撤销** | -| 21 | `generate_image` | `prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`referenceMediaRefs?[]`,`folderId?` | `prompt` | `opentake-gen`：异步提交 placeholder | -| 22 | `generate_audio` | `prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | （无） | `opentake-gen`：TTS / 文生乐 / 视频配乐；时间线区间结果自动落轨 | -| 23 | `upscale_media` | `mediaRef*`,`model?`,`sourceClipId?` | `mediaRef` | `opentake-gen`：升分辨率 placeholder | +| 20 | `generate_video` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`duration?`,`aspectRatio?`,`resolution?`,`startFrameMediaRef?`,`endFrameMediaRef?`,`sourceVideoMediaRef?`,`sourceClipId?`,`referenceImageMediaRefs?[]`,`referenceVideoMediaRefs?[]`,`referenceAudioMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：先持久化占位再异步提交；进度/取消/重试/恢复；**花钱、不可撤销** | +| 21 | `generate_image` | `costAuthorized*:bool`,`prompt*`,`name?`,`model?`,`aspectRatio?`,`resolution?`,`quality?`,`numImages?:1..4`,`referenceMediaRefs?[]`,`folderId?` | `costAuthorized,prompt` | `opentake-gen`：N 个有序占位，逐项下载/失败终局化 | +| 22 | `generate_audio` | `costAuthorized*:bool`,`prompt?`,`name?`,`model?`,`voice?`,`lyrics?`,`styleInstructions?`,`instrumental?`,`duration?`,`videoSourceStartFrame?`,`videoSourceEndFrame?`,`videoSourceMediaRef?`,`folderId?` | `costAuthorized` | `opentake-gen`：TTS / 文生乐 / 能力允许时的视频配乐；时间线区间先渲染再上传 | +| 23 | `upscale_media` | `costAuthorized*:bool`,`mediaRef*`,`model?`,`sourceClipId?` | `costAuthorized,mediaRef` | `opentake-gen`：严格 2x 新资产，源资产不变 | + +生成请求由 MCP/Chat 共用的 Tauri GenerationBridge 执行。接受请求前必须有显式成本授权；占位与 generation log 先原子持久化，后台再上传/提交/轮询。固定状态为 queued/generating/downloading/finalizing/ready/failed/cancelled；重启时有 provider job id 则恢复轮询，无 id 则以固定错误码要求显式重试，避免重复扣费。结果 URL 不落盘，下载后探测真实容器再导入。 | 24 | `import_media` | `source*`{三选一 `url?`(HTTPS≤1GB)/`path?`(本地，可目录递归)/`bytes?`(base64≤~15MB)，`mimeType?`},`name?`,`folderId?` | `source` | `opentake-core`/`opentake-project`：url 后台下载、path/bytes 同步；扩展名白名单 | ### D. 媒体库组织（写，7 个）——均可撤销，均支持「单条参数 或 `entries[]` 批量」二选一 diff --git "a/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" "b/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" index bffaa0d3..e5e515b3 100644 --- "a/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" +++ "b/docs/upstream-analysis/04-MCP\344\270\216Agent\345\267\245\345\205\267.md" @@ -112,6 +112,8 @@ require('mcp-remote/dist/proxy.js'); | **upscale_media** | 升分辨率(视频/图) | `mediaRef`,`model?`,`sourceClipId?`(只 upscale trim 段) | `EditSubmitter.submitUpscale(...)`。`+Generate.swift:313` | | **import_media** | 导入外部媒体(其它 MCP / 本地文件)。**异构来源的桥** | `source`{三选一:`url`(HTTPS,≤1GB,后台下载)/`path`(本地,可目录递归)/`bytes`(base64,≤~15MB),`mimeType?`},`name?`,`folderId?` | url 走带 `ImportDownloadDelegate`(超限取消)的后台下载→`editor.importMediaAsset`;path/bytes 同步;目录走 `editor.importFinderItems`。校验 https/无凭据/有 host/扩展名白名单。`+Import.swift:11` | +> **OpenTake 复刻状态(2026-07-29)**：四个生成工具已接 provider-neutral Tauri GenerationBridge，并额外要求 `costAuthorized:true`。fal/Replicate/OpenAI/ElevenLabs BYOK 与托管模式共用耐久占位、日志、进度、取消、重试、重启恢复和安全下载终局化；配置 provider 的确定性生产路径测试已覆盖 image/video/audio/upscale，N 图按下标配对，超分结果必须严格 2x 且不修改源资产。当前内置音频目录没有 `inputs:["video"]` 模型，因此视频配乐输入会按模型能力明确拒绝，时间线区间渲染上传路径保留给未来兼容模型。 + ### D. 媒体库组织(写,7 个) 全部在 `+Folders.swift`,均可撤销,均支持"单条参数 或 `entries[]` 批量"二选一: diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 4a3418b4..f3472553 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -56,7 +56,8 @@ sha2 = "0.10" uuid = { workspace = true } # Optional account scaffold: verify a token only against a user-configured # backend. Rustls keeps the desktop build independent of a system OpenSSL. -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +futures-util = "0.3" # Tokio is also required by the always-built MCP server (URL import cancellation, # timers and the loopback runtime), so it cannot be optional in the minimal diff --git a/src-tauri/src/account.rs b/src-tauri/src/account.rs index 5073b4c5..bea429ea 100644 --- a/src-tauri/src/account.rs +++ b/src-tauri/src/account.rs @@ -100,6 +100,15 @@ struct BoundCredential { backend_url: String, } +/// Stored, backend-bound credential for managed generation. The tuple remains +/// inside the Rust process and is never returned by a Tauri command. +pub(crate) fn generation_credential() -> Result, String> { + let store = keyring_store(); + let backend = load_backend_url(&store)?; + Ok(load_bound_credential(&store, backend.as_deref())? + .map(|credential| (credential.backend_url, credential.token))) +} + fn advance_generation(runtime: &mut AccountRuntime) -> u64 { runtime.generation = runtime.generation.wrapping_add(1); runtime.generation diff --git a/src-tauri/src/chat.rs b/src-tauri/src/chat.rs index e3be6fb0..d33efdf3 100644 --- a/src-tauri/src/chat.rs +++ b/src-tauri/src/chat.rs @@ -23,6 +23,7 @@ use opentake_agent::chat::{ }; use opentake_agent::mcp::core_handle::{AppCoreHandle, CoreHandle}; use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::mcp::generation::GenerationBridge; use opentake_agent::tools::result::ToolResult; use opentake_gen::{KeyStore, KeyringStore}; @@ -89,21 +90,49 @@ impl ChatProjectContext { } impl ChatState { + #[cfg(test)] + pub fn new( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + ) -> Self { + Self::new_inner(core, workflows_dir, cache_root, models_dir, None) + } + /// Build the state in `setup`: a dispatcher over the live core + workflow /// registry + the same media bridge the desktop MCP server uses. - pub fn new( + pub fn new_with_generation( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + generation_bridge: Arc, + ) -> Self { + Self::new_inner( + core, + workflows_dir, + cache_root, + models_dir, + Some(generation_bridge), + ) + } + + fn new_inner( core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_dir: PathBuf, + generation_bridge: Option>, ) -> Self { let handle: Arc = Arc::new(AppCoreHandle::new(core.clone())); let registry = Arc::new(RwLock::new(crate::mcp::build_registry(&workflows_dir))); let bridge = crate::mcp::build_media_bridge(core.clone(), cache_root, models_dir); - let dispatcher = Arc::new(Dispatcher::with_bridge( + let dispatcher = Arc::new(Dispatcher::with_bridges( handle, registry.clone(), Some(bridge), + generation_bridge, )); let store: Arc = Arc::new(KeyringStore::new()); let sessions = Arc::new(Mutex::new(HashMap::new())); diff --git a/src-tauri/src/export.rs b/src-tauri/src/export.rs index 48bb9918..a3892167 100644 --- a/src-tauri/src/export.rs +++ b/src-tauri/src/export.rs @@ -995,6 +995,7 @@ pub fn run_export( #[derive(Default)] pub(crate) struct ExportRunOptions<'a> { pub(crate) control: Option<&'a ExportControl>, + pub(crate) external_cancel: Option, pub(crate) on_progress: Option, pub(crate) frame_range: Option<(i32, i32)>, pub(crate) output_file: Option, @@ -1009,6 +1010,7 @@ pub(crate) fn run_export_with_control( mut options: ExportRunOptions<'_>, ) -> Result { let control = options.control; + let external_cancel = options.external_cancel.clone(); let on_progress = options.on_progress; let defer_completion = options.defer_completion; let reserved_output = options.output_file.is_some(); @@ -1068,7 +1070,11 @@ pub(crate) fn run_export_with_control( let mut last_progress_emit = Instant::now(); for f in start_frame..end_frame { - if control.is_some_and(|c| c.is_cancelled()) { + if control.is_some_and(|c| c.is_cancelled()) + || external_cancel + .as_ref() + .is_some_and(MediaCancelToken::is_cancelled) + { // `abort` kills + waits on the ffmpeg child (unlike a plain `drop`, // which would orphan the process and race the file removal below). encoder.abort(); @@ -1927,6 +1933,7 @@ fn save_range_as_media_workflow( &req, ExportRunOptions { control: Some(control), + external_cancel: None, on_progress: Some(Arc::clone(&on_progress)), frame_range: Some((in_frame, out_frame)), output_file: Some(output_file), diff --git a/src-tauri/src/generation.rs b/src-tauri/src/generation.rs new file mode 100644 index 00000000..58e54521 --- /dev/null +++ b/src-tauri/src/generation.rs @@ -0,0 +1,2922 @@ +//! Production asynchronous generation bridge shared by MCP and in-app Chat. +//! +//! Durable placeholder state is committed before provider submission. Provider +//! keys stay in the OS keychain; signed result URLs and provider diagnostics are +//! never persisted. Terminal downloads are probed and imported through retained +//! project-media capabilities before the original placeholder identity becomes +//! ready. + +use std::collections::{BTreeSet, HashMap}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use base64::Engine as _; +use futures_util::StreamExt; +use opentake_agent::mcp::generation::{ + finalize_terminal_outputs, DownloadedGenerationArtifact, GenerationArtifactDownloader, + GenerationBridge, GenerationFinalizationStore, GenerationRequest, GenerationSubmission, +}; +use opentake_agent::tools::args::{ + GenerateAudioArgs, GenerateImageArgs, GenerateVideoArgs, UpscaleMediaArgs, +}; +use opentake_core::{ + AppCore, GenerationStateUpdate, PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, +}; +use opentake_domain::{ClipType, GenerationInput, GenerationJobStatus, MediaResolver, Timeline}; +use opentake_gen::catalog::cost::cost_for_input; +use opentake_gen::{ + build_params, Catalog, CatalogEntry, ElevenLabsAdapter, FalAdapter, GenClient, GenError, + JobStatus, KeyStore, KeyringStore, ModelKind, ModelRoute, OpenAiAdapter, ProviderKey, + ProviderRegistry, ReplicateAdapter, ReqwestTransport, StaticToken, UiCapabilities, +}; +use opentake_media::{MediaCancelToken, MediaEngine}; + +use crate::library::ProjectMediaCapability; + +const RESULT_BYTES_MAX: u64 = 1024 * 1024 * 1024; +const DATA_URL_ENCODED_MAX: usize = 512 * 1024 * 1024; +const RESULT_REDIRECT_MAX: usize = 5; + +#[derive(Default)] +struct GenerationRuntime { + jobs: Mutex>, + terminal_leases: Mutex>, + completed: Mutex>, +} + +#[derive(Clone)] +pub(crate) struct TauriGenerationBridge { + core: AppCore, + engine: Arc, + staging_root: PathBuf, + runtime: Arc, + clients: Arc, +} + +trait GenerationClientFactory: Send + Sync { + fn configured_byok_prefixes(&self) -> BTreeSet; + fn has_managed_credential(&self) -> bool; + fn build(&self, provider: &str, managed: bool) -> Result; +} + +struct ProductionGenerationClientFactory; + +impl GenerationClientFactory for ProductionGenerationClientFactory { + fn configured_byok_prefixes(&self) -> BTreeSet { + let store = KeyringStore::new(); + [ + ProviderKey::Fal, + ProviderKey::Replicate, + ProviderKey::OpenAI, + ProviderKey::ElevenLabs, + ] + .into_iter() + .filter_map(|key| { + (&store as &dyn KeyStore) + .load_key(key) + .ok() + .flatten() + .map(|_| key.prefix().to_string()) + }) + .collect() + } + + fn has_managed_credential(&self) -> bool { + crate::account::generation_credential() + .ok() + .flatten() + .is_some() + } + + fn build(&self, provider: &str, managed: bool) -> Result { + build_client(provider, managed) + } +} + +struct PreparedDispatch { + plan: PreparedGenerationJob, + references: Vec, + timeline_span: Option, + requires_source_video: bool, + model_kind: ModelKind, + managed: bool, +} + +struct PreparedTimelineSpan { + timeline: Timeline, + manifest: opentake_domain::MediaManifest, + project_dir: Option, + start_frame: i32, + end_frame: i32, +} + +#[derive(Clone)] +struct PreparedReference { + path: PathBuf, + fallback: &'static str, + trim_range: Option<(f64, f64)>, +} + +struct StagedCleanup { + path: PathBuf, + armed: bool, +} + +impl StagedCleanup { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn preserve(mut self) { + self.armed = false; + } +} + +impl Drop for StagedCleanup { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_file(&self.path); + } + } +} + +impl PreparedReference { + fn whole(path: PathBuf, fallback: &'static str) -> Self { + Self { + path, + fallback, + trim_range: None, + } + } +} + +pub(crate) fn build_bridge( + core: AppCore, + cache_root: PathBuf, + models_dir: PathBuf, +) -> Arc { + Arc::new(TauriGenerationBridge { + core, + engine: Arc::new(MediaEngine::new(cache_root.clone(), models_dir)), + staging_root: cache_root.join("generation-staging"), + runtime: Arc::new(GenerationRuntime::default()), + clients: Arc::new(ProductionGenerationClientFactory), + }) +} + +#[cfg(test)] +fn build_bridge_with_clients( + core: AppCore, + cache_root: PathBuf, + models_dir: PathBuf, + clients: Arc, +) -> Arc { + Arc::new(TauriGenerationBridge { + core, + engine: Arc::new(MediaEngine::new(cache_root.clone(), models_dir)), + staging_root: cache_root.join("generation-staging"), + runtime: Arc::new(GenerationRuntime::default()), + clients, + }) +} + +impl TauriGenerationBridge { + pub(crate) fn cancel(&self, job_id: &str) -> bool { + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(job_id) + .is_some_and(|token| { + token.cancel(); + true + }) + } + + pub(crate) fn retry( + &self, + job_id: &str, + cost_authorized: bool, + ) -> Result { + if !cost_authorized { + return Err("cost authorization is required before retry".to_string()); + } + let snapshot = self.core.runtime_snapshot(); + let outputs = snapshot + .media + .entries + .iter() + .filter(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.job_id.as_deref()) + == Some(job_id) + }) + .collect::>(); + let first = outputs + .first() + .ok_or_else(|| "generation job does not exist".to_string())?; + if outputs.iter().any(|entry| { + !matches!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Failed | GenerationJobStatus::Cancelled) + ) + }) { + return Err("only a failed or cancelled generation can be retried".to_string()); + } + let input = first + .generation_input + .as_ref() + .ok_or_else(|| "generation provenance is missing".to_string())?; + let catalog = Catalog::builtin(); + let model = catalog + .entries() + .iter() + .find(|entry| entry.id == input.model) + .ok_or_else(|| "generation model is no longer available".to_string())?; + let request = match model.kind { + ModelKind::Video => { + let frames = input.image_url_asset_ids.clone().unwrap_or_default(); + GenerationRequest::Video(GenerateVideoArgs { + cost_authorized: Some(true), + prompt: input.prompt.clone(), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + duration: Some(input.duration), + aspect_ratio: Some(input.aspect_ratio.clone()), + resolution: input.resolution.clone(), + start_frame_media_ref: frames.first().cloned(), + end_frame_media_ref: frames.get(1).cloned(), + source_video_media_ref: input.source_asset_id.clone(), + source_clip_id: input.source_clip_id.clone(), + reference_image_media_refs: input.reference_image_asset_ids.clone(), + reference_video_media_refs: input.reference_video_asset_ids.clone(), + reference_audio_media_refs: input.reference_audio_asset_ids.clone(), + folder_id: first.folder_id.clone(), + }) + } + ModelKind::Image => GenerationRequest::Image(GenerateImageArgs { + cost_authorized: Some(true), + prompt: input.prompt.clone(), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + aspect_ratio: Some(input.aspect_ratio.clone()), + resolution: input.resolution.clone(), + quality: input.quality.clone(), + num_images: Some(outputs.len() as i32), + reference_media_refs: input.reference_image_asset_ids.clone(), + folder_id: first.folder_id.clone(), + }), + ModelKind::Audio => GenerationRequest::Audio(GenerateAudioArgs { + cost_authorized: Some(true), + prompt: Some(input.prompt.clone()), + name: Some(first.name.clone()), + model: Some(input.model.clone()), + voice: input.voice.clone(), + lyrics: input.lyrics.clone(), + style_instructions: input.style_instructions.clone(), + instrumental: input.instrumental, + duration: Some(input.duration), + video_source_start_frame: input.source_start_frame, + video_source_end_frame: input.source_end_frame, + video_source_media_ref: input.source_asset_id.clone(), + folder_id: first.folder_id.clone(), + }), + ModelKind::Upscale => GenerationRequest::Upscale(UpscaleMediaArgs { + cost_authorized: Some(true), + media_ref: input + .source_asset_id + .clone() + .ok_or_else(|| "upscale source provenance is missing".to_string())?, + model: Some(input.model.clone()), + source_clip_id: input.source_clip_id.clone(), + }), + }; + self.submit(request, &MediaCancelToken::new()) + } + + /// Resume provider polling for durable non-terminal jobs after a project is + /// opened. A queued record without a provider id is deliberately failed and + /// exposed for explicit retry: resubmitting it automatically could create a + /// second paid job if the process died between provider acceptance and the + /// durable id write. + pub(crate) fn recover_current_project(&self) -> usize { + #[derive(Default)] + struct RecoveryJob { + provider: String, + provider_job_id: Option, + placeholders: Vec<(usize, String)>, + has_active_output: bool, + } + + let snapshot = self.core.runtime_snapshot(); + let Some(project_dir) = snapshot.project_dir.clone() else { + return 0; + }; + let mut recoverable = HashMap::::new(); + for entry in &snapshot.media.entries { + let Some(input) = entry.generation_input.as_ref() else { + continue; + }; + let Some(job_id) = input.job_id.as_ref() else { + continue; + }; + let job = recoverable.entry(job_id.clone()).or_default(); + if job.provider.is_empty() { + job.provider = input.provider.clone().unwrap_or_default(); + } + if job.provider_job_id.is_none() { + job.provider_job_id = input.provider_job_id.clone(); + } + job.placeholders + .push((input.output_index.unwrap_or(usize::MAX), entry.id.clone())); + if matches!( + input.status, + Some( + GenerationJobStatus::Queued + | GenerationJobStatus::Generating + | GenerationJobStatus::Downloading + | GenerationJobStatus::Finalizing + ) + ) { + job.has_active_output = true; + } + } + + let mut resumed = 0; + for (job_id, mut job) in recoverable { + if !job.has_active_output { + continue; + } + let already_running = self + .runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains_key(&job_id); + if already_running { + continue; + } + job.placeholders.sort_by_key(|(index, _)| *index); + let placeholder_ids = job + .placeholders + .into_iter() + .map(|(_, asset_id)| asset_id) + .collect::>(); + let Some(provider_job_id) = job.provider_job_id else { + let _ = self.core.update_generation_job_for_project( + snapshot.project_epoch, + &project_dir, + &job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Failed, + progress: None, + error_code: Some("GENERATION_RESTART_RETRY_REQUIRED".to_string()), + provider_job_id: None, + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ); + continue; + }; + if job.provider.is_empty() { + self.fail_nonterminal_outputs( + snapshot.project_epoch, + &project_dir, + &placeholder_ids, + "GENERATION_RECOVERY_STATE_INVALID", + ); + continue; + } + let cancel = MediaCancelToken::new(); + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.clone(), cancel.clone()); + let bridge = self.clone(); + let recovery_dir = project_dir.clone(); + let managed = !provider_job_id.starts_with(&format!("{}::", job.provider)); + tauri::async_runtime::spawn(async move { + bridge + .run_recovered_job( + snapshot.project_epoch, + recovery_dir, + job_id, + placeholder_ids, + job.provider, + managed, + provider_job_id, + cancel, + ) + .await; + }); + resumed += 1; + } + resumed + } + + fn fail_nonterminal_outputs( + &self, + project_epoch: u64, + project_dir: &Path, + placeholder_ids: &[String], + code: &str, + ) { + let snapshot = self.core.runtime_snapshot(); + for asset_id in placeholder_ids { + let terminal = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == *asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status) + .is_some_and(|status| { + matches!( + status, + GenerationJobStatus::Ready + | GenerationJobStatus::Failed + | GenerationJobStatus::Cancelled + ) + }); + if !terminal { + let _ = self.core.fail_generation_output_for_project( + project_epoch, + project_dir, + asset_id, + code, + Some(now_apple_reference_seconds()), + ); + } + } + } + + fn cancel_nonterminal_outputs( + &self, + project_epoch: u64, + project_dir: &Path, + placeholder_ids: &[String], + ) { + for asset_id in placeholder_ids { + let _ = self.core.cancel_generation_output_for_project( + project_epoch, + project_dir, + asset_id, + Some(now_apple_reference_seconds()), + ); + } + } + + fn configured_byok_prefixes(&self) -> BTreeSet { + self.clients.configured_byok_prefixes() + } + + fn has_managed_credential(&self) -> bool { + self.clients.has_managed_credential() + } + + fn prepare(&self, request: GenerationRequest) -> Result { + let snapshot = self.core.runtime_snapshot(); + snapshot + .project_dir + .as_deref() + .ok_or_else(|| "Save the project before starting generation".to_string())?; + let configured = self.configured_byok_prefixes(); + let managed_available = self.has_managed_credential(); + let catalog = Catalog::builtin(); + + match request { + GenerationRequest::Video(args) => { + let entry = select_model( + &catalog, + ModelKind::Video, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Video(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid video capabilities".to_string()); + }; + let duration = args + .duration + .map(|value| value.max(0) as u32) + .or_else(|| caps.durations.first().copied()) + .unwrap_or(0); + if !caps.durations.is_empty() && !caps.durations.contains(&duration) { + return Err("duration is not supported by the selected model".to_string()); + } + let aspect_ratio = args + .aspect_ratio + .clone() + .or_else(|| caps.aspect_ratios.first().cloned()) + .unwrap_or_default(); + if !caps.aspect_ratios.is_empty() && !caps.aspect_ratios.contains(&aspect_ratio) { + return Err("aspectRatio is not supported by the selected model".to_string()); + } + validate_choice( + "resolution", + args.resolution.as_deref(), + caps.resolutions.as_deref(), + )?; + + let frames = [ + args.start_frame_media_ref.clone(), + args.end_frame_media_ref.clone(), + ] + .into_iter() + .flatten() + .collect::>(); + if args.start_frame_media_ref.is_some() && !caps.supports_first_frame { + return Err("selected model does not support a first frame".to_string()); + } + if args.end_frame_media_ref.is_some() && !caps.supports_last_frame { + return Err("selected model does not support a last frame".to_string()); + } + let image_refs = args.reference_image_media_refs.clone().unwrap_or_default(); + let video_refs = args.reference_video_media_refs.clone().unwrap_or_default(); + let audio_refs = args.reference_audio_media_refs.clone().unwrap_or_default(); + validate_reference_count("image", image_refs.len(), caps.max_reference_images)?; + validate_reference_count("video", video_refs.len(), caps.max_reference_videos)?; + validate_reference_count("audio", audio_refs.len(), caps.max_reference_audios)?; + if let Some(max) = caps.max_total_references { + if image_refs.len() + video_refs.len() + audio_refs.len() > max as usize { + return Err( + "too many combined references for the selected model".to_string() + ); + } + } + if caps.frames_and_references_exclusive + && !frames.is_empty() + && (!image_refs.is_empty() || !video_refs.is_empty() || !audio_refs.is_empty()) + { + return Err( + "frames and references are mutually exclusive for this model".to_string(), + ); + } + if caps.requires_source_video && args.source_video_media_ref.is_none() { + return Err("selected model requires sourceVideoMediaRef".to_string()); + } + if caps.requires_reference_image && image_refs.is_empty() { + return Err("selected model requires an image reference".to_string()); + } + let source_trim = validate_source_clip( + &snapshot.timeline, + args.source_clip_id.as_deref(), + args.source_video_media_ref.as_deref(), + )?; + + let mut references = Vec::new(); + if caps.requires_source_video { + if let Some(source) = args.source_video_media_ref.as_deref() { + references.push(PreparedReference { + path: resolve_media(&snapshot, source, ClipType::Video)?, + fallback: "video", + trim_range: source_trim, + }); + } + } else { + for media_ref in &frames { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Image)?, + "image", + )); + } + } + for media_ref in &image_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Image)?, + "image", + )); + } + for media_ref in &video_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Video)?, + "video", + )); + } + for media_ref in &audio_refs { + references.push(PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Audio)?, + "audio", + )); + } + + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: args.prompt.clone(), + model: entry.id.clone(), + duration: duration as i32, + aspect_ratio, + resolution: args.resolution.clone(), + image_url_asset_ids: (!frames.is_empty()).then_some(frames), + reference_image_asset_ids: (!image_refs.is_empty()).then_some(image_refs), + reference_video_asset_ids: (!video_refs.is_empty()).then_some(video_refs), + reference_audio_asset_ids: (!audio_refs.is_empty()).then_some(audio_refs), + generate_audio: Some(true), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &args.prompt, "Generated video"), + kind: ClipType::Video, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: args.source_video_media_ref, + source_clip_id: args.source_clip_id, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span: None, + requires_source_video: caps.requires_source_video, + model_kind: ModelKind::Video, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Image(args) => { + let entry = select_model( + &catalog, + ModelKind::Image, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Image(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid image capabilities".to_string()); + }; + validate_choice( + "aspectRatio", + args.aspect_ratio.as_deref(), + Some(&caps.aspect_ratios), + )?; + validate_choice( + "resolution", + args.resolution.as_deref(), + caps.resolutions.as_deref(), + )?; + validate_choice( + "quality", + args.quality.as_deref(), + caps.qualities.as_deref(), + )?; + let refs = args.reference_media_refs.clone().unwrap_or_default(); + if !refs.is_empty() && !caps.supports_image_reference { + return Err("selected model does not support image references".to_string()); + } + if refs.len() > caps.max_images as usize { + return Err("too many image references for the selected model".to_string()); + } + let references = refs + .iter() + .map(|media_ref| { + resolve_media(&snapshot, media_ref, ClipType::Image) + .map(|path| PreparedReference::whole(path, "image")) + }) + .collect::, _>>()?; + let output_count = args.num_images.unwrap_or(1).clamp(1, 4) as usize; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: args.prompt.clone(), + model: entry.id.clone(), + duration: 0, + aspect_ratio: args + .aspect_ratio + .clone() + .or_else(|| caps.aspect_ratios.first().cloned()) + .unwrap_or_default(), + resolution: args.resolution.clone(), + quality: args.quality.clone(), + num_images: Some(output_count as i32), + reference_image_asset_ids: (!refs.is_empty()).then_some(refs), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &args.prompt, "Generated image"), + kind: ClipType::Image, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count, + source_asset_id: None, + source_clip_id: None, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span: None, + requires_source_video: false, + model_kind: ModelKind::Image, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Audio(args) => { + let entry = select_model( + &catalog, + ModelKind::Audio, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Audio(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid audio capabilities".to_string()); + }; + let prompt = args.prompt.clone().unwrap_or_default(); + if prompt.chars().count() < caps.min_prompt_length as usize { + return Err("prompt is too short for the selected audio model".to_string()); + } + if args.lyrics.is_some() && !caps.supports_lyrics { + return Err("selected model does not support lyrics".to_string()); + } + if args.instrumental == Some(true) && !caps.supports_instrumental { + return Err("selected model does not support instrumental mode".to_string()); + } + if args.style_instructions.is_some() && !caps.supports_style_instructions { + return Err("selected model does not support style instructions".to_string()); + } + let timeline_span = match ( + args.video_source_start_frame, + args.video_source_end_frame, + ) { + (None, None) => None, + (Some(start), Some(end)) => { + if args.video_source_media_ref.is_some() { + return Err( + "timeline span and videoSourceMediaRef are mutually exclusive" + .to_string(), + ); + } + let total = snapshot.timeline.total_frames(); + if start < 0 || end <= start || end > total { + return Err( + "video source frame range is outside the timeline".to_string() + ); + } + Some(PreparedTimelineSpan { + timeline: snapshot.timeline.clone(), + manifest: snapshot.media.clone(), + project_dir: snapshot.project_dir.clone(), + start_frame: start, + end_frame: end, + }) + } + _ => return Err( + "videoSourceStartFrame and videoSourceEndFrame must be provided together" + .to_string(), + ), + }; + if (timeline_span.is_some() || args.video_source_media_ref.is_some()) + && !caps + .inputs + .as_ref() + .is_some_and(|inputs| inputs.iter().any(|input| input == "video")) + { + return Err("selected audio model does not support a video source".to_string()); + } + let references = match args.video_source_media_ref.as_deref() { + Some(media_ref) => vec![PreparedReference::whole( + resolve_media(&snapshot, media_ref, ClipType::Video)?, + "video", + )], + None => Vec::new(), + }; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: prompt.clone(), + model: entry.id.clone(), + duration: args.duration.unwrap_or_else(|| { + timeline_span + .as_ref() + .map(|span| { + ((span.end_frame - span.start_frame) as f64 + / span.timeline.fps.max(1) as f64) + .ceil() as i32 + }) + .unwrap_or(0) + }), + aspect_ratio: String::new(), + voice: args.voice, + lyrics: args.lyrics, + style_instructions: args.style_instructions, + instrumental: args.instrumental, + reference_video_asset_ids: args + .video_source_media_ref + .clone() + .map(|id| vec![id]), + source_start_frame: args.video_source_start_frame, + source_end_frame: args.video_source_end_frame, + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: display_name(args.name.as_deref(), &prompt, "Generated audio"), + kind: ClipType::Audio, + folder_id: args.folder_id, + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: args.video_source_media_ref, + source_clip_id: None, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references, + timeline_span, + requires_source_video: false, + model_kind: ModelKind::Audio, + managed: !configured.contains(&provider) && managed_available, + }) + } + GenerationRequest::Upscale(args) => { + let source = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == args.media_ref) + .ok_or_else(|| "upscale source asset does not exist".to_string())?; + if !matches!(source.kind, ClipType::Image | ClipType::Video) { + return Err("upscale source must be image or video".to_string()); + } + let source_trim = validate_source_clip( + &snapshot.timeline, + args.source_clip_id.as_deref(), + Some(&args.media_ref), + )?; + let entry = select_model( + &catalog, + ModelKind::Upscale, + args.model.as_deref(), + &configured, + managed_available, + )?; + let UiCapabilities::Upscale(caps) = &entry.ui_capabilities else { + return Err("selected model has invalid upscale capabilities".to_string()); + }; + let source_kind = if source.kind == ClipType::Image { + "image" + } else { + "video" + }; + if !caps.supported_types.iter().any(|kind| kind == source_kind) { + return Err("selected upscaler does not support the source type".to_string()); + } + let source_path = resolve_media(&snapshot, &args.media_ref, source.kind)?; + let provider = provider_prefix(&entry.id)?; + let input = GenerationInput { + prompt: String::new(), + model: entry.id.clone(), + duration: source.duration.max(0.0).round() as i32, + aspect_ratio: String::new(), + source_asset_id: Some(args.media_ref.clone()), + source_clip_id: args.source_clip_id.clone(), + ..Default::default() + }; + let estimated_cost_credits = cost_for_input(entry, &input); + Ok(PreparedDispatch { + plan: PreparedGenerationJob { + name: format!("{} 2x", source.name), + kind: source.kind, + folder_id: source.folder_id.clone(), + provider: provider.clone(), + input, + output_count: 1, + source_asset_id: Some(args.media_ref), + source_clip_id: args.source_clip_id, + estimated_cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + references: vec![PreparedReference { + path: source_path, + fallback: source_kind, + trim_range: source_trim, + }], + timeline_span: None, + requires_source_video: false, + model_kind: ModelKind::Upscale, + managed: !configured.contains(&provider) && managed_available, + }) + } + } + } + + async fn run_job( + self, + project_epoch: u64, + project_dir: PathBuf, + local_job_id: String, + placeholder_ids: Vec, + prepared: PreparedDispatch, + cancel: MediaCancelToken, + ) { + let result = self + .run_job_inner( + project_epoch, + &project_dir, + &local_job_id, + &placeholder_ids, + &prepared, + &cancel, + ) + .await; + if let Err(code) = result { + if cancel.is_cancelled() { + self.cancel_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids); + } else { + self.fail_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids, &code); + } + } + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&local_job_id); + } + + async fn run_job_inner( + &self, + project_epoch: u64, + project_dir: &Path, + local_job_id: &str, + placeholder_ids: &[String], + prepared: &PreparedDispatch, + cancel: &MediaCancelToken, + ) -> Result<(), String> { + cancelled(cancel)?; + let client = self + .clients + .build(&prepared.plan.provider, prepared.managed)?; + let mut references = prepared.references.clone(); + let timeline_cleanup = if let Some(span) = prepared.timeline_span.as_ref() { + std::fs::create_dir_all(&self.staging_root) + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())?; + let destination = self.staging_root.join(format!( + "{local_job_id}-{}.timeline.mp4", + uuid::Uuid::new_v4() + )); + let timeline = span.timeline.clone(); + let manifest = span.manifest.clone(); + let project_dir = span.project_dir.clone(); + let start_frame = span.start_frame; + let end_frame = span.end_frame; + let output = destination.clone(); + let export_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + crate::export::run_export_with_control( + &timeline, + &manifest, + &project_dir, + &crate::export::ExportRequest { + out_path: output.to_string_lossy().into_owned(), + codec: crate::export::ExportCodec::default(), + quality: crate::export::ExportQuality::default(), + }, + crate::export::ExportRunOptions { + external_cancel: Some(export_cancel), + frame_range: Some((start_frame, end_frame)), + ..crate::export::ExportRunOptions::default() + }, + ) + }) + .await + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())? + .map_err(|error| { + if error == crate::export::CANCELLED_SENTINEL { + "GENERATION_CANCELLED".to_string() + } else { + "GENERATION_SOURCE_PREPROCESS_FAILED".to_string() + } + })?; + references.push(PreparedReference::whole(destination.clone(), "video")); + Some(StagedCleanup::new(destination)) + } else { + None + }; + let mut uploaded = Vec::with_capacity(references.len()); + for reference in &references { + cancelled(cancel)?; + let staged_trim = if let Some((start, end)) = reference.trim_range { + let destination = self + .staging_root + .join(format!("{local_job_id}-{}.trim.mp4", uuid::Uuid::new_v4())); + let source = reference.path.clone(); + let output = destination.clone(); + let trim_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + opentake_media::trim_video_range(&source, &output, start, end, &trim_cancel) + }) + .await + .map_err(|_| "GENERATION_SOURCE_PREPROCESS_FAILED".to_string())? + .map_err(|error| { + if matches!(error, opentake_media::MediaError::Cancelled) { + "GENERATION_CANCELLED".to_string() + } else { + "GENERATION_SOURCE_PREPROCESS_FAILED".to_string() + } + })?; + Some(destination) + } else { + None + }; + let upload_path = staged_trim.as_deref().unwrap_or(&reference.path); + let content_type = opentake_gen::content_type_for(upload_path, reference.fallback); + let uploaded_url = if prepared.managed { + client.upload_reference(upload_path, &content_type).await + } else { + client + .upload_reference_via(&prepared.plan.provider, upload_path, &content_type) + .await + }; + if let Some(path) = staged_trim { + let _ = std::fs::remove_file(path); + } + let uploaded_url = uploaded_url.map_err(|error| { + generation_provider_error_code(&error, "GENERATION_REFERENCE_UPLOAD_FAILED") + })?; + uploaded.push(uploaded_url); + } + drop(timeline_cleanup); + cancelled(cancel)?; + let params = build_params( + &prepared.plan.input, + &uploaded, + prepared.model_kind, + prepared.requires_source_video, + ); + let provider_job_id = if prepared.managed { + client + .submit(&prepared.plan.input.model, params, Some(local_job_id)) + .await + } else { + client.submit_byok(&prepared.plan.input.model, params).await + } + .map_err(|error| generation_provider_error_code(&error, "GENERATION_SUBMIT_FAILED"))?; + self.core + .update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.15), + error_code: None, + provider_job_id: Some(provider_job_id.clone()), + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ) + .map_err(|_| "GENERATION_STATE_PERSIST_FAILED".to_string())?; + + self.watch_and_finalize( + project_epoch, + project_dir, + local_job_id, + placeholder_ids, + client, + &provider_job_id, + cancel, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn watch_and_finalize( + &self, + project_epoch: u64, + project_dir: &Path, + local_job_id: &str, + placeholder_ids: &[String], + client: GenClient, + provider_job_id: &str, + cancel: &MediaCancelToken, + ) -> Result<(), String> { + let stream = client.watch(provider_job_id); + futures_util::pin_mut!(stream); + loop { + cancelled(cancel)?; + let next = tokio::select! { + item = stream.next() => item, + () = wait_for_cancel(cancel) => return Err("GENERATION_CANCELLED".to_string()), + }; + let job = next + .ok_or_else(|| "GENERATION_PROVIDER_STREAM_ENDED".to_string())? + .map_err(|error| { + generation_provider_error_code(&error, "GENERATION_PROVIDER_POLL_FAILED") + })?; + match job.status { + JobStatus::Queued => {} + JobStatus::Running => { + let _ = self.core.update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.5), + error_code: None, + provider_job_id: Some(provider_job_id.to_string()), + cost_credits: None, + created_at: Some(now_apple_reference_seconds()), + }, + ); + } + JobStatus::Failed => return Err("GENERATION_PROVIDER_FAILED".to_string()), + JobStatus::Succeeded => { + self.core + .update_generation_job_for_project( + project_epoch, + project_dir, + local_job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Downloading, + progress: Some(0.8), + error_code: None, + provider_job_id: Some(provider_job_id.to_string()), + cost_credits: job.cost_credits, + created_at: Some(now_apple_reference_seconds()), + }, + ) + .map_err(|_| "GENERATION_STATE_PERSIST_FAILED".to_string())?; + let store = TauriFinalizationStore { + bridge: self.clone(), + project_epoch, + project_dir: project_dir.to_path_buf(), + }; + let staging_root = self.staging_root.clone(); + let urls = job.result_urls.unwrap_or_default(); + let terminal_job_id = local_job_id.to_string(); + let terminal_placeholder_ids = placeholder_ids.to_vec(); + let download_cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + let downloader = + SecureResultDownloader::new(staging_root, download_cancel)?; + finalize_terminal_outputs( + &store, + &downloader, + &terminal_job_id, + &terminal_placeholder_ids, + &urls, + ) + }) + .await + .map_err(|_| "GENERATION_FINALIZE_TASK_FAILED".to_string())? + .map_err(|error| { + if error == "GENERATION_CANCELLED" { + error + } else { + "GENERATION_FINALIZE_FAILED".to_string() + } + })?; + return Ok(()); + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn run_recovered_job( + self, + project_epoch: u64, + project_dir: PathBuf, + local_job_id: String, + placeholder_ids: Vec, + provider: String, + managed: bool, + provider_job_id: String, + cancel: MediaCancelToken, + ) { + let result = match self.clients.build(&provider, managed) { + Ok(client) => { + self.watch_and_finalize( + project_epoch, + &project_dir, + &local_job_id, + &placeholder_ids, + client, + &provider_job_id, + &cancel, + ) + .await + } + Err(_) => Err("GENERATION_RECOVERY_AUTH_UNAVAILABLE".to_string()), + }; + if let Err(code) = result { + if cancel.is_cancelled() { + self.cancel_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids); + } else { + self.fail_nonterminal_outputs(project_epoch, &project_dir, &placeholder_ids, &code); + } + } + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&local_job_id); + } +} + +impl GenerationBridge for TauriGenerationBridge { + fn can_generate(&self) -> bool { + !self.configured_byok_prefixes().is_empty() || self.has_managed_credential() + } + + fn submit( + &self, + request: GenerationRequest, + cancel: &MediaCancelToken, + ) -> Result { + cancelled(cancel)?; + let prepared = self.prepare(request)?; + let snapshot = self.core.runtime_snapshot(); + let project_dir = snapshot + .project_dir + .clone() + .ok_or_else(|| "Save the project before starting generation".to_string())?; + let committed = self + .core + .begin_generation_job_for_project( + snapshot.project_epoch, + &project_dir, + prepared.plan.clone(), + ) + .map_err(|error| error.to_string())?; + let background_cancel = MediaCancelToken::new(); + self.runtime + .jobs + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(committed.job_id.clone(), background_cancel.clone()); + let bridge = self.clone(); + let job_id = committed.job_id.clone(); + let placeholder_ids = committed.placeholder_asset_ids.clone(); + tauri::async_runtime::spawn(async move { + bridge + .run_job( + snapshot.project_epoch, + project_dir, + job_id, + placeholder_ids, + prepared, + background_cancel, + ) + .await; + }); + Ok(GenerationSubmission { + job_id: committed.job_id, + placeholder_asset_ids: committed.placeholder_asset_ids, + status: "queued".to_string(), + }) + } +} + +#[derive(Clone)] +struct TauriFinalizationStore { + bridge: TauriGenerationBridge, + project_epoch: u64, + project_dir: PathBuf, +} + +impl GenerationFinalizationStore for TauriFinalizationStore { + fn claim_terminal(&self, job_id: &str) -> Result { + if self + .bridge + .runtime + .completed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains(job_id) + { + return Ok(false); + } + Ok(self + .bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.to_string())) + } + + fn release_terminal(&self, job_id: &str) -> Result<(), String> { + self.bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(job_id); + Ok(()) + } + + fn finalize_output( + &self, + asset_id: &str, + artifact: DownloadedGenerationArtifact, + ) -> Result<(), String> { + let snapshot = self.bridge.core.runtime_snapshot(); + let entry = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| "generation placeholder disappeared".to_string())?; + let probe = self + .bridge + .engine + .probe(&artifact.path) + .map_err(|_| "downloaded generation result could not be probed".to_string())?; + let actual_kind = if probe.has_video { + if probe.duration_secs > 0.0 { + ClipType::Video + } else { + ClipType::Image + } + } else if probe.has_audio { + ClipType::Audio + } else { + return Err("downloaded generation result has no supported stream".to_string()); + }; + if actual_kind != entry.kind { + return Err("downloaded generation result has the wrong media type".to_string()); + } + if let Some(source_id) = entry + .generation_input + .as_ref() + .and_then(|input| input.source_asset_id.as_deref()) + { + let source = snapshot + .media + .entries + .iter() + .find(|source| source.id == source_id) + .ok_or_else(|| "upscale source disappeared".to_string())?; + if let (Some(source_width), Some(source_height), Some(width), Some(height)) = ( + source.source_width, + source.source_height, + probe.width, + probe.height, + ) { + if width as i32 != source_width.saturating_mul(2) + || height as i32 != source_height.saturating_mul(2) + { + return Err( + "upscale result is not exactly 2x the source dimensions".to_string() + ); + } + } + } + let extension = result_extension( + &artifact.media_type, + actual_kind, + probe.format_name.as_deref(), + )?; + let leaf = format!("{asset_id}.{extension}"); + let capability = ProjectMediaCapability::open_verified( + &self.bridge.core, + self.project_epoch, + &self.project_dir, + true, + )?; + let mut import = capability.create_import(Path::new(&leaf))?; + let mut source = std::fs::File::open(&artifact.path).map_err(|error| error.to_string())?; + std::io::copy(&mut source, import.file_mut()).map_err(|error| error.to_string())?; + import + .file() + .sync_all() + .map_err(|error| error.to_string())?; + if !capability.matches_leaf(&import)? { + return Err("generation import identity changed".to_string()); + } + self.bridge + .core + .finalize_generation_output_for_project( + self.project_epoch, + &self.project_dir, + PreparedGenerationOutput { + asset_id: asset_id.to_string(), + relative_path: format!("media/{leaf}"), + probe: ProbedMedia { + duration_secs: probe.duration_secs, + width: probe.width.map(|value| value as i32), + height: probe.height.map(|value| value as i32), + fps: probe.fps, + has_audio: probe.has_audio, + }, + created_at: Some(now_apple_reference_seconds()), + }, + ) + .map_err(|error| error.to_string())?; + import.commit(); + let _ = std::fs::remove_file(&artifact.path); + Ok(()) + } + + fn fail_output(&self, asset_id: &str, code: &str) -> Result<(), String> { + self.bridge + .core + .fail_generation_output_for_project( + self.project_epoch, + &self.project_dir, + asset_id, + code, + Some(now_apple_reference_seconds()), + ) + .map_err(|error| error.to_string()) + } + + fn complete_job(&self, job_id: &str, _succeeded: usize, _failed: usize) -> Result<(), String> { + self.bridge + .runtime + .terminal_leases + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(job_id); + self.bridge + .runtime + .completed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(job_id.to_string()); + Ok(()) + } +} + +struct SecureResultDownloader { + client: reqwest::blocking::Client, + staging_root: PathBuf, + cancel: MediaCancelToken, +} + +impl SecureResultDownloader { + fn new(staging_root: PathBuf, cancel: MediaCancelToken) -> Result { + std::fs::create_dir_all(&staging_root) + .map_err(|_| "generation staging directory is unavailable".to_string())?; + let client = reqwest::blocking::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(10 * 60)) + .build() + .map_err(|_| "generation result client initialization failed".to_string())?; + Ok(Self { + client, + staging_root, + cancel, + }) + } + + fn write_staging( + &self, + asset_id: &str, + media_type: String, + bytes: &[u8], + ) -> Result { + if bytes.len() as u64 > RESULT_BYTES_MAX { + return Err("generation result exceeds the download limit".to_string()); + } + let path = self + .staging_root + .join(format!("{asset_id}-{}.download", uuid::Uuid::new_v4())); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + let mut file = options + .open(&path) + .map_err(|_| "generation staging file creation failed".to_string())?; + let cleanup = StagedCleanup::new(path.clone()); + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|_| "generation staging write failed".to_string())?; + cleanup.preserve(); + Ok(DownloadedGenerationArtifact { + path, + media_type, + byte_size: bytes.len() as u64, + }) + } +} + +impl GenerationArtifactDownloader for SecureResultDownloader { + fn download( + &self, + asset_id: &str, + raw_url: &str, + ) -> Result { + cancelled(&self.cancel)?; + if raw_url.starts_with("data:") { + if raw_url.len() > DATA_URL_ENCODED_MAX { + return Err("generation data URL exceeds the download limit".to_string()); + } + let (header, encoded) = raw_url + .split_once(',') + .ok_or_else(|| "generation data URL is malformed".to_string())?; + let media_type = header + .strip_prefix("data:") + .and_then(|value| value.strip_suffix(";base64")) + .filter(|value| { + value.starts_with("image/") + || value.starts_with("audio/") + || value.starts_with("video/") + }) + .ok_or_else(|| "generation data URL media type is unsupported".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| "generation data URL base64 is invalid".to_string())?; + return self.write_staging(asset_id, media_type.to_string(), &bytes); + } + + let mut current = validate_result_https_url(raw_url)?; + for redirect_count in 0..=RESULT_REDIRECT_MAX { + cancelled(&self.cancel)?; + let mut response = self + .client + .get(current.clone()) + .send() + .map_err(|_| "generation result download failed".to_string())?; + if response + .remote_addr() + .is_some_and(|address| !is_public_result_ip(address.ip())) + { + return Err("generation result resolved to a private address".to_string()); + } + if response.status().is_redirection() { + if redirect_count == RESULT_REDIRECT_MAX { + return Err("generation result exceeded redirect limit".to_string()); + } + let location = response + .headers() + .get(reqwest::header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| "generation result redirect is invalid".to_string())?; + current = validate_result_https_url( + current + .join(location) + .map_err(|_| "generation result redirect is invalid".to_string())? + .as_str(), + )?; + continue; + } + if !response.status().is_success() { + return Err("generation result download returned an error".to_string()); + } + if response + .content_length() + .is_some_and(|length| length > RESULT_BYTES_MAX) + { + return Err("generation result exceeds the download limit".to_string()); + } + let media_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .unwrap_or("application/octet-stream") + .trim() + .to_ascii_lowercase(); + let path = self + .staging_root + .join(format!("{asset_id}-{}.download", uuid::Uuid::new_v4())); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|_| "generation staging file creation failed".to_string())?; + let cleanup = StagedCleanup::new(path.clone()); + let mut total = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + loop { + cancelled(&self.cancel)?; + let count = response + .read(&mut buffer) + .map_err(|_| "generation result stream failed".to_string())?; + if count == 0 { + break; + } + total = total.saturating_add(count as u64); + if total > RESULT_BYTES_MAX { + return Err("generation result exceeds the download limit".to_string()); + } + file.write_all(&buffer[..count]) + .map_err(|_| "generation staging write failed".to_string())?; + } + file.sync_all() + .map_err(|_| "generation staging write failed".to_string())?; + cleanup.preserve(); + return Ok(DownloadedGenerationArtifact { + path, + media_type, + byte_size: total, + }); + } + Err("generation result download failed".to_string()) + } +} + +fn build_client(provider: &str, managed: bool) -> Result { + if managed { + let (backend, token) = crate::account::generation_credential()? + .ok_or_else(|| "managed generation credential is unavailable".to_string())?; + let base = reqwest::Url::parse(&(backend + "/")) + .map_err(|_| "managed generation backend is invalid".to_string())?; + return Ok(GenClient::managed(base, Arc::new(StaticToken(token)))); + } + let store = KeyringStore::new(); + let key = match provider { + "fal" => ProviderKey::Fal, + "replicate" => ProviderKey::Replicate, + "openai" => ProviderKey::OpenAI, + "elevenlabs" => ProviderKey::ElevenLabs, + _ => return Err("generation provider is unsupported".to_string()), + }; + let secret = (&store as &dyn KeyStore) + .load_key(key) + .map_err(|_| "generation provider key could not be loaded".to_string())? + .ok_or_else(|| "generation provider key is not configured".to_string())?; + let transport = Arc::new(ReqwestTransport::new()); + let registry = match provider { + "fal" => ProviderRegistry::new().with(Arc::new(FalAdapter::new(transport, secret))), + "replicate" => { + ProviderRegistry::new().with(Arc::new(ReplicateAdapter::new(transport, secret))) + } + "openai" => ProviderRegistry::new().with(Arc::new(OpenAiAdapter::new(transport, secret))), + "elevenlabs" => { + ProviderRegistry::new().with(Arc::new(ElevenLabsAdapter::new(transport, secret))) + } + _ => unreachable!(), + }; + Ok(GenClient::byok(registry, Catalog::builtin())) +} + +fn generation_provider_error_code(error: &GenError, fallback: &str) -> String { + match error { + GenError::Unauthenticated | GenError::NotConfigured => "GENERATION_AUTH_FAILED", + GenError::InsufficientCredits(_) => "GENERATION_INSUFFICIENT_CREDITS", + GenError::Api { status: 429, .. } => "GENERATION_RATE_LIMITED", + _ => fallback, + } + .to_string() +} + +fn select_model<'a>( + catalog: &'a Catalog, + kind: ModelKind, + requested: Option<&str>, + configured: &BTreeSet, + managed: bool, +) -> Result<&'a CatalogEntry, String> { + if let Some(requested) = requested { + let entry = catalog + .by_id(requested) + .ok_or_else(|| "generation model does not exist".to_string())?; + if entry.kind != kind { + return Err("generation model has the wrong media kind".to_string()); + } + let provider = provider_prefix(&entry.id)?; + if !managed && !configured.contains(&provider) { + return Err("selected model provider is not configured".to_string()); + } + return Ok(entry); + } + catalog + .entries() + .iter() + .find(|entry| { + entry.kind == kind + && (managed + || provider_prefix(&entry.id) + .ok() + .is_some_and(|provider| configured.contains(&provider))) + }) + .ok_or_else(|| "no configured provider supports this generation type".to_string()) +} + +fn resolve_media( + snapshot: &opentake_core::ProjectRuntimeSnapshot, + media_ref: &str, + expected_kind: ClipType, +) -> Result { + let entry = snapshot + .media + .entries + .iter() + .find(|entry| entry.id == media_ref) + .ok_or_else(|| format!("referenced media does not exist: {media_ref}"))?; + if entry.kind != expected_kind { + return Err(format!("referenced media has the wrong type: {media_ref}")); + } + let path = MediaResolver::new(&snapshot.media, snapshot.project_dir.as_deref()) + .expected_path(media_ref) + .ok_or_else(|| format!("referenced media cannot be resolved: {media_ref}"))?; + if !path.is_file() { + return Err(format!("referenced media is offline: {media_ref}")); + } + Ok(path) +} + +fn validate_source_clip( + timeline: &Timeline, + clip_id: Option<&str>, + media_ref: Option<&str>, +) -> Result, String> { + let Some(clip_id) = clip_id else { + return Ok(None); + }; + let clip = timeline + .tracks + .iter() + .flat_map(|track| &track.clips) + .find(|clip| clip.id == clip_id) + .ok_or_else(|| "sourceClipId does not exist".to_string())?; + if media_ref.is_some_and(|media_ref| clip.media_ref != media_ref) { + return Err("sourceClipId does not reference the requested media".to_string()); + } + if timeline.fps <= 0 || clip.duration_frames <= 0 { + return Err("sourceClipId has no valid visible source range".to_string()); + } + let start = clip.trim_start_frame.max(0) as f64 / timeline.fps as f64; + let consumed = clip.source_frames_consumed(); + if consumed <= 0 { + return Err("sourceClipId has no valid visible source range".to_string()); + } + Ok(Some((start, start + consumed as f64 / timeline.fps as f64))) +} + +fn validate_choice( + field: &str, + value: Option<&str>, + allowed: Option<&[String]>, +) -> Result<(), String> { + if let (Some(value), Some(allowed)) = (value, allowed) { + if !allowed.is_empty() && !allowed.iter().any(|candidate| candidate == value) { + return Err(format!("{field} is not supported by the selected model")); + } + } + Ok(()) +} + +fn validate_reference_count(label: &str, count: usize, max: u32) -> Result<(), String> { + if count > max as usize { + Err(format!( + "too many {label} references for the selected model" + )) + } else { + Ok(()) + } +} + +fn provider_prefix(model: &str) -> Result { + ModelRoute::parse(model) + .map(|route| route.prefix) + .map_err(|_| "generation model id is invalid".to_string()) +} + +fn display_name(requested: Option<&str>, prompt: &str, fallback: &str) -> String { + requested + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| { + let value = prompt.trim().chars().take(30).collect::(); + (!value.is_empty()).then_some(value) + }) + .unwrap_or_else(|| fallback.to_string()) +} + +fn result_extension( + media_type: &str, + kind: ClipType, + format_name: Option<&str>, +) -> Result<&'static str, String> { + if let Some(format) = format_name { + let formats = format.split(',').collect::>(); + let detected = match kind { + ClipType::Image if formats.contains("png_pipe") => Some("png"), + ClipType::Image if formats.contains("jpeg_pipe") || formats.contains("image2") => { + Some("jpg") + } + ClipType::Image if formats.contains("webp_pipe") => Some("webp"), + ClipType::Video if formats.contains("mov") || formats.contains("mp4") => Some("mp4"), + ClipType::Audio if formats.contains("mp3") => Some("mp3"), + ClipType::Audio if formats.contains("wav") => Some("wav"), + ClipType::Audio if formats.contains("mov") || formats.contains("mp4") => Some("m4a"), + _ => None, + }; + if let Some(extension) = detected { + return Ok(extension); + } + } + match media_type { + "image/png" => Ok("png"), + "image/jpeg" | "image/jpg" => Ok("jpg"), + "image/webp" => Ok("webp"), + "video/mp4" => Ok("mp4"), + "video/quicktime" => Ok("mov"), + "audio/mpeg" => Ok("mp3"), + "audio/wav" | "audio/x-wav" => Ok("wav"), + "audio/mp4" => Ok("m4a"), + "application/octet-stream" => match kind { + ClipType::Image => Ok("png"), + ClipType::Video => Ok("mp4"), + ClipType::Audio => Ok("mp3"), + ClipType::Text | ClipType::Lottie => { + Err("generated media type is unsupported".to_string()) + } + }, + _ => Err("generation result content type is unsupported".to_string()), + } +} + +fn validate_result_https_url(raw: &str) -> Result { + let url = + reqwest::Url::parse(raw).map_err(|_| "generation result URL is invalid".to_string())?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.port().is_some_and(|port| port != 443) + { + return Err("generation result URL is not an accepted HTTPS URL".to_string()); + } + let host = url.host_str().unwrap_or_default().to_ascii_lowercase(); + if host == "localhost" + || host.ends_with(".localhost") + || host.ends_with(".local") + || host.ends_with(".internal") + || host + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok_and(|address| !is_public_result_ip(address)) + { + return Err("generation result URL host is not public".to_string()); + } + Ok(url) +} + +fn is_public_result_ip(address: std::net::IpAddr) -> bool { + match address { + std::net::IpAddr::V4(ip) => { + let [a, b, _, _] = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.is_multicast() + || a == 0 + || (a == 100 && (64..=127).contains(&b)) + || (a == 198 && (18..=19).contains(&b))) + } + std::net::IpAddr::V6(ip) => { + if let Some(mapped) = ip.to_ipv4_mapped() { + return is_public_result_ip(std::net::IpAddr::V4(mapped)); + } + let octets = ip.octets(); + !(ip.is_loopback() + || ip.is_unspecified() + || ip.is_unique_local() + || ip.is_unicast_link_local() + || ip.is_multicast() + || octets[..4] == [0x20, 0x01, 0x0d, 0xb8] + || (octets[0] == 0xfe && octets[1] & 0xc0 == 0xc0)) + } + } +} + +fn cancelled(cancel: &MediaCancelToken) -> Result<(), String> { + if cancel.is_cancelled() { + Err("GENERATION_CANCELLED".to_string()) + } else { + Ok(()) + } +} + +async fn wait_for_cancel(cancel: &MediaCancelToken) { + while !cancel.is_cancelled() { + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn now_apple_reference_seconds() -> f64 { + const APPLE_REFERENCE_UNIX_OFFSET: f64 = 978_307_200.0; + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs_f64() - APPLE_REFERENCE_UNIX_OFFSET) + .unwrap_or(0.0) +} + +#[tauri::command] +pub fn generation_cancel( + bridge: tauri::State<'_, Arc>, + job_id: String, +) -> Result { + Ok(bridge.cancel(&job_id)) +} + +#[tauri::command] +pub fn generation_retry( + bridge: tauri::State<'_, Arc>, + job_id: String, + cost_authorized: bool, +) -> Result { + bridge.retry(&job_id, cost_authorized) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + use image::{DynamicImage, ImageFormat}; + use opentake_agent::mcp::core_handle::AppCoreHandle; + use opentake_agent::mcp::dispatch::Dispatcher; + use opentake_agent::plugin::registry::PluginRegistry; + use opentake_gen::{AuthMode, FalAdapter, HttpResponse, Method, MockTransport}; + use opentake_project::{GenerationLog, Project}; + use serde_json::json; + use std::sync::RwLock; + + #[derive(Clone)] + struct FixtureClients { + client: GenClient, + } + + impl GenerationClientFactory for FixtureClients { + fn configured_byok_prefixes(&self) -> BTreeSet { + BTreeSet::from([ + "fal".to_string(), + "openai".to_string(), + "replicate".to_string(), + ]) + } + + fn has_managed_credential(&self) -> bool { + false + } + + fn build(&self, _provider: &str, _managed: bool) -> Result { + Ok(self.client.clone()) + } + } + + fn fixture_client_with_interval(mock: &MockTransport, interval: Duration) -> GenClient { + let transport = Arc::new(mock.clone()); + let fal = FalAdapter::new(transport.clone(), "fixture-secret").with_base("https://mockfal"); + let openai = + OpenAiAdapter::new(transport.clone(), "fixture-secret").with_base("https://mockoai/v1"); + let replicate = + ReplicateAdapter::new(transport, "fixture-secret").with_base("https://mockrep/v1"); + GenClient::with_transport( + AuthMode::Byok { + registry: ProviderRegistry::new() + .with(Arc::new(fal)) + .with(Arc::new(openai)) + .with(Arc::new(replicate)), + catalog: Catalog::builtin(), + }, + Arc::new(mock.clone()), + ) + .with_poll_interval(interval) + } + + fn fixture_client(mock: &MockTransport) -> GenClient { + fixture_client_with_interval(mock, Duration::ZERO) + } + + fn saved_core() -> (tempfile::TempDir, PathBuf, AppCore) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Generation.opentake"); + let mut project = Project::new(&bundle); + project.generation_log = Some(GenerationLog::new()); + project.save().unwrap(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + (temp, bundle, core) + } + + fn image_plan() -> PreparedGenerationJob { + PreparedGenerationJob { + name: "Recovered image".to_string(), + kind: ClipType::Image, + folder_id: None, + provider: "fal".to_string(), + input: GenerationInput { + prompt: "fixture".to_string(), + model: "fal:flux-pro".to_string(), + duration: 0, + aspect_ratio: "1:1".to_string(), + num_images: Some(1), + ..Default::default() + }, + output_count: 1, + source_asset_id: None, + source_clip_id: None, + estimated_cost_credits: None, + created_at: Some(800_000_000.0), + } + } + + fn runtime_dirs(bundle: &Path) -> (PathBuf, PathBuf) { + let root = bundle.parent().unwrap(); + (root.join("cache"), root.join("models")) + } + + fn png_bytes(width: u32, height: u32) -> Vec { + let mut bytes = Cursor::new(Vec::new()); + DynamicImage::new_rgba8(width, height) + .write_to(&mut bytes, ImageFormat::Png) + .unwrap(); + bytes.into_inner() + } + + fn png_data_url_for(width: u32, height: u32) -> String { + format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png_bytes(width, height)) + ) + } + + fn png_data_url() -> String { + png_data_url_for(2, 2) + } + + fn wav_bytes() -> Vec { + let sample_rate = 8_000_u32; + let sample_count = sample_rate / 10; + let data_size = sample_count * 2; + let mut bytes = Vec::with_capacity((44 + data_size) as usize); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_size).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&sample_rate.to_le_bytes()); + bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + bytes.extend_from_slice(&2_u16.to_le_bytes()); + bytes.extend_from_slice(&16_u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_size.to_le_bytes()); + bytes.resize((44 + data_size) as usize, 0); + bytes + } + + fn mp4_data_url(directory: &Path) -> String { + let path = directory.join("generation-fixture.mp4"); + let status = std::process::Command::new("ffmpeg") + .args([ + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "color=c=black:s=16x16:d=0.1:r=10", + "-pix_fmt", + "yuv420p", + ]) + .arg(&path) + .status() + .unwrap(); + assert!(status.success()); + let encoded = + base64::engine::general_purpose::STANDARD.encode(std::fs::read(path).unwrap()); + format!("data:video/mp4;base64,{encoded}") + } + + fn saved_core_with_source_image() -> (tempfile::TempDir, PathBuf, AppCore) { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Upscale.opentake"); + let mut project = Project::new(&bundle); + project.generation_log = Some(GenerationLog::new()); + project + .manifest + .entries + .push(opentake_domain::MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: opentake_domain::MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(2), + source_height: Some(2), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + std::fs::create_dir_all(bundle.join("media")).unwrap(); + std::fs::write(bundle.join("media/source.png"), png_bytes(2, 2)).unwrap(); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + (temp, bundle, core) + } + + async fn wait_for_ready_model( + core: &AppCore, + model: &str, + ) -> opentake_domain::MediaManifestEntry { + for _ in 0..150 { + if let Some(entry) = core.media().entries.into_iter().find(|entry| { + entry.generation_input.as_ref().is_some_and(|input| { + input.model == model && input.status == Some(GenerationJobStatus::Ready) + }) + }) { + return entry; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!( + "generation did not become ready for {model}: {:?}", + core.media().entries + ); + } + + #[test] + fn result_url_validation_rejects_local_and_private_network_targets() { + assert!(validate_result_https_url("https://cdn.example.com/result.png").is_ok()); + for url in [ + "https://localhost/result.png", + "https://service.local/result.png", + "https://127.0.0.1/result.png", + "https://10.0.0.1/result.png", + "https://169.254.1.1/result.png", + "https://[::1]/result.png", + "https://[fc00::1]/result.png", + "https://[::ffff:127.0.0.1]/result.png", + "https://[2001:db8::1]/result.png", + ] { + assert!( + validate_result_https_url(url).is_err(), + "private result URL accepted: {url}" + ); + } + } + + #[test] + fn restart_before_provider_id_requires_explicit_retry_without_resubmission() { + let (_temp, bundle, core) = saved_core(); + let committed = core + .begin_generation_job_for_project(1, &bundle, image_plan()) + .unwrap(); + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + reopened.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + + assert_eq!(bridge.recover_current_project(), 0); + let persisted = reopened.media(); + let input = persisted + .entries + .iter() + .find(|entry| entry.id == committed.placeholder_asset_ids[0]) + .unwrap() + .generation_input + .as_ref() + .unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Failed)); + assert_eq!( + input.error_code.as_deref(), + Some("GENERATION_RESTART_RETRY_REQUIRED") + ); + assert!(mock.calls().is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn restart_with_provider_id_resumes_and_finalizes_offline_fixture() { + let (_temp, bundle, core) = saved_core(); + let runtime = core.runtime_snapshot(); + let committed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, image_plan()) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.25), + error_code: None, + provider_job_id: Some("fal::flux-pro|recover-1".to_string()), + cost_credits: None, + created_at: Some(800_000_001.0), + }, + ) + .unwrap(); + + let reopened = AppCore::new(); + let reopened_snapshot = reopened.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/recover-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/recover-1", + 200, + json!({"images": [{"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + reopened.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + + assert_eq!(bridge.recover_current_project(), 1); + let asset_id = committed.placeholder_asset_ids[0].clone(); + for _ in 0..100 { + let status = reopened + .media() + .entries + .iter() + .find(|entry| entry.id == asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status); + if status == Some(GenerationJobStatus::Ready) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let entry = reopened + .media() + .entries + .into_iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Ready) + ); + assert_eq!(entry.source_width, Some(2)); + assert_eq!(entry.source_height, Some(2)); + assert!(MediaResolver::new(&reopened.media(), Some(&bundle)) + .expected_path(&entry.id) + .unwrap() + .is_file()); + assert_eq!( + reopened.project_revision().project_epoch, + reopened_snapshot.project_epoch + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_path_persists_and_finalizes_ordered_mock_results() { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "dispatch-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/dispatch-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/dispatch-1", + 200, + json!({"images": [{"url": png_data_url()}, {"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ); + + let unauthorized = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": false, + "prompt": "ordered fixture", + "model": "fal:flux-pro", + "numImages": 2 + }), + ); + assert!(unauthorized.is_error); + assert!(mock.calls().is_empty()); + + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "ordered fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1", + "numImages": 2 + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + for _ in 0..100 { + let generated = core + .media() + .entries + .into_iter() + .filter(|entry| entry.generation_input.is_some()) + .collect::>(); + if generated.len() == 2 + && generated.iter().all(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.status) + == Some(GenerationJobStatus::Ready) + }) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let mut generated = core + .media() + .entries + .into_iter() + .filter(|entry| entry.generation_input.is_some()) + .collect::>(); + generated.sort_by_key(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.output_index) + }); + assert_eq!(generated.len(), 2); + assert!( + generated.iter().all(|entry| { + entry + .generation_input + .as_ref() + .and_then(|input| input.status) + == Some(GenerationJobStatus::Ready) + && entry.source_width == Some(2) + && entry.source_height == Some(2) + }), + "generated outputs: {generated:?}" + ); + assert_eq!( + generated + .iter() + .filter_map(|entry| entry + .generation_input + .as_ref() + .and_then(|input| input.output_index)) + .collect::>(), + vec![0, 1] + ); + assert!(generated.iter().all(|entry| { + MediaResolver::new(&core.media(), Some(&bundle)) + .expected_path(&entry.id) + .is_some_and(|path| path.is_file()) + })); + } + + #[tokio::test(flavor = "multi_thread")] + async fn configured_provider_smoke_covers_video_audio_and_upscale() { + let (video_temp, video_bundle, video_core) = saved_core(); + let video_mock = MockTransport::new(); + video_mock.on( + Method::Post, + "https://mockfal/kling-video", + 200, + json!({"request_id": "video-1", "status": "IN_QUEUE"}), + ); + video_mock.on( + Method::Get, + "https://mockfal/kling-video/requests/video-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + video_mock.on( + Method::Get, + "https://mockfal/kling-video/requests/video-1", + 200, + json!({"video": {"url": mp4_data_url(video_temp.path())}}), + ); + let (cache, models) = runtime_dirs(&video_bundle); + let video_bridge = build_bridge_with_clients( + video_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&video_mock), + }), + ); + let video_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(video_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(video_bridge), + ); + let video_result = video_dispatcher.dispatch( + "generate_video", + json!({ + "costAuthorized": true, + "prompt": "fixture video", + "model": "fal:kling-video", + "duration": 5, + "aspectRatio": "16:9", + "resolution": "720p" + }), + ); + assert!(!video_result.is_error, "{}", video_result.text_joined()); + let video = wait_for_ready_model(&video_core, "fal:kling-video").await; + assert_eq!(video.kind, ClipType::Video); + assert_eq!(video.source_width, Some(16)); + assert_eq!(video.source_height, Some(16)); + + let (_audio_temp, audio_bundle, audio_core) = saved_core(); + let audio_mock = MockTransport::new(); + let mut audio_response = HttpResponse::new(200, wav_bytes()); + audio_response + .headers + .push(("Content-Type".to_string(), "audio/wav".to_string())); + audio_mock.on_raw( + Method::Post, + "https://mockoai/v1/audio/speech", + audio_response, + ); + let (cache, models) = runtime_dirs(&audio_bundle); + let audio_bridge = build_bridge_with_clients( + audio_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&audio_mock), + }), + ); + let audio_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(audio_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(audio_bridge), + ); + let audio_result = audio_dispatcher.dispatch( + "generate_audio", + json!({ + "costAuthorized": true, + "prompt": "fixture speech", + "model": "openai:tts-1", + "voice": "alloy" + }), + ); + assert!(!audio_result.is_error, "{}", audio_result.text_joined()); + let audio = wait_for_ready_model(&audio_core, "openai:tts-1").await; + assert_eq!(audio.kind, ClipType::Audio); + assert_eq!(audio.has_audio, Some(true)); + assert!(audio.duration > 0.0); + + let (_upscale_temp, upscale_bundle, upscale_core) = saved_core_with_source_image(); + let source_before = std::fs::read(upscale_bundle.join("media/source.png")).unwrap(); + let upscale_mock = MockTransport::new(); + upscale_mock.on( + Method::Post, + "https://mockrep/v1/files", + 200, + json!({"urls": {"get": "https://fixtures.invalid/source.png"}}), + ); + upscale_mock.on( + Method::Post, + "https://mockrep/v1/predictions", + 200, + json!({"id": "upscale-1", "status": "starting"}), + ); + upscale_mock.on( + Method::Get, + "https://mockrep/v1/predictions/upscale-1", + 200, + json!({"id": "upscale-1", "status": "succeeded", "output": png_data_url_for(4, 4)}), + ); + let (cache, models) = runtime_dirs(&upscale_bundle); + let upscale_bridge = build_bridge_with_clients( + upscale_core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&upscale_mock), + }), + ); + let upscale_dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(upscale_core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(upscale_bridge), + ); + let upscale_result = upscale_dispatcher.dispatch( + "upscale_media", + json!({ + "costAuthorized": true, + "mediaRef": "source-image", + "model": "replicate:topaz-upscale" + }), + ); + assert!(!upscale_result.is_error, "{}", upscale_result.text_joined()); + let upscale = wait_for_ready_model(&upscale_core, "replicate:topaz-upscale").await; + assert_eq!(upscale.kind, ClipType::Image); + assert_eq!(upscale.source_width, Some(4)); + assert_eq!(upscale.source_height, Some(4)); + assert_eq!( + std::fs::read(upscale_bundle.join("media/source.png")).unwrap(), + source_before + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_cancel_terminalizes_without_importing_media() { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "cancel-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/cancel-1/status", + 200, + json!({"status": "IN_QUEUE"}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client_with_interval(&mock, Duration::from_secs(2)), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge.clone()), + ); + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "cancel fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1" + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + let (job_id, asset_id) = loop { + if let Some(entry) = core + .media() + .entries + .into_iter() + .find(|entry| entry.generation_input.is_some()) + { + let input = entry.generation_input.as_ref().unwrap(); + if input.provider_job_id.is_some() { + break (input.job_id.clone().unwrap(), entry.id); + } + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + assert!(bridge.cancel(&job_id)); + for _ in 0..100 { + let status = core + .media() + .entries + .iter() + .find(|entry| entry.id == asset_id) + .and_then(|entry| entry.generation_input.as_ref()) + .and_then(|input| input.status); + if status == Some(GenerationJobStatus::Cancelled) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let entry = core + .media() + .entries + .into_iter() + .find(|entry| entry.id == asset_id) + .unwrap(); + assert_eq!( + entry + .generation_input + .as_ref() + .and_then(|input| input.status), + Some(GenerationJobStatus::Cancelled) + ); + assert!(MediaResolver::new(&core.media(), Some(&bundle)) + .expected_path(&entry.id) + .is_some_and(|path| !path.exists())); + } + + #[test] + fn upscale_finalization_is_exactly_two_x_and_preserves_source_bytes() { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("Upscale.opentake"); + let source_bytes = png_bytes(3, 2); + let mut project = Project::new(&bundle); + project + .manifest + .entries + .push(opentake_domain::MediaManifestEntry { + id: "source-image".to_string(), + name: "source.png".to_string(), + kind: ClipType::Image, + source: opentake_domain::MediaSource::Project { + relative_path: "media/source.png".to_string(), + }, + duration: 0.0, + generation_input: None, + source_width: Some(3), + source_height: Some(2), + source_fps: None, + has_audio: Some(false), + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + project.save().unwrap(); + std::fs::create_dir_all(bundle.join("media")).unwrap(); + std::fs::write(bundle.join("media/source.png"), &source_bytes).unwrap(); + let core = AppCore::new(); + let snapshot = core.open_project(&bundle).unwrap(); + let mock = MockTransport::new(); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache.clone(), + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let mut plan = image_plan(); + plan.provider = "replicate".to_string(); + plan.input.model = "replicate:topaz-upscale".to_string(); + plan.source_asset_id = Some("source-image".to_string()); + let committed = core + .begin_generation_job_for_project(snapshot.project_epoch, &bundle, plan) + .unwrap(); + core.update_generation_job_for_project( + snapshot.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Generating, + progress: Some(0.5), + error_code: None, + provider_job_id: Some("replicate::fixture".to_string()), + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + core.update_generation_job_for_project( + snapshot.project_epoch, + &bundle, + &committed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Downloading, + progress: Some(0.8), + error_code: None, + provider_job_id: None, + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + std::fs::create_dir_all(&cache).unwrap(); + let staged = cache.join("upscale.png"); + std::fs::write(&staged, png_bytes(6, 4)).unwrap(); + let store = TauriFinalizationStore { + bridge: bridge.as_ref().clone(), + project_epoch: snapshot.project_epoch, + project_dir: bundle.clone(), + }; + store + .finalize_output( + &committed.placeholder_asset_ids[0], + DownloadedGenerationArtifact { + path: staged, + media_type: "image/png".to_string(), + byte_size: 0, + }, + ) + .unwrap(); + let media = core.media(); + let source = media + .entries + .iter() + .find(|entry| entry.id == "source-image") + .unwrap(); + let output = media + .entries + .iter() + .find(|entry| entry.id == committed.placeholder_asset_ids[0]) + .unwrap(); + assert_eq!( + (source.source_width, source.source_height), + (Some(3), Some(2)) + ); + assert_eq!( + (output.source_width, output.source_height), + (Some(6), Some(4)) + ); + assert_eq!( + std::fs::read(bundle.join("media/source.png")).unwrap(), + source_bytes + ); + } + + async fn assert_submit_failure(status: u16, body: serde_json::Value, expected_code: &str) { + let (_temp, bundle, core) = saved_core(); + let mock = MockTransport::new(); + mock.on(Method::Post, "https://mockfal/flux-pro", status, body); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + let dispatcher = Dispatcher::with_bridges( + Arc::new(AppCoreHandle::new(core.clone())), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + Some(bridge), + ); + let accepted = dispatcher.dispatch( + "generate_image", + json!({ + "costAuthorized": true, + "prompt": "failure fixture", + "model": "fal:flux-pro", + "aspectRatio": "1:1" + }), + ); + assert!(!accepted.is_error, "{}", accepted.text_joined()); + for _ in 0..100 { + let terminal = core + .media() + .entries + .iter() + .find_map(|entry| entry.generation_input.as_ref()) + .is_some_and(|input| input.status == Some(GenerationJobStatus::Failed)); + if terminal { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let input = core + .media() + .entries + .into_iter() + .find_map(|entry| entry.generation_input) + .unwrap(); + assert_eq!(input.status, Some(GenerationJobStatus::Failed)); + assert_eq!(input.error_code.as_deref(), Some(expected_code)); + } + + #[tokio::test(flavor = "multi_thread")] + async fn production_dispatch_maps_auth_and_rate_limit_to_safe_fixed_codes() { + assert_submit_failure( + 401, + json!({"error": {"code": "unauthenticated", "message": "private"}}), + "GENERATION_AUTH_FAILED", + ) + .await; + assert_submit_failure( + 429, + json!({"error": {"code": "rate_limited", "message": "private"}}), + "GENERATION_RATE_LIMITED", + ) + .await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn retry_requires_fresh_cost_authorization_and_creates_a_new_job() { + let (_temp, bundle, core) = saved_core(); + let runtime = core.runtime_snapshot(); + let failed = core + .begin_generation_job_for_project(runtime.project_epoch, &bundle, image_plan()) + .unwrap(); + core.update_generation_job_for_project( + runtime.project_epoch, + &bundle, + &failed.job_id, + GenerationStateUpdate { + status: GenerationJobStatus::Failed, + progress: None, + error_code: Some("GENERATION_PROVIDER_FAILED".to_string()), + provider_job_id: None, + cost_credits: None, + created_at: None, + }, + ) + .unwrap(); + let mock = MockTransport::new(); + mock.on( + Method::Post, + "https://mockfal/flux-pro", + 200, + json!({"request_id": "retry-1", "status": "IN_QUEUE"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/retry-1/status", + 200, + json!({"status": "COMPLETED"}), + ); + mock.on( + Method::Get, + "https://mockfal/flux-pro/requests/retry-1", + 200, + json!({"images": [{"url": png_data_url()}]}), + ); + let (cache, models) = runtime_dirs(&bundle); + let bridge = build_bridge_with_clients( + core.clone(), + cache, + models, + Arc::new(FixtureClients { + client: fixture_client(&mock), + }), + ); + assert!(bridge.retry(&failed.job_id, false).is_err()); + assert!(mock.calls().is_empty()); + let retried = bridge.retry(&failed.job_id, true).unwrap(); + assert_ne!(retried.job_id, failed.job_id); + for _ in 0..100 { + let ready = core.media().entries.iter().any(|entry| { + entry.generation_input.as_ref().is_some_and(|input| { + input.job_id.as_deref() == Some(retried.job_id.as_str()) + && input.status == Some(GenerationJobStatus::Ready) + }) + }); + if ready { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let statuses = core + .media() + .entries + .iter() + .filter_map(|entry| entry.generation_input.as_ref()) + .map(|input| (input.job_id.clone().unwrap(), input.status.unwrap())) + .collect::>(); + assert_eq!( + statuses.get(&failed.job_id), + Some(&GenerationJobStatus::Failed) + ); + assert_eq!( + statuses.get(&retried.job_id), + Some(&GenerationJobStatus::Ready) + ); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 66eb206f..6ba94615 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -14,6 +14,7 @@ mod commands; // drive the export orchestrator (`export::run_export`) against the library // target. The Tauri command itself is registered below like the other modules. pub mod export; +mod generation; mod haptic; mod library; mod mcp; @@ -116,17 +117,21 @@ pub fn run() { .app_data_dir() .unwrap_or_else(|_| std::env::temp_dir()) .join("workflows"); + let generation_bridge = + generation::build_bridge(core.clone(), cache_root.clone(), models_dir.clone()); mcp::spawn( core.clone(), workflows_dir.clone(), cache_root.clone(), models_dir.clone(), + generation_bridge.clone(), ); - let chat_state = chat::ChatState::new( + let chat_state = chat::ChatState::new_with_generation( core.clone(), workflows_dir, cache_root.clone(), models_dir.clone(), + generation_bridge.clone(), ); // A global favorite must never silently become a temporary file. @@ -142,6 +147,7 @@ pub fn run() { }; app.manage(core); + app.manage(generation_bridge); app.manage(chat_state); app.manage(MediaState::new(engine)); app.manage(PrewarmScheduler::new(initial_project_epoch)); @@ -207,6 +213,8 @@ pub fn run() { export::export_video, export::save_range_as_media, export::cancel_export, + generation::generation_cancel, + generation::generation_retry, secret::secret_save, secret::secret_load, secret::secret_delete, @@ -315,6 +323,11 @@ fn forward_event(app: &tauri::AppHandle, event: &CoreEvent) { if let Some(prewarm) = app.try_state::() { prewarm.activate_project(*project_epoch); } + if let Some(generation) = + app.try_state::>() + { + generation.recover_current_project(); + } } #[cfg(feature = "playback-engine")] { diff --git a/src-tauri/src/mcp.rs b/src-tauri/src/mcp.rs index 2b7c2009..95422a0a 100644 --- a/src-tauri/src/mcp.rs +++ b/src-tauri/src/mcp.rs @@ -30,6 +30,7 @@ use std::io::Read; use base64::Engine as _; use opentake_agent::mcp::core_handle::{AppCoreHandle, CoreHandle}; +use opentake_agent::mcp::generation::GenerationBridge; use opentake_agent::mcp::media_bridge::{ BridgeError, ImportOutcome, ImportSource, InspectMediaRequest, InspectMediaResult, InspectResult, InspectedFrame, InspectedMediaFrame, MediaBridge, SearchCandidate, @@ -252,7 +253,13 @@ pub(crate) fn build_registry(workflows_dir: &Path) -> PluginRegistry { /// the same paths the UI's [`MediaEngine`] uses, so the bridge's imports land in /// the same caches. A bind failure (port in use) is logged, not fatal — the app /// keeps running without the agent network face. -pub fn spawn(core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_dir: PathBuf) { +pub fn spawn( + core: AppCore, + workflows_dir: PathBuf, + cache_root: PathBuf, + models_dir: PathBuf, + generation_bridge: Arc, +) { let handle: Arc = Arc::new(AppCoreHandle::new(core.clone())); let bridge = build_media_bridge(core, cache_root, models_dir); let registry = Arc::new(RwLock::new(build_registry(&workflows_dir))); @@ -264,7 +271,15 @@ pub fn spawn(core: AppCore, workflows_dir: PathBuf, cache_root: PathBuf, models_ return; } }; - if let Err(e) = server::serve_with_bridge(addr, handle, registry, Some(bridge)).await { + if let Err(e) = server::serve_with_bridges( + addr, + handle, + registry, + Some(bridge), + Some(generation_bridge), + ) + .await + { eprintln!("[mcp] server stopped: {e}"); } }); diff --git a/src-tauri/src/media.rs b/src-tauri/src/media.rs index dd1eaf95..b47a39e5 100644 --- a/src-tauri/src/media.rs +++ b/src-tauri/src/media.rs @@ -35,7 +35,8 @@ use opentake_core::{ PreparedMediaFolderRef, PreparedMediaImportOp, ProbedMedia, }; use opentake_domain::{ - ClipType, GenerationInput, MediaManifest, MediaManifestEntry, MediaSource, Timeline, + ClipType, GenerationInput, GenerationJobStatus, MediaManifest, MediaManifestEntry, MediaSource, + Timeline, }; use opentake_media::library::{FavoriteRequest, LibraryStore, PreparedFavorite}; #[cfg(test)] @@ -116,6 +117,10 @@ pub struct MediaItemDto { /// always `None` in practice — the Inspector renders those sections only /// when it is present, matching upstream's `if let gen = asset.generationInput`. pub generation_input: Option, + /// Durable async lifecycle projected from `generation_input`. + pub generation_status: String, + pub generation_progress: Option, + pub generation_error_code: Option, /// `true` when the asset's source file is not on disk (moved / deleted / /// offline). Derived from file existence on every read (mirrors upstream /// `MediaResolver.isMissing`), so it clears automatically once a `relink_media` @@ -144,9 +149,31 @@ impl MediaItemDto { // produced by importing (always external) but handled for safety. MediaSource::Project { .. } => None, }; - // Missing = we can resolve a local source path and it doesn't exist. + let generation_input = entry.generation_input.as_ref(); + let generation_status = match generation_input.and_then(|input| input.status) { + Some(GenerationJobStatus::Queued | GenerationJobStatus::Generating) => "generating", + Some(GenerationJobStatus::Downloading | GenerationJobStatus::Finalizing) => { + "downloading" + } + Some(GenerationJobStatus::Failed) => "failed", + Some(GenerationJobStatus::Cancelled) => "cancelled", + Some(GenerationJobStatus::Ready) | None => "none", + } + .to_string(); + let generation_pending = matches!( + generation_input.and_then(|input| input.status), + Some( + GenerationJobStatus::Queued + | GenerationJobStatus::Generating + | GenerationJobStatus::Downloading + | GenerationJobStatus::Finalizing + ) + ); + // Missing = a finalized source can resolve and is absent. Pending + // placeholders intentionally have no file yet and are not "offline". // An unresolvable (e.g. remote-only) source is not flagged missing. - let missing = resolved.as_ref().map(|p| !p.exists()).unwrap_or(false); + let missing = + !generation_pending && resolved.as_ref().map(|p| !p.exists()).unwrap_or(false); let thumbnail = if missing { None } else { @@ -177,6 +204,9 @@ impl MediaItemDto { folder_id: entry.folder_id.clone(), file_size, generation_input: entry.generation_input.clone(), + generation_status, + generation_progress: generation_input.and_then(|input| input.progress), + generation_error_code: generation_input.and_then(|input| input.error_code.clone()), missing, favorite, } @@ -4102,6 +4132,9 @@ mod tests { folder_id: None, file_size: Some(2048), generation_input: None, + generation_status: "none".into(), + generation_progress: None, + generation_error_code: None, missing: false, favorite: true, }; diff --git a/src-tauri/src/secret.rs b/src-tauri/src/secret.rs index 061c763e..bdc52366 100644 --- a/src-tauri/src/secret.rs +++ b/src-tauri/src/secret.rs @@ -22,14 +22,15 @@ pub struct SecretStatus { masked: String, } -/// Allowed BYOK chat providers → stable keychain account strings, following the -/// `-api-key` convention from [`opentake_gen::keys`]. Validating the -/// provider here means an unknown value can never address an arbitrary keychain -/// entry — the only writable accounts are the three the UI offers. +/// Allowed chat + generation providers → fixed keychain accounts. Validating +/// here prevents arbitrary keychain account access from the WebView. fn account_for(provider: &str) -> Result<&'static str, String> { match provider { "anthropic" => Ok("anthropic-api-key"), + "fal" => Ok("fal-api-key"), + "replicate" => Ok("replicate-api-key"), "openai" => Ok("openai-api-key"), + "elevenlabs" => Ok("elevenlabs-api-key"), "google" => Ok("google-api-key"), other => Err(format!("unknown provider: {other}")), } @@ -106,7 +107,10 @@ mod tests { #[test] fn account_mapping_is_stable_for_known_providers() { assert_eq!(account_for("anthropic").unwrap(), "anthropic-api-key"); + assert_eq!(account_for("fal").unwrap(), "fal-api-key"); + assert_eq!(account_for("replicate").unwrap(), "replicate-api-key"); assert_eq!(account_for("openai").unwrap(), "openai-api-key"); + assert_eq!(account_for("elevenlabs").unwrap(), "elevenlabs-api-key"); assert_eq!(account_for("google").unwrap(), "google-api-key"); } diff --git a/web/src/components/media/MediaPanel.tsx b/web/src/components/media/MediaPanel.tsx index 33f7af8b..bb470839 100644 --- a/web/src/components/media/MediaPanel.tsx +++ b/web/src/components/media/MediaPanel.tsx @@ -50,7 +50,15 @@ import { BoundedCache } from "../../lib/lru"; import { childFolders, folderTrail, normalizeFolderId } from "../../lib/folderTree"; import { useProjectStore } from "../../store/projectStore"; import { addMediaToTimeline } from "../../store/editActions"; -import { extractAudio, generateThumbnail, getWaveform, preloadMedia, toggleFavorite } from "../../lib/api"; +import { + cancelGeneration, + retryGeneration, + extractAudio, + generateThumbnail, + getWaveform, + preloadMedia, + toggleFavorite, +} from "../../lib/api"; import { saveDialog } from "../../lib/dialog"; import type { MediaFolder, MediaItem } from "../../lib/types"; import { MediaTabBar, MediaSubTabBar, MATERIAL_SUB_TABS, AUDIO_SUB_TABS } from "./MediaTabBar"; @@ -807,6 +815,10 @@ function MediaCard({ item }: { item: MediaItem }) { const durationFrames = Math.round(item.duration * fps); const selected = previewMediaId === item.id; const favorite = item.favorite ?? false; + const generationActive = + item.generationStatus === "generating" || item.generationStatus === "downloading"; + const generationFailed = + item.generationStatus === "failed" || item.generationStatus === "cancelled"; const thumbnailKey = mediaThumbnailKey(item); const [lazyThumbnail, setLazyThumbnail] = useState( item.thumbnail ?? mediaThumbnailCache.get(thumbnailKey) ?? null, @@ -923,20 +935,28 @@ function MediaCard({ item }: { item: MediaItem }) { return (
{ + if (generationActive) return; setPreviewMedia(item.id); // Warm poster/sprite/waveform caches so preview + a later timeline drop // are instant instead of decoding on the interaction path. void preloadMedia(item.id); }} - onDoubleClick={() => void addMediaToTimeline(item)} + onDoubleClick={() => { + if (!generationActive && !generationFailed) void addMediaToTimeline(item); + }} onMouseEnter={() => setHovered(true)} onMouseLeave={() => setHovered(false)} title={item.name} - style={{ display: "flex", flexDirection: "column", gap: 4, cursor: "grab" }} + style={{ + display: "flex", + flexDirection: "column", + gap: 4, + cursor: generationActive ? "progress" : generationFailed ? "default" : "grab", + }} > {/* Thumbnail: generated cache image only. Missing thumbnails are requested lazily as cards enter view, so import/list commands stay cheap. */} @@ -991,6 +1011,98 @@ function MediaCard({ item }: { item: MediaItem }) { {formatTimecode(durationFrames, fps)} )} + {generationActive && ( +
event.stopPropagation()} + style={{ + position: "absolute", + inset: 0, + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: 6, + background: "rgba(15,18,24,0.76)", + color: "#fff", + textAlign: "center", + padding: 8, + }} + > + + + {item.generationStatus === "downloading" ? "正在下载结果" : "正在生成"} + {typeof item.generationProgress === "number" + ? ` ${Math.round(item.generationProgress * 100)}%` + : ""} + + {item.generationInput?.jobId && ( + + )} +
+ )} + {generationFailed && ( +
event.stopPropagation()} + style={{ + position: "absolute", + inset: 0, + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: 4, + background: "rgba(180,35,35,0.55)", + color: "#fff", + textAlign: "center", + padding: 6, + }} + > + + + {item.generationStatus === "cancelled" + ? "生成已取消" + : item.generationErrorCode ?? "GENERATION_FAILED"} + + {item.generationInput?.jobId && ( + + )} +
+ )} {/* Offline overlay: the source file is missing. Relink keeps the asset id, so the timeline clips referencing it recover (no re-import). */} {item.missing && ( diff --git a/web/src/components/settings/SettingsView.tsx b/web/src/components/settings/SettingsView.tsx index 8a6fce17..2942de6a 100644 --- a/web/src/components/settings/SettingsView.tsx +++ b/web/src/components/settings/SettingsView.tsx @@ -488,7 +488,10 @@ function ImportPane() { const PROVIDERS: Array<{ id: ByokProvider; label: string }> = [ { id: "anthropic", label: "Anthropic" }, + { id: "fal", label: "fal.ai" }, + { id: "replicate", label: "Replicate" }, { id: "openai", label: "OpenAI" }, + { id: "elevenlabs", label: "ElevenLabs" }, { id: "google", label: "Google" }, ]; diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 0539205c..9f7c475e 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -312,6 +312,23 @@ export async function cancelExport(operationId: string): Promise { if (invokeImpl) await invokeImpl("cancel_export", { operationId }); } +export async function cancelGeneration(jobId: string): Promise { + await ensureTauri(); + if (invokeImpl) return invokeImpl("generation_cancel", { jobId }); + return false; +} + +export async function retryGeneration( + jobId: string, + costAuthorized: boolean, +): Promise<{ jobId: string; placeholderAssetIds: string[]; status: string }> { + await ensureTauri(); + if (invokeImpl) { + return invokeImpl("generation_retry", { jobId, costAuthorized }); + } + throw new Error("Generation retry is available only in the desktop app"); +} + /** Progress payload for `"export://progress"`: `done` of `total` frames * composited so far. */ export interface ExportProgress { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 3af11306..5f93a599 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -587,6 +587,9 @@ export interface MediaItem { * sections (mirror of `MediaItemDto.generationInput` / upstream * `MediaAsset.generationInput`). */ generationInput?: GenerationInput | null; + generationStatus?: "none" | "generating" | "downloading" | "failed" | "cancelled"; + generationProgress?: number | null; + generationErrorCode?: string | null; /** `true` when the source file is offline (moved/deleted). Derived from file * existence on the backend; clears after a successful relink. */ missing?: boolean; @@ -610,6 +613,18 @@ export interface GenerationInput { referenceImageAssetIds?: string[] | null; referenceVideoAssetIds?: string[] | null; referenceAudioAssetIds?: string[] | null; + jobId?: string | null; + provider?: string | null; + providerJobId?: string | null; + status?: "queued" | "generating" | "downloading" | "finalizing" | "ready" | "failed" | "cancelled"; + progress?: number | null; + errorCode?: string | null; + outputIndex?: number | null; + sourceAssetId?: string | null; + sourceClipId?: string | null; + sourceStartFrame?: number | null; + sourceEndFrame?: number | null; + estimatedCostCredits?: number | null; } /** A media-library folder (flat list; nest via `parentFolderId`). */ diff --git a/web/src/store/settingsStore.ts b/web/src/store/settingsStore.ts index a5f0c36c..a544c832 100644 --- a/web/src/store/settingsStore.ts +++ b/web/src/store/settingsStore.ts @@ -11,7 +11,13 @@ import { create } from "zustand"; import { isTauri } from "../lib/api"; export type Theme = "dark" | "light"; -export type ByokProvider = "anthropic" | "openai" | "google"; +export type ByokProvider = + | "anthropic" + | "fal" + | "replicate" + | "openai" + | "elevenlabs" + | "google"; export type WindowSizeOpt = "standard" | "compact"; const LS = { @@ -31,7 +37,13 @@ function loadString(key: string): string | null { } function loadProvider(): ByokProvider { const v = loadString(LS.byokProvider); - return v === "openai" || v === "google" ? v : "anthropic"; + return v === "fal" || + v === "replicate" || + v === "openai" || + v === "elevenlabs" || + v === "google" + ? v + : "anthropic"; } function loadWindowSize(): WindowSizeOpt { if (typeof localStorage === "undefined") return "standard"; From 8a7527ac29426b5fa7a6c19f6aa4d47e01ff8af4 Mon Sep 17 00:00:00 2001 From: baiqing Date: Wed, 29 Jul 2026 22:46:58 +0800 Subject: [PATCH 2/4] fix: release project root before generation publish --- crates/opentake-core/src/session.rs | 13 ++++- crates/opentake-project/src/bundle.rs | 50 +++++++++++++++++++ .../generation-finalization-2026-07-29.md | 6 +++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/crates/opentake-core/src/session.rs b/crates/opentake-core/src/session.rs index f4bbf19a..0b51ef69 100644 --- a/crates/opentake-core/src/session.rs +++ b/crates/opentake-core/src/session.rs @@ -1012,7 +1012,18 @@ impl EditorSession { // Generation spans media.json + generation-log.json. Publish a complete // sibling bundle so both become visible at one rename commit point. // The source root carries media/chat/thumbnail into the fresh stage. - let new_root = project.publish_complete_to(&target, self.project_root.as_ref())?; + let source_root = self.project_root.take().ok_or(CoreError::NoProjectOpen)?; + let new_root = match project.publish_complete_replacing_root(&target, source_root) { + Ok(root) => root, + Err(error) => { + // A pre-commit failure restores the original target; recover + // retained authority when possible while preserving the exact + // publication error for the caller. Post-commit ambiguity stays + // fail-closed if the target cannot be reopened. + self.project_root = ProjectRoot::open(&target).ok(); + return Err(error.into()); + } + }; self.project_root = Some(new_root); self.generation_log_component_present = true; Ok(target) diff --git a/crates/opentake-project/src/bundle.rs b/crates/opentake-project/src/bundle.rs index fcd51111..9fedbce2 100644 --- a/crates/opentake-project/src/bundle.rs +++ b/crates/opentake-project/src/bundle.rs @@ -318,6 +318,31 @@ impl Project { } publisher.publish() } + + /// Replace the same bundle represented by an owned retained root. + /// + /// Windows refuses to rename a directory while a process still owns an + /// open directory handle to it. Complete same-target transactions therefore + /// stage and copy through the retained authority first, explicitly close + /// that authority, and only then enter the existing journaled publication + /// commit. Save-As keeps using [`Self::publish_complete_to`] because its + /// source and destination are distinct. + pub fn publish_complete_replacing_root( + &self, + bundle: impl AsRef, + media_source: ProjectRoot, + ) -> Result { + let encoded = EncodedProject::prepare(self)?; + let publisher = ProjectRoot::begin_replace(bundle.as_ref())?; + encoded.write_to(publisher.stage())?; + media_source.copy_media_to(publisher.stage())?; + media_source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + media_source.copy_thumbnail_to(publisher.stage())?; + } + drop(media_source); + publisher.publish() + } } fn sha256_hex(bytes: &[u8]) -> String { @@ -640,6 +665,31 @@ mod tests { ); } + #[test] + fn complete_publish_replaces_the_owned_source_root() { + let tmp = TmpDir::new("complete-same-target"); + let target = tmp.path().join("Project.opentake"); + let mut project = Project::new(&target); + project.timeline.fps = 24; + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/clip.bin"), b"media").unwrap(); + fs::write(target.join("thumbnail.jpg"), b"cover").unwrap(); + let source_root = ProjectRoot::open(&target).unwrap(); + + project.timeline.fps = 48; + let published = project + .publish_complete_replacing_root(&target, source_root) + .expect("same-target publication must release the old root before rename"); + + assert_eq!( + Project::open_from_root(&published).unwrap().timeline.fps, + 48 + ); + assert_eq!(fs::read(target.join("media/clip.bin")).unwrap(), b"media"); + assert_eq!(fs::read(target.join("thumbnail.jpg")).unwrap(), b"cover"); + } + #[cfg(unix)] #[test] fn media_copy_failure_leaves_an_existing_target_tree_byte_exact() { diff --git a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md index 1739d9c5..2cab01e0 100644 --- a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md +++ b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md @@ -31,3 +31,9 @@ git diff --check ``` Final result: all commands passed. Workspace tests reported zero failures; repository-declared ignored tests remained ignored. Web result: 70 test files, 703 tests passed. Vite emitted only the pre-existing chunk-size/dynamic-import optimization warnings. + +## Windows CI correction + +The first exact-tree Windows full-product run (`30459985870`, job `90603221524`) failed all three `generation_persistence` tests with Win32 error 32 while replacing `Generation.opentake`. The durable generation transaction had correctly staged a complete sibling bundle, but the live session still retained an open `ProjectRoot` directory handle when the journaled publisher tried to rename the old target. Windows forbids that rename even though macOS permits it. + +The corrected path copies the complete bundle through the retained root, explicitly consumes and closes that root, and only then invokes the existing journal/backup/restore publication commit. Save-As continues to retain its distinct source root. `complete_publish_replaces_the_owned_source_root` pins this same-target Windows requirement, and the three generation persistence tests plus full format/Clippy/workspace gates pass locally after the correction. The replacement keeps whole-bundle atomicity; it does not fall back to two independently visible JSON writes. From 1a1f3c6852e8a2c684a4dcace07e197aa78d3962 Mon Sep 17 00:00:00 2001 From: baiqing Date: Thu, 30 Jul 2026 19:26:38 +0800 Subject: [PATCH 3/4] fix: release Windows generation handles --- .../automated/generation-finalization-2026-07-29.md | 2 ++ src-tauri/src/generation.rs | 6 ++++++ src-tauri/src/library.rs | 9 +++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md index 2cab01e0..9a28aea1 100644 --- a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md +++ b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md @@ -37,3 +37,5 @@ Final result: all commands passed. Workspace tests reported zero failures; repos The first exact-tree Windows full-product run (`30459985870`, job `90603221524`) failed all three `generation_persistence` tests with Win32 error 32 while replacing `Generation.opentake`. The durable generation transaction had correctly staged a complete sibling bundle, but the live session still retained an open `ProjectRoot` directory handle when the journaled publisher tried to rename the old target. Windows forbids that rename even though macOS permits it. The corrected path copies the complete bundle through the retained root, explicitly consumes and closes that root, and only then invokes the existing journal/backup/restore publication commit. Save-As continues to retain its distinct source root. `complete_publish_replaces_the_owned_source_root` pins this same-target Windows requirement, and the three generation persistence tests plus full format/Clippy/workspace gates pass locally after the correction. The replacement keeps whole-bundle atomicity; it does not fall back to two independently visible JSON writes. + +The next exact-tree Windows run (`30462600088`, job `90612253763`) proved that two Tauri-layer handle lifetimes also had to be corrected. Six generation tests failed: four finalization flows held an uncommitted project-media leaf without `FILE_SHARE_DELETE` while complete-bundle publication renamed the project, and two restart tests kept the pre-restart `AppCore` alive while independently opening the same bundle. The import handle now shares deletion on Windows while retaining delete access, identity validation, and handle-relative rollback. The restart fixtures explicitly drop the old core before constructing the new process-equivalent core. Local `generation::tests` (9/9), `library::tests` (22/22), Tauri Clippy, formatting, and diff checks pass; the exact Windows CI rerun is the authoritative remaining platform confirmation. diff --git a/src-tauri/src/generation.rs b/src-tauri/src/generation.rs index 58e54521..6d387910 100644 --- a/src-tauri/src/generation.rs +++ b/src-tauri/src/generation.rs @@ -2174,6 +2174,10 @@ mod tests { let committed = core .begin_generation_job_for_project(1, &bundle, image_plan()) .unwrap(); + // Model a real process restart. A second independent AppCore retaining + // the old bundle at the same time is not a supported runtime state and + // prevents same-target directory publication on Windows. + drop(core); let reopened = AppCore::new(); reopened.open_project(&bundle).unwrap(); let mock = MockTransport::new(); @@ -2227,6 +2231,8 @@ mod tests { ) .unwrap(); + // Model a real process restart before opening the same bundle again. + drop(core); let reopened = AppCore::new(); let reopened_snapshot = reopened.open_project(&bundle).unwrap(); let mock = MockTransport::new(); diff --git a/src-tauri/src/library.rs b/src-tauri/src/library.rs index 183530fd..68dd359d 100644 --- a/src-tauri/src/library.rs +++ b/src-tauri/src/library.rs @@ -621,11 +621,16 @@ impl ProjectMediaCapability { use cap_std::fs::OpenOptionsExt; use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE}; use windows_sys::Win32::Storage::FileSystem::{ - DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + DELETE, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, }; options .access_mode(GENERIC_READ | GENERIC_WRITE | DELETE) - .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE); + // Generation publishes media.json + generation-log.json by + // replacing the complete bundle. Keep the retained import + // handle authoritative without blocking that directory rename + // on Windows; identity checks and handle-relative rollback + // remain valid after a namespace move. + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); } let file = self .media From 84d869822a6b0707516d35be57c40d0e79811ac9 Mon Sep 17 00:00:00 2001 From: baiqing Date: Thu, 30 Jul 2026 20:56:53 +0800 Subject: [PATCH 4/4] fix: publish generated media atomically --- crates/opentake-core/src/core.rs | 42 +++++++- crates/opentake-core/src/session.rs | 32 ++++++ .../tests/generation_persistence.rs | 21 +++- crates/opentake-project/src/bundle.rs | 102 ++++++++++++++++++ crates/opentake-project/src/project_root.rs | 51 +++++++++ .../generation-finalization-2026-07-29.md | 6 +- src-tauri/src/generation.rs | 33 ++---- src-tauri/src/library.rs | 8 +- 8 files changed, 260 insertions(+), 35 deletions(-) diff --git a/crates/opentake-core/src/core.rs b/crates/opentake-core/src/core.rs index 4c5e170a..cfe4f0b7 100644 --- a/crates/opentake-core/src/core.rs +++ b/crates/opentake-core/src/core.rs @@ -722,6 +722,31 @@ impl AppCore { ) } + /// Finalize a generated output and stream its media bytes into the same + /// complete-bundle publication as the manifest and generation log. + pub fn finalize_generation_output_with_media_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + output: PreparedGenerationOutput, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result<()> { + let expected_relative_path = format!("media/{media_leaf}"); + if output.relative_path != expected_relative_path { + return Err(CoreError::Media( + "generation output path does not match its media leaf".to_string(), + )); + } + self.persist_generation_mutation_using( + expected_project_epoch, + expected_project_dir, + |editor, ids| editor.finalize_generation_output(output, ids), + |editor| editor.save_generation_state_with_media(media_leaf, media_byte_size, media), + ) + } + pub fn fail_generation_output_for_project( &self, expected_project_epoch: u64, @@ -756,6 +781,21 @@ impl AppCore { expected_project_epoch: u64, expected_project_dir: &Path, mutate: impl FnOnce(&mut EditorSession, &dyn IdGen) -> Result, + ) -> Result { + self.persist_generation_mutation_using( + expected_project_epoch, + expected_project_dir, + mutate, + |editor| editor.save_generation_state(), + ) + } + + fn persist_generation_mutation_using( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + mutate: impl FnOnce(&mut EditorSession, &dyn IdGen) -> Result, + persist: impl FnOnce(&mut EditorSession) -> Result, ) -> Result { let (value, count, written) = { let mut session = self.lock(); @@ -763,7 +803,7 @@ impl AppCore { let checkpoint = session.editor.checkpoint_generation_state(); let result = (|| { let value = mutate(&mut session.editor, self.ids.as_ref())?; - let written = session.editor.save_generation_state()?; + let written = persist(&mut session.editor)?; Ok((value, written)) })(); match result { diff --git a/crates/opentake-core/src/session.rs b/crates/opentake-core/src/session.rs index 0b51ef69..ced3d523 100644 --- a/crates/opentake-core/src/session.rs +++ b/crates/opentake-core/src/session.rs @@ -1029,6 +1029,38 @@ impl EditorSession { Ok(target) } + pub(crate) fn save_generation_state_with_media( + &mut self, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result { + self.ensure_mutable()?; + let target = self.project_dir.clone().ok_or(CoreError::NoProjectOpen)?; + let mut project = + Project::new_with_compatibility(target.clone(), self.compatibility.clone()); + project.timeline = self.state.timeline.clone(); + project.manifest = self.state.manifest.clone(); + project.generation_log = Some(self.generation_log.clone()); + let source_root = self.project_root.take().ok_or(CoreError::NoProjectOpen)?; + let new_root = match project.publish_complete_replacing_root_with_media( + &target, + source_root, + media_leaf, + media_byte_size, + media, + ) { + Ok(root) => root, + Err(error) => { + self.project_root = ProjectRoot::open(&target).ok(); + return Err(error.into()); + } + }; + self.project_root = Some(new_root); + self.generation_log_component_present = true; + Ok(target) + } + pub(crate) fn checkpoint_generation_state(&self) -> GenerationStateCheckpoint { GenerationStateCheckpoint { manifest: self.state.manifest.clone(), diff --git a/crates/opentake-core/tests/generation_persistence.rs b/crates/opentake-core/tests/generation_persistence.rs index e9350db2..94bf0eec 100644 --- a/crates/opentake-core/tests/generation_persistence.rs +++ b/crates/opentake-core/tests/generation_persistence.rs @@ -125,9 +125,10 @@ fn placeholders_job_events_and_finalized_output_survive_restart() { ) .unwrap(); - let relative_path = format!("media/{asset_id}.png"); - fs::write(bundle.join(&relative_path), b"upscaled-bytes").unwrap(); - core.finalize_generation_output_for_project( + let media_leaf = format!("{asset_id}.png"); + let relative_path = format!("media/{media_leaf}"); + let mut generated_media = std::io::Cursor::new(b"upscaled-bytes"); + core.finalize_generation_output_with_media_for_project( runtime.project_epoch, &bundle, PreparedGenerationOutput { @@ -142,6 +143,9 @@ fn placeholders_job_events_and_finalized_output_survive_restart() { }, created_at: Some(800_000_002.0), }, + &media_leaf, + 14, + &mut generated_media, ) .unwrap(); @@ -167,11 +171,20 @@ fn placeholders_job_events_and_finalized_output_survive_restart() { .unwrap(); assert_eq!(output.source_width, Some(8)); assert_eq!(output.source_height, Some(6)); - assert_eq!(output.source, MediaSource::Project { relative_path }); + assert_eq!( + output.source, + MediaSource::Project { + relative_path: relative_path.clone() + } + ); assert_eq!( output.generation_input.as_ref().unwrap().status, Some(GenerationJobStatus::Ready) ); + assert_eq!( + fs::read(bundle.join(&relative_path)).unwrap(), + b"upscaled-bytes" + ); assert_eq!(fs::read(bundle.join("thumbnail.jpg")).unwrap(), b"cover"); let log = reopened.generation_log(); diff --git a/crates/opentake-project/src/bundle.rs b/crates/opentake-project/src/bundle.rs index 9fedbce2..075a40c6 100644 --- a/crates/opentake-project/src/bundle.rs +++ b/crates/opentake-project/src/bundle.rs @@ -343,6 +343,36 @@ impl Project { drop(media_source); publisher.publish() } + + /// Replace the owned source bundle while adding one generated media leaf + /// directly to the unpublished sibling stage. + /// + /// This keeps the media bytes, `media.json`, and `generation-log.json` in + /// one directory-publication transaction. In particular, callers do not + /// need to retain an open handle inside the live target across its Windows + /// rename commit point. + pub fn publish_complete_replacing_root_with_media( + &self, + bundle: impl AsRef, + media_source: ProjectRoot, + media_leaf: &str, + media_byte_size: u64, + media: &mut dyn std::io::Read, + ) -> Result { + let encoded = EncodedProject::prepare(self)?; + let publisher = ProjectRoot::begin_replace(bundle.as_ref())?; + encoded.write_to(publisher.stage())?; + media_source.copy_media_to(publisher.stage())?; + publisher + .stage() + .write_new_media_leaf(media_leaf, media_byte_size, media)?; + media_source.copy_chat_sessions_to(publisher.stage())?; + if self.thumbnail.is_none() { + media_source.copy_thumbnail_to(publisher.stage())?; + } + drop(media_source); + publisher.publish() + } } fn sha256_hex(bytes: &[u8]) -> String { @@ -690,6 +720,78 @@ mod tests { assert_eq!(fs::read(target.join("thumbnail.jpg")).unwrap(), b"cover"); } + #[test] + fn complete_publish_streams_generated_media_into_the_new_bundle() { + let tmp = TmpDir::new("complete-generated-media"); + let target = tmp.path().join("Project.opentake"); + let mut project = Project::new(&target); + project.timeline.fps = 24; + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/source.bin"), b"source").unwrap(); + let source_root = ProjectRoot::open(&target).unwrap(); + let mut generated = std::io::Cursor::new(b"generated"); + + project + .publish_complete_replacing_root_with_media( + &target, + source_root, + "output.bin", + 9, + &mut generated, + ) + .expect("generated media must share the bundle publication commit"); + + assert_eq!( + fs::read(target.join("media/source.bin")).unwrap(), + b"source" + ); + assert_eq!( + fs::read(target.join("media/output.bin")).unwrap(), + b"generated" + ); + } + + #[test] + fn generated_media_stream_failure_preserves_the_live_bundle_byte_exact() { + struct FailingReader(bool); + + impl std::io::Read for FailingReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0 { + return Err(std::io::Error::other("injected media read failure")); + } + self.0 = true; + let bytes = b"partial"; + buffer[..bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + } + + let tmp = TmpDir::new("complete-generated-media-failure"); + let target = tmp.path().join("Project.opentake"); + let project = Project::new(&target); + project.save().unwrap(); + fs::create_dir_all(target.join("media")).unwrap(); + fs::write(target.join("media/source.bin"), b"source").unwrap(); + let before = tree_receipt(&target); + let source_root = ProjectRoot::open(&target).unwrap(); + let mut generated = FailingReader(false); + + project + .publish_complete_replacing_root_with_media( + &target, + source_root, + "output.bin", + 14, + &mut generated, + ) + .expect_err("a failed generated media stream must abort publication"); + + assert_eq!(tree_receipt(&target), before); + assert!(!target.join("media/output.bin").exists()); + } + #[cfg(unix)] #[test] fn media_copy_failure_leaves_an_existing_target_tree_byte_exact() { diff --git a/crates/opentake-project/src/project_root.rs b/crates/opentake-project/src/project_root.rs index d3bc2347..4faa1e9c 100644 --- a/crates/opentake-project/src/project_root.rs +++ b/crates/opentake-project/src/project_root.rs @@ -157,6 +157,57 @@ impl ProjectRoot { self.copy_directory_component_to(destination, crate::layout::MEDIA_DIR, "media-copy") } + /// Write one fresh media leaf into this retained bundle. + /// + /// Complete generation publication uses this only on an unpublished stage, + /// after the existing media tree has been copied. The final leaf is created + /// with `create_new`, streamed without an ambient destination path, checked + /// against the downloader's exact byte count, synced, and kept only after + /// every write succeeds. + pub(crate) fn write_new_media_leaf( + &self, + name: &str, + expected_bytes: u64, + source: &mut dyn Read, + ) -> Result<()> { + validate_leaf(name).map_err(|error| { + ProjectError::io(self.path.join(crate::layout::MEDIA_DIR).join(name), error) + })?; + let media_path = self.path.join(crate::layout::MEDIA_DIR); + match self.dir.create_dir(crate::layout::MEDIA_DIR) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(ProjectError::io(&media_path, error)), + } + let media = self + .dir + .open_dir_nofollow(crate::layout::MEDIA_DIR) + .map_err(|error| ProjectError::io(&media_path, error))?; + let mut leaf = TransactionLeaf::create(&media, name) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + let copied = std::io::copy( + &mut source.take(expected_bytes.saturating_add(1)), + leaf.handle.as_file_mut(), + ) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + if copied != expected_bytes { + return Err(ProjectError::io( + media_path.join(name), + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "generated media size changed before publication", + ), + )); + } + leaf.handle + .as_file_mut() + .flush() + .and_then(|()| leaf.handle.as_file().sync_all()) + .map_err(|error| ProjectError::io(media_path.join(name), error))?; + leaf.cleanup_on_drop = false; + Ok(()) + } + /// Copy project-local Agent conversations during complete-bundle /// publication (Save As / archive) through retained no-follow roots. pub fn copy_chat_sessions_to(&self, destination: &ProjectRoot) -> Result<()> { diff --git a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md index 9a28aea1..ece025f2 100644 --- a/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md +++ b/docs/audit/2026-07-14/runtime-artifacts/automated/generation-finalization-2026-07-29.md @@ -38,4 +38,8 @@ The first exact-tree Windows full-product run (`30459985870`, job `90603221524`) The corrected path copies the complete bundle through the retained root, explicitly consumes and closes that root, and only then invokes the existing journal/backup/restore publication commit. Save-As continues to retain its distinct source root. `complete_publish_replaces_the_owned_source_root` pins this same-target Windows requirement, and the three generation persistence tests plus full format/Clippy/workspace gates pass locally after the correction. The replacement keeps whole-bundle atomicity; it does not fall back to two independently visible JSON writes. -The next exact-tree Windows run (`30462600088`, job `90612253763`) proved that two Tauri-layer handle lifetimes also had to be corrected. Six generation tests failed: four finalization flows held an uncommitted project-media leaf without `FILE_SHARE_DELETE` while complete-bundle publication renamed the project, and two restart tests kept the pre-restart `AppCore` alive while independently opening the same bundle. The import handle now shares deletion on Windows while retaining delete access, identity validation, and handle-relative rollback. The restart fixtures explicitly drop the old core before constructing the new process-equivalent core. Local `generation::tests` (9/9), `library::tests` (22/22), Tauri Clippy, formatting, and diff checks pass; the exact Windows CI rerun is the authoritative remaining platform confirmation. +The next exact-tree Windows run (`30462600088`, job `90612253763`) proved that two Tauri-layer handle lifetimes also had to be corrected. Six generation tests failed: four finalization flows held an uncommitted project-media leaf without `FILE_SHARE_DELETE` while complete-bundle publication renamed the project, and two restart tests kept the pre-restart `AppCore` alive while independently opening the same bundle. The import handle now shares deletion on Windows while retaining delete access, identity validation, and handle-relative rollback. The restart fixtures explicitly drop the old core before constructing the new process-equivalent core. + +A later full-product Windows run (`30538580323`, job `90857715826`) narrowed the remaining error-32 failures to five successful-provider finalization paths. Sharing deletion on the leaf was insufficient because `ProjectMediaCapability` also retained root and `media/` directory handles across the live bundle rename. Finalization now streams the downloaded artifact directly into the unpublished sibling stage through `ProjectRoot`; the generated media leaf, `media.json`, and `generation-log.json` therefore share the existing journaled whole-bundle commit. The stream is bounded to the downloader-recorded byte size, and an undersized, oversized, or failed stream aborts without changing the live bundle. `generated_media_stream_failure_preserves_the_live_bundle_byte_exact` pins that rollback guarantee. + +After this correction, local `generation::tests` pass 9/9, the complete workspace test run reports zero failures (only repository-declared hardware probes remain ignored), workspace Clippy with warnings denied passes, and formatting/diff checks pass. The next exact-tree Windows CI rerun is the authoritative remaining platform confirmation. diff --git a/src-tauri/src/generation.rs b/src-tauri/src/generation.rs index 6d387910..33b6c516 100644 --- a/src-tauri/src/generation.rs +++ b/src-tauri/src/generation.rs @@ -2,9 +2,8 @@ //! //! Durable placeholder state is committed before provider submission. Provider //! keys stay in the OS keychain; signed result URLs and provider diagnostics are -//! never persisted. Terminal downloads are probed and imported through retained -//! project-media capabilities before the original placeholder identity becomes -//! ready. +//! never persisted. Terminal downloads are probed, then streamed into the same +//! complete-bundle publication that makes the original placeholder ready. use std::collections::{BTreeSet, HashMap}; use std::io::{Read, Write}; @@ -33,8 +32,6 @@ use opentake_gen::{ }; use opentake_media::{MediaCancelToken, MediaEngine}; -use crate::library::ProjectMediaCapability; - const RESULT_BYTES_MAX: u64 = 1024 * 1024 * 1024; const DATA_URL_ENCODED_MAX: usize = 512 * 1024 * 1024; const RESULT_REDIRECT_MAX: usize = 5; @@ -1381,25 +1378,10 @@ impl GenerationFinalizationStore for TauriFinalizationStore { probe.format_name.as_deref(), )?; let leaf = format!("{asset_id}.{extension}"); - let capability = ProjectMediaCapability::open_verified( - &self.bridge.core, - self.project_epoch, - &self.project_dir, - true, - )?; - let mut import = capability.create_import(Path::new(&leaf))?; let mut source = std::fs::File::open(&artifact.path).map_err(|error| error.to_string())?; - std::io::copy(&mut source, import.file_mut()).map_err(|error| error.to_string())?; - import - .file() - .sync_all() - .map_err(|error| error.to_string())?; - if !capability.matches_leaf(&import)? { - return Err("generation import identity changed".to_string()); - } self.bridge .core - .finalize_generation_output_for_project( + .finalize_generation_output_with_media_for_project( self.project_epoch, &self.project_dir, PreparedGenerationOutput { @@ -1414,9 +1396,11 @@ impl GenerationFinalizationStore for TauriFinalizationStore { }, created_at: Some(now_apple_reference_seconds()), }, + &leaf, + artifact.byte_size, + &mut source, ) .map_err(|error| error.to_string())?; - import.commit(); let _ = std::fs::remove_file(&artifact.path); Ok(()) } @@ -2736,7 +2720,8 @@ mod tests { .unwrap(); std::fs::create_dir_all(&cache).unwrap(); let staged = cache.join("upscale.png"); - std::fs::write(&staged, png_bytes(6, 4)).unwrap(); + let staged_bytes = png_bytes(6, 4); + std::fs::write(&staged, &staged_bytes).unwrap(); let store = TauriFinalizationStore { bridge: bridge.as_ref().clone(), project_epoch: snapshot.project_epoch, @@ -2748,7 +2733,7 @@ mod tests { DownloadedGenerationArtifact { path: staged, media_type: "image/png".to_string(), - byte_size: 0, + byte_size: staged_bytes.len() as u64, }, ) .unwrap(); diff --git a/src-tauri/src/library.rs b/src-tauri/src/library.rs index 68dd359d..4c0cfa53 100644 --- a/src-tauri/src/library.rs +++ b/src-tauri/src/library.rs @@ -625,11 +625,9 @@ impl ProjectMediaCapability { }; options .access_mode(GENERIC_READ | GENERIC_WRITE | DELETE) - // Generation publishes media.json + generation-log.json by - // replacing the complete bundle. Keep the retained import - // handle authoritative without blocking that directory rename - // on Windows; identity checks and handle-relative rollback - // remain valid after a namespace move. + // Keep the retained import compatible with a whole-bundle + // rename on Windows while preserving identity validation and + // handle-relative rollback after a namespace move. .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); } let file = self