From 61a4fb63b8f6674b6354566b9c9c23231d72bf7c Mon Sep 17 00:00:00 2001 From: Cui Hao Date: Mon, 20 Jul 2026 15:36:19 +0800 Subject: [PATCH 1/2] feat: resolve 10 open issues (#148,#147,#161,#202,#25,#49,#160,#131,#65,#34) on fresh upstream/main base --- Cargo.lock | 29 ++ crates/opentake-domain/src/lib.rs | 5 +- crates/opentake-domain/src/media.rs | 107 +++++ crates/opentake-domain/src/timeline.rs | 6 +- crates/opentake-media/Cargo.toml | 5 + crates/opentake-media/src/decode/mod.rs | 4 +- crates/opentake-media/src/decode/pcm.rs | 216 ++++++++++ crates/opentake-media/src/ff.rs | 57 +++ crates/opentake-media/src/lib.rs | 9 +- crates/opentake-motion/Cargo.toml | 14 + crates/opentake-motion/src/renderer.rs | 87 ++++ crates/opentake-ops/src/command.rs | 28 +- crates/opentake-ops/src/ops/mod.rs | 2 +- crates/opentake-ops/src/ops/tracks.rs | 25 ++ crates/opentake-ops/tests/command_apply.rs | 55 +++ crates/opentake-render/src/gpu/compositor.rs | 178 +++++++-- crates/opentake-render/src/gpu/lottie.rs | 123 ++++++ crates/opentake-render/src/gpu/mod.rs | 4 +- crates/opentake-render/src/lib.rs | 3 +- docs/visual-validation-checklist.md | 50 +++ src-tauri/Cargo.toml | 10 +- src-tauri/src/commands.rs | 7 + src-tauri/src/lib.rs | 31 +- src-tauri/src/media.rs | 195 ++++++++- src-tauri/src/motion.rs | 340 ++++++++++++++++ src-tauri/src/render.rs | 54 ++- src-tauri/tauri.conf.json | 2 +- web/src/components/inspector/Inspector.tsx | 44 +- web/src/components/media/FolderBrowser.tsx | 377 ++++++++++++++++++ web/src/components/media/MediaPanel.tsx | 123 +++--- web/src/components/preview/Preview.tsx | 29 +- .../preview/TimelinePlaybackLayer.tsx | 83 ++++ .../preview/timelinePlayback.test.ts | 77 +++- .../components/preview/timelinePlayback.ts | 33 ++ .../components/timeline/TrackHeaderColumn.tsx | 24 +- web/src/i18n/dict.ts | 18 + web/src/lib/api.ts | 100 ++++- web/src/lib/audioCache.test.ts | 103 +++++ web/src/lib/audioCache.ts | 96 +++++ web/src/lib/codecSupport.ts | 71 ++++ web/src/lib/fallback.ts | 12 + web/src/lib/theme.ts | 31 +- web/src/lib/types.ts | 94 +++++ web/src/lib/zones.test.ts | 2 +- web/src/store/editActions.test.ts | 1 + web/src/store/editActions.ts | 7 + web/src/store/mediaActions.ts | 21 + 47 files changed, 2869 insertions(+), 123 deletions(-) create mode 100644 crates/opentake-render/src/gpu/lottie.rs create mode 100644 docs/visual-validation-checklist.md create mode 100644 src-tauri/src/motion.rs create mode 100644 web/src/components/media/FolderBrowser.tsx create mode 100644 web/src/lib/audioCache.test.ts create mode 100644 web/src/lib/audioCache.ts create mode 100644 web/src/lib/codecSupport.ts diff --git a/Cargo.lock b/Cargo.lock index a52fba0d..aa7625f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1360,6 +1360,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126522985748cb6a56a966037c3d86d94414be7a01d7749eccf71f7290f6eacd" dependencies = [ "anyhow", + "tar", + "ureq", + "xz2", + "zip", ] [[package]] @@ -2671,6 +2675,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "mach2" version = "0.4.3" @@ -3531,6 +3546,7 @@ dependencies = [ "opentake-domain", "opentake-gen", "opentake-media", + "opentake-motion", "opentake-ops", "opentake-project", "opentake-render", @@ -4371,6 +4387,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -6097,14 +6114,17 @@ checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ "base64 0.22.1", "der", + "flate2", "log", "native-tls", "percent-encoding", + "rustls", "rustls-pki-types", "socks", "ureq-proto", "utf8-zero", "webpki-root-certs", + "webpki-roots", ] [[package]] @@ -7218,6 +7238,15 @@ version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yazi" version = "0.1.6" diff --git a/crates/opentake-domain/src/lib.rs b/crates/opentake-domain/src/lib.rs index 8892dff7..e168d42b 100644 --- a/crates/opentake-domain/src/lib.rs +++ b/crates/opentake-domain/src/lib.rs @@ -45,8 +45,9 @@ pub use keyframe::{ KeyframeInterpolatable, KeyframeTrack, }; pub use media::{ - GenerationInput, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, MediaManifestEntry, - MediaResolver, MediaSource, + is_motion_ref, motion_hash_from_ref, motion_ref_for_hash, GenerationInput, GenerationStatus, + MediaAsset, MediaFolder, MediaManifest, MediaManifestEntry, MediaResolver, MediaSource, + MOTION_REF_PREFIX, }; pub use signal::{ ContextSignal, EditingSkeleton, EditingStage, StageGuidance, TrackHint, TrackRole, diff --git a/crates/opentake-domain/src/media.rs b/crates/opentake-domain/src/media.rs index 3ae9ac47..99666710 100644 --- a/crates/opentake-domain/src/media.rs +++ b/crates/opentake-domain/src/media.rs @@ -26,6 +26,28 @@ use crate::clip_type::ClipType; /// Where a media file lives. Encoded externally-tagged to match Swift's /// synthesized `Codable` for an enum with associated values: /// `{"external":{"absolutePath":"..."}}` / `{"project":{"relativePath":"..."}}`. +/// +/// # Why there is no `Motion` variant +/// +/// `MediaSource` models *where a file lives on disk* — an absolute path +/// (`External`) or a project-bundle-relative path (`Project`). A motion-graphic +/// clip is not a single file: it is a content-addressed sequence of RGBA frames +/// produced by `opentake-motion` and cached under a SHA-256 hash. Forcing it +/// into this enum would: +/// +/// - Break the 1:1 Swift `Codable` round-trip (upstream has no motion variant). +/// - Force every `match` on `MediaSource` (`MediaResolver::expected_path`, +/// `resolve_source_path`, `MediaItemDto::from_entry`, `to_manifest_entry`) to +/// handle an arm that has no on-disk file path. +/// - Conflate "file location" with "media kind" — the kind is already carried +/// by `ClipType` / `Clip.media_ref`. +/// +/// Instead, motion clips reuse the existing `Clip.media_ref: String` field with +/// a dedicated URI scheme: `motion://`. The compositor's media +/// resolver recognizes the prefix and maps the hash to a `MotionClipSource` +/// (see `opentake-motion::integration`). This keeps the domain crate +/// dependency-free and the `MediaSource` enum focused on file location, while +/// motion clips flow through the same `media_ref` channel as every other clip. #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum MediaSource { @@ -35,6 +57,48 @@ pub enum MediaSource { Project { relative_path: String }, } +/// The URI scheme prefix marking a `media_ref` as a rendered motion-graphic +/// clip (e.g. `motion://a1b2c3…`). The rest of the string is the +/// `opentake-motion` content hash (SHA-256 hex) that names the frame cache dir. +pub const MOTION_REF_PREFIX: &str = "motion://"; + +/// `true` when `media_ref` refers to a rendered motion-graphic clip +/// (`motion://`), as opposed to a manifest asset id or file path. +/// +/// Pure; safe to call on any `media_ref` string. Used by the render/compositor +/// layer to route the ref to a `MotionClipSource` instead of the file decoder. +pub fn is_motion_ref(media_ref: &str) -> bool { + media_ref.starts_with(MOTION_REF_PREFIX) +} + +/// Extract the content hash from a `motion://` ref, or `None` when the +/// ref is not a motion ref or carries an empty hash. The returned slice borrows +/// from `media_ref` (no allocation). +/// +/// ``` +/// use opentake_domain::motion_hash_from_ref; +/// assert_eq!( +/// motion_hash_from_ref("motion://abc123"), +/// Some("abc123") +/// ); +/// assert_eq!(motion_hash_from_ref("asset-id-1"), None); +/// assert_eq!(motion_hash_from_ref("motion://"), None); // empty hash +/// ``` +pub fn motion_hash_from_ref(media_ref: &str) -> Option<&str> { + let hash = media_ref.strip_prefix(MOTION_REF_PREFIX)?; + if hash.is_empty() { + None + } else { + Some(hash) + } +} + +/// Build a `motion://` media_ref from a content hash. The inverse of +/// [`motion_hash_from_ref`]. Pure; does not validate the hash format. +pub fn motion_ref_for_hash(hash: &str) -> String { + format!("{MOTION_REF_PREFIX}{hash}") +} + /// Full serializable input snapshot for a generated asset. 1:1 port of /// `GenerationInput`. `prompt` / `model` / `duration` / `aspect_ratio` are /// required upstream; everything else is optional. @@ -566,6 +630,49 @@ mod tests { assert_eq!(s, back); } + // --- Motion media_ref helpers (motion://) --- + + #[test] + fn is_motion_ref_recognizes_prefix() { + assert!(is_motion_ref("motion://abc123")); + assert!(is_motion_ref("motion://deadbeef")); + // Not a motion ref: manifest asset ids, file paths, plain strings. + assert!(!is_motion_ref("asset-id-1")); + assert!(!is_motion_ref("/abs/clip.mp4")); + assert!(!is_motion_ref("media/clip.mov")); + assert!(!is_motion_ref("")); + // Prefix must be exact — "motion:" without "//" is not a motion ref. + assert!(!is_motion_ref("motion:abc")); + } + + #[test] + fn motion_hash_from_ref_extracts_hash() { + assert_eq!( + motion_hash_from_ref("motion://a1b2c3d4e5"), + Some("a1b2c3d4e5") + ); + // A realistic 64-char SHA-256 hex hash. + let hash = "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"; + let r = motion_ref_for_hash(hash); + assert!(is_motion_ref(&r)); + assert_eq!(motion_hash_from_ref(&r), Some(hash)); + } + + #[test] + fn motion_hash_from_ref_rejects_non_motion_and_empty() { + assert_eq!(motion_hash_from_ref("asset-id-1"), None); + assert_eq!(motion_hash_from_ref("/path/x.mp4"), None); + // Prefix present but empty hash -> None (no valid motion clip). + assert_eq!(motion_hash_from_ref("motion://"), None); + } + + #[test] + fn motion_ref_for_hash_roundtrips() { + let r = motion_ref_for_hash("abc"); + assert_eq!(r, "motion://abc"); + assert_eq!(motion_hash_from_ref(&r), Some("abc")); + } + // --- MediaManifest version fallback --- #[test] diff --git a/crates/opentake-domain/src/timeline.rs b/crates/opentake-domain/src/timeline.rs index fd22ae5c..97b27e69 100644 --- a/crates/opentake-domain/src/timeline.rs +++ b/crates/opentake-domain/src/timeline.rs @@ -92,6 +92,8 @@ pub struct Track { #[serde(default)] pub muted: bool, #[serde(default)] + pub soloed: bool, + #[serde(default)] pub hidden: bool, #[serde(default = "default_sync_locked")] pub sync_locked: bool, @@ -107,6 +109,7 @@ impl Track { id: id.into(), kind, muted: false, + soloed: false, hidden: false, sync_locked: true, clips: Vec::new(), @@ -219,12 +222,13 @@ mod tests { #[test] fn track_decode_defaults_missing_fields() { - // Only `type` present; id->"", muted/hidden->false, sync_locked->true. + // Only `type` present; id->"", muted/soloed/hidden->false, sync_locked->true. let json = r#"{"type":"audio"}"#; let t: Track = serde_json::from_str(json).unwrap(); assert_eq!(t.kind, ClipType::Audio); assert_eq!(t.id, ""); assert!(!t.muted); + assert!(!t.soloed); assert!(!t.hidden); assert!(t.sync_locked); assert!(t.clips.is_empty()); diff --git a/crates/opentake-media/Cargo.toml b/crates/opentake-media/Cargo.toml index 8afcafae..a6352852 100644 --- a/crates/opentake-media/Cargo.toml +++ b/crates/opentake-media/Cargo.toml @@ -68,6 +68,11 @@ default = [] # Test-only cross-crate fault boundaries. Shipped builds do not expose these # hooks; integration tests opt in through a dev-dependency feature union. test-faults = [] +# Enable ffmpeg-sidecar's `download_ffmpeg` feature so `ensure_ffmpeg` can +# auto-download the binaries at first run when they aren't on PATH. Pulls in +# ureq + rustls + zip/tar/xz2 (the crate's only HTTP/TLS stack). Enabled by the +# Tauri desktop shell so end users get a working ffmpeg without manual install. +ffmpeg-download = ["ffmpeg-sidecar/download_ffmpeg"] # Real SigLIP2 inference via ONNX Runtime. `download-binaries` lets ort fetch a # prebuilt onnxruntime *when this feature is explicitly enabled* — it is off by # default, so plain `cargo build`/`cargo test` never touch the network. diff --git a/crates/opentake-media/src/decode/mod.rs b/crates/opentake-media/src/decode/mod.rs index 34cbdec5..f30ee4d1 100644 --- a/crates/opentake-media/src/decode/mod.rs +++ b/crates/opentake-media/src/decode/mod.rs @@ -12,8 +12,8 @@ pub use frame::{ fit_within, FrameRequest, }; pub use pcm::{ - extract_pcm, extract_pcm_cancellable, extract_pcm_cancellable_with_progress, PcmBuffer, - PcmFormat, PcmProgressCallback, PcmSpec, + extract_pcm, extract_pcm_cancellable, extract_pcm_cancellable_with_progress, + extract_pcm_chunk, PcmBuffer, PcmChunk, PcmFormat, PcmProgressCallback, PcmSpec, }; pub use stream::{ spawn_video_stream, StreamDecodeControl, StreamVideoFrame, VideoStream, VideoStreamRequest, diff --git a/crates/opentake-media/src/decode/pcm.rs b/crates/opentake-media/src/decode/pcm.rs index 8a3b5d6e..df532dbd 100644 --- a/crates/opentake-media/src/decode/pcm.rs +++ b/crates/opentake-media/src/decode/pcm.rs @@ -489,6 +489,145 @@ pub(super) fn decode_raw_pcm_cancellable( validate_pcm_output(path, status, stdout, stderr, reader_cap) } +/// Decoded PCM chunk for streaming playback (#160). Unlike [`PcmBuffer`] (which +/// averages channels to mono for transcription/waveform), `PcmChunk` PRESERVES +/// the multi-channel layout as interleaved f32 — stereo stays stereo — so the +/// Web Audio API can re-wrap it as an `AudioBuffer` without re-decoding. +/// +/// `samples` is interleaved: `frame f, channel c` lives at +/// `samples[f * channels + c]`. `frame_count` is the per-channel sample count +/// (so `samples.len() == frame_count * channels as usize`). +#[derive(Clone, Debug, PartialEq)] +pub struct PcmChunk { + /// Interleaved f32 samples (length = `frame_count * channels`). + pub samples: Vec, + pub sample_rate: u32, + pub channels: u16, + /// Per-channel sample count (independent of channel count). + pub frame_count: usize, +} + +impl PcmChunk { + /// Duration in seconds implied by `frame_count` and `sample_rate`. + pub fn duration_secs(&self) -> f64 { + if self.sample_rate == 0 { + return 0.0; + } + self.frame_count as f64 / self.sample_rate as f64 + } +} + +/// Build the ffmpeg arg list for a chunked seek+duration decode (`-ss`/`-t`) +/// to raw f32le on stdout. Mirrors [`pcm_args`] but uses `-t` (duration) instead +/// of `-to` (end), matching the streaming-chunk pattern: seek to `start_time_sec` +/// and read exactly `duration_sec` more. +fn pcm_chunk_args( + path: &Path, + sample_rate: u32, + channels: u16, + start_time_sec: f64, + duration_sec: f64, +) -> Vec { + let mut args: Vec = Vec::new(); + // Seek BEFORE `-i` for fast keyframe-accurate seek (ffmpeg's input-seeking + // path), which is what chunked streaming wants: it skips the decode cost + // for the discarded prefix. Same placement as `pcm_args`'s `-ss`. + args.push("-ss".into()); + args.push(format!("{:.6}", start_time_sec.max(0.0))); + args.push("-t".into()); + args.push(format!("{:.6}", duration_sec.max(0.0))); + args.push("-i".into()); + args.push(path.to_string_lossy().into_owned()); + args.push("-vn".into()); // drop video + args.push("-ac".into()); + args.push(channels.to_string()); + args.push("-ar".into()); + args.push(sample_rate.to_string()); + args.push("-f".into()); + args.push("f32le".into()); // chunk pipeline is f32-only (no s16 path) + args.push("-".into()); + args +} + +/// Convert interleaved raw f32le bytes to interleaved f32 samples. Channels +/// are preserved (no mono averaging) — the chunk pipeline keeps stereo. +fn raw_f32_to_interleaved(bytes: &[u8], channels: u16) -> (Vec, usize) { + let ch = channels.max(1) as usize; + let total_samples = bytes.len() / 4; + let frame_count = total_samples / ch; + let mut out = Vec::with_capacity(total_samples); + for i in 0..total_samples { + let off = i * 4; + out.push(f32::from_le_bytes([ + bytes[off], + bytes[off + 1], + bytes[off + 2], + bytes[off + 3], + ])); + } + (out, frame_count) +} + +/// Decode a chunk of `media_path`'s first audio track to interleaved f32, +/// preserving the multi-channel layout (stereo stays stereo). `start_time_sec` +/// is the seek position; `duration_sec` is the chunk length. Defaults match the +/// streaming pipeline: 48 kHz, stereo, 5 s chunks (#160). +/// +/// Uses ffmpeg with `-ss` (input seek) and `-t` (duration) for efficient +/// chunked extraction — same spawn/read/wait/error pattern as [`extract_pcm`]. +/// Errors with `NoTrack("audio", …)` when the file has no audio stream. +pub fn extract_pcm_chunk( + media_path: &str, + start_time_sec: f64, + duration_sec: f64, + target_sample_rate: u32, + channels: u16, +) -> Result { + let path = Path::new(media_path); + // Cheap guard: confirm an audio track exists before spawning the decoder. + if let Ok(p) = probe::probe(path) { + if !p.has_audio { + return Err(MediaError::no_track("audio", path)); + } + } + + let mut child = ff::ffmpeg() + .args(pcm_chunk_args( + path, + target_sample_rate, + channels, + start_time_sec, + duration_sec, + )) + .spawn() + .map_err(|e| MediaError::Ffmpeg(format!("spawn: {e}")))?; + + // Read raw f32le straight off stdout. + let mut raw = Vec::new(); + if let Some(mut stdout) = child.take_stdout() { + stdout + .read_to_end(&mut raw) + .map_err(|e| MediaError::Ffmpeg(format!("read stdout: {e}")))?; + } + let status = child.wait().map_err(MediaError::Io)?; + if !status.success() && raw.is_empty() { + return Err(MediaError::no_track("audio", path)); + } + // ffmpeg can exit 0 with empty stdout when metadata says audio exists but + // no decodable samples: treat as no audio track (same guard as `extract_pcm`). + if raw.is_empty() { + return Err(MediaError::no_track("audio", path)); + } + + let (samples, frame_count) = raw_f32_to_interleaved(&raw, channels); + Ok(PcmChunk { + samples, + sample_rate: target_sample_rate, + channels, + frame_count, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -797,6 +936,83 @@ mod tests { assert!(error.to_string().contains("decoder failed")); } + #[test] + fn pcm_chunk_duration_from_interleaved_stereo() { + // 2ch, 48k, 2s → frame_count = 96_000, samples.len = 192_000. + let c = PcmChunk { + samples: vec![0.0; 192_000], + sample_rate: 48_000, + channels: 2, + frame_count: 96_000, + }; + assert!((c.duration_secs() - 2.0).abs() < 1e-9); + } + + #[test] + fn pcm_chunk_duration_handles_zero_rate() { + let c = PcmChunk { + samples: vec![0.0; 100], + sample_rate: 0, + channels: 2, + frame_count: 50, + }; + assert_eq!(c.duration_secs(), 0.0); + } + + #[test] + fn pcm_chunk_args_emits_ss_and_t() { + let args = pcm_chunk_args(Path::new("/a.mp4"), 48_000, 2, 1.5, 5.0); + let ss = args.iter().position(|a| a == "-ss").unwrap(); + assert_eq!(args[ss + 1], "1.500000"); + let t = args.iter().position(|a| a == "-t").unwrap(); + assert_eq!(args[t + 1], "5.000000"); + assert!(args.windows(2).any(|w| w == ["-ar", "48000"])); + assert!(args.windows(2).any(|w| w == ["-ac", "2"])); + assert!(args.windows(2).any(|w| w == ["-f", "f32le"])); + assert!(args.iter().any(|a| a == "-vn")); + assert!(!args.iter().any(|a| a == "-to")); + } + + #[test] + fn pcm_chunk_args_clamps_negative_seek() { + let args = pcm_chunk_args(Path::new("/a.mp4"), 48_000, 2, -1.0, 5.0); + let ss = args.iter().position(|a| a == "-ss").unwrap(); + assert_eq!(args[ss + 1], "0.000000"); + } + + #[test] + fn raw_f32_interleaved_preserves_stereo_channels() { + // frame0: L=1.0 R=0.0 ; frame1: L=-0.5 R=0.5 → no averaging. + let mut bytes = Vec::new(); + for v in [1.0f32, 0.0, -0.5, 0.5] { + bytes.extend_from_slice(&v.to_le_bytes()); + } + let (samples, frames) = raw_f32_to_interleaved(&bytes, 2); + assert_eq!(samples.len(), 4); + assert_eq!(frames, 2); + assert_eq!(samples, vec![1.0, 0.0, -0.5, 0.5]); + } + + #[test] + fn raw_f32_interleaved_mono_passthrough() { + let mut bytes = Vec::new(); + for v in [0.25f32, -0.75, 0.5] { + bytes.extend_from_slice(&v.to_le_bytes()); + } + let (samples, frames) = raw_f32_to_interleaved(&bytes, 1); + assert_eq!(samples, vec![0.25, -0.75, 0.5]); + assert_eq!(frames, 3); + } + + #[test] + fn raw_f32_interleaved_partial_trailing_frame_ignored() { + // 9 bytes = 2 full f32 samples + 1 stray byte → 2 samples. + let bytes = [0u8, 0, 0, 0, 0, 0, 0x80, 0x3f, 7]; + let (samples, frames) = raw_f32_to_interleaved(&bytes, 1); + assert_eq!(samples.len(), 2); + assert_eq!(frames, 2); + } + #[test] fn duration_from_mono_samples() { let b = PcmBuffer { diff --git a/crates/opentake-media/src/ff.rs b/crates/opentake-media/src/ff.rs index 4bfcb91b..da5c2041 100644 --- a/crates/opentake-media/src/ff.rs +++ b/crates/opentake-media/src/ff.rs @@ -50,6 +50,63 @@ pub fn ffprobe_available() -> bool { .unwrap_or(false) } +/// Ensure ffmpeg/ffprobe are available, auto-downloading them when the +/// `ffmpeg-download` feature is enabled and the binaries aren't on PATH. +/// +/// After a successful download the binaries land adjacent to the current +/// executable (ffmpeg-sidecar's `sidecar_dir`), so we set the +/// `OPENTAKE_FFMPEG` / `OPENTAKE_FFPROBE` env overrides to point at them — +/// [`ffmpeg_path`] / [`ffprobe_path`] read those overrides, so the rest of the +/// media layer finds the binaries without each call site changing. +/// +/// When the feature is disabled (the default for offline library builds), this +/// returns an error if the binaries are missing, so callers can log a clear +/// message rather than failing silently on the first decode. +pub fn ensure_ffmpeg() -> crate::error::Result<()> { + if ffmpeg_available() && ffprobe_available() { + return Ok(()); + } + + #[cfg(feature = "ffmpeg-download")] + { + // `auto_download` internally checks `ffmpeg_is_installed` (adjacent to + // the exe, then PATH) and short-circuits, so calling it when ffmpeg IS + // available is a no-op. On a fresh install it fetches + unpacks the + // platform release into `sidecar_dir` (next to the app binary). + ffmpeg_sidecar::download::auto_download() + .map_err(|e| crate::error::MediaError::Ffmpeg(format!("auto-download: {e}")))?; + + // Point our env overrides at the freshly downloaded binaries so + // `ffmpeg_path()` / `ffprobe_path()` resolve to them on every call. + let ff = ffmpeg_sidecar::paths::ffmpeg_path(); + if ff.is_file() { + std::env::set_var("OPENTAKE_FFMPEG", &ff); + } + // ffprobe sits in the same directory as ffmpeg. + if let Some(parent) = ff.parent() { + let probe = parent.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" }); + if probe.is_file() { + std::env::set_var("OPENTAKE_FFPROBE", &probe); + } + } + + if ffmpeg_available() { + return Ok(()); + } + return Err(crate::error::MediaError::Ffmpeg( + "ffmpeg still unavailable after auto-download".into(), + )); + } + + #[cfg(not(feature = "ffmpeg-download"))] + { + Err(crate::error::MediaError::Ffmpeg( + "ffmpeg not found on PATH; enable the `ffmpeg-download` feature for auto-download" + .into(), + )) + } +} + /// Run `ffprobe -of json -show_streams -show_format ` and return parsed /// JSON. Zero decoding — header/stream parameters only. pub fn ffprobe_json(path: &std::path::Path) -> crate::error::Result { diff --git a/crates/opentake-media/src/lib.rs b/crates/opentake-media/src/lib.rs index 762bcd73..68cd5362 100644 --- a/crates/opentake-media/src/lib.rs +++ b/crates/opentake-media/src/lib.rs @@ -56,9 +56,10 @@ pub use probe::{probe, MediaProbe}; pub use decode::{ decode_frame_at, decode_frame_at_cancellable, decode_frames_at, decode_frames_at_cancellable, decode_pcm_interleaved, decode_pcm_interleaved_cancellable, extract_pcm, - extract_pcm_cancellable, extract_pcm_cancellable_with_progress, FrameRequest, PcmBuffer, - PcmFormat, PcmProgressCallback, PcmSpec, StreamDecodeControl, StreamVideoFrame, VideoStream, - VideoStreamRequest, DEFAULT_VIDEO_STREAM_QUEUE_CAPACITY, + extract_pcm_cancellable, extract_pcm_cancellable_with_progress, extract_pcm_chunk, + FrameRequest, PcmBuffer, PcmChunk, PcmFormat, PcmProgressCallback, PcmSpec, + StreamDecodeControl, StreamVideoFrame, VideoStream, VideoStreamRequest, + DEFAULT_VIDEO_STREAM_QUEUE_CAPACITY, }; pub use encode::{ExportPreset, ExportResolution, VideoCodec, VideoEncoder}; @@ -107,7 +108,7 @@ pub use ort_worker::ExecutionProvider; /// ffmpeg/ffprobe availability probes (re-exported for integration tests and /// host-capability checks). pub mod ffmpeg_status { - pub use crate::ff::{ffmpeg_available, ffprobe_available}; + pub use crate::ff::{ensure_ffmpeg, ffmpeg_available, ffprobe_available}; } /// Facade bundling the media engine's roots for `opentake-core` (SPEC §8.4). diff --git a/crates/opentake-motion/Cargo.toml b/crates/opentake-motion/Cargo.toml index 3d206684..7439c106 100644 --- a/crates/opentake-motion/Cargo.toml +++ b/crates/opentake-motion/Cargo.toml @@ -32,4 +32,18 @@ default = [] # Gates the real headless-Chromium (CDP) backend behind a feature so neither the # default build nor CI tests require a Chromium binary. The skeleton compiles # unconditionally; only the live CDP wiring is feature-gated (see renderer.rs). +# +# When the CDP integration (Issue #14) lands, this feature will pull in a CDP +# client crate, e.g.: +# chromium = ["dep:chromiumoxide"] +# [dependencies.chromiumoxide] +# version = "0.7" +# default-features = false +# features = ["tokio-runtime"] # or "async-std-runtime" +# optional = true +# +# Until then the feature is empty and `HeadlessChromiumRenderer::render` returns +# `MotionError::RendererUnavailable`. `HeadlessChromiumRenderer::chrome_available` +# provides a best-effort runtime binary check so the dispatch layer can produce a +# precise "Chrome not found" message and fall back to `StubRenderer`. chromium = [] diff --git a/crates/opentake-motion/src/renderer.rs b/crates/opentake-motion/src/renderer.rs index d7c92d5b..a6d3f14d 100644 --- a/crates/opentake-motion/src/renderer.rs +++ b/crates/opentake-motion/src/renderer.rs @@ -341,6 +341,93 @@ impl HeadlessChromiumRenderer { .map(|i| i as f64 / req.fps as f64) .collect() } + + /// Best-effort check for a Chrome/Chromium binary on the system PATH and a + /// handful of well-known install locations. Returns `true` when a candidate + /// binary is found, `false` otherwise. + /// + /// The dispatch layer (e.g. the Tauri `render_motion_clip` command) calls + /// this to decide whether to attempt the CDP backend or fall back to + /// [`StubRenderer`]. Because the live CDP wiring is not yet implemented + /// (Issue #14 TODO), this returning `true` does NOT mean a render will + /// succeed — [`render`] still returns [`MotionError::RendererUnavailable`] + /// until the `chromium` feature lands its CDP client. The check exists so + /// the dispatch can produce a precise "Chrome not found" error message + /// instead of the generic "backend not implemented" one. + /// + /// Pure-ish (reads env + stats files); does not spawn a process. + pub fn chrome_available() -> bool { + chrome_binary_path().is_some() + } +} + +/// Locate a Chrome/Chromium binary by scanning the `PATH` environment variable +/// and a small set of well-known per-OS install locations. Returns the first +/// candidate path that exists as a file, or `None`. +/// +/// This is intentionally lightweight (no version probe, no launch). The real +/// CDP integration (#14) will replace it with a proper binary locator + +/// version check (likely via `chromiumoxide`'s own discovery). +fn chrome_binary_path() -> Option { + use std::path::PathBuf; + + // Candidate binary names per OS. + let names: &[&str] = if cfg!(target_os = "windows") { + &["chrome.exe", "chromium.exe"] + } else if cfg!(target_os = "macos") { + &["chrome", "chromium", "Google Chrome", "Chromium"] + } else { + &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + ] + }; + + // 1. PATH lookup. + if let Some(path) = std::env::var_os("PATH") { + for dir in std::env::split_paths(&path) { + for name in names { + let candidate = dir.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + } + } + + // 2. Well-known install locations (outside PATH). + let known: &[&str] = if cfg!(target_os = "macos") { + &[ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ] + } else if cfg!(target_os = "windows") { + &[ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] + } else { + &[ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/usr/local/bin/chromium", + "/opt/google/chrome/chrome", + "/snap/bin/chromium", + ] + }; + for p in known { + let candidate = PathBuf::from(p); + if candidate.is_file() { + return Some(candidate); + } + } + + None } impl MotionRenderer for HeadlessChromiumRenderer { diff --git a/crates/opentake-ops/src/command.rs b/crates/opentake-ops/src/command.rs index 823601b0..c57c86db 100644 --- a/crates/opentake-ops/src/command.rs +++ b/crates/opentake-ops/src/command.rs @@ -392,6 +392,10 @@ pub enum EditCommand { hidden: Option, sync_locked: Option, }, + /// Toggle the `soloed` flag on one track. When any track is soloed, non-soloed + /// tracks are silenced in the render layer (the data model only stores the + /// flag). 1:1 with the upstream track-header solo toggle. + ToggleSolo { track_index: usize }, /// Create a media-library folder. CreateFolder { name: String, @@ -572,6 +576,7 @@ pub fn apply( hidden, sync_locked, } => set_track_props(state, track_index, muted, hidden, sync_locked), + EditCommand::ToggleSolo { track_index } => toggle_solo_cmd(state, track_index), EditCommand::CreateFolder { name, parent_folder_id, @@ -821,6 +826,28 @@ fn swap_tracks(state: &mut EditorState, a: usize, b: usize) -> Result Result { + if track_index >= state.timeline.tracks.len() { + return Err(EditError::Invalid(format!( + "trackIndex {track_index} out of range" + ))); + } + transact( + state, + "Toggle Solo", + |_| "Toggled track solo".to_string(), + |st| { + let new_val = ops::toggle_solo(&mut st.timeline, track_index); + // Summary reflects the resulting state so the caller can echo it. + let _ = new_val; + Ok(Vec::new()) + }, + ) +} + /// Swap the positions of two clips. The op refuses (leaves the timeline /// untouched) when the swap would overlap a third clip; `transact` then reports /// `changed = false`, so a refused swap is a clean no-op with no undo entry. @@ -841,7 +868,6 @@ fn swap_clips(state: &mut EditorState, a: String, b: String) -> Result bool { + let track = &mut timeline.tracks[track_index]; + track.soloed = !track.soloed; + track.soloed +} + /// First audio track free over `[start_frame, start_frame + duration)`, else /// `None`. 1:1 port of `availableAudioTrackIndex(startFrame:duration:)`. pub fn available_audio_track_index( @@ -194,4 +203,20 @@ mod tests { assert_eq!(n, 1); assert_eq!(tl.tracks.len(), 1); } + + #[test] + fn toggle_solo_flips_track_soloed() { + let mut tl = tl_v_a(); + assert!(!tl.tracks[0].soloed); + // Toggle on. + let new_val = toggle_solo(&mut tl, 0); + assert!(new_val); + assert!(tl.tracks[0].soloed); + // Toggle back off. + let new_val = toggle_solo(&mut tl, 0); + assert!(!new_val); + assert!(!tl.tracks[0].soloed); + // Other tracks are unaffected. + assert!(!tl.tracks[1].soloed); + } } diff --git a/crates/opentake-ops/tests/command_apply.rs b/crates/opentake-ops/tests/command_apply.rs index f84f8099..c9d9fccd 100644 --- a/crates/opentake-ops/tests/command_apply.rs +++ b/crates/opentake-ops/tests/command_apply.rs @@ -972,6 +972,61 @@ fn swap_tracks_cross_type_is_noop_without_undo_entry() { ); } +// ---- toggle_solo ---------------------------------------------------------- + +#[test] +fn toggle_solo_toggles_track_soloed() { + let mut st = state(vec![ + video_track("v0", true, vec![clip("a", 0, 30)]), + audio_track("a0", true, vec![clip("b", 0, 30)]), + ]); + let g = SeqIdGen::default(); + assert!(!st.timeline.tracks[0].soloed); + + // Toggle on -> track 0 soloed, track 1 untouched. + let res = apply( + &mut st, + EditCommand::ToggleSolo { track_index: 0 }, + &g, + ) + .unwrap(); + assert!(res.changed); + assert_eq!(res.action_name, "Toggle Solo"); + assert_eq!(res.timeline_version, 1); + assert!(st.timeline.tracks[0].soloed); + assert!(!st.timeline.tracks[1].soloed); + + // Toggle back off -> soloed cleared, version bumped again. + let res = apply( + &mut st, + EditCommand::ToggleSolo { track_index: 0 }, + &g, + ) + .unwrap(); + assert!(res.changed); + assert_eq!(res.timeline_version, 2); + assert!(!st.timeline.tracks[0].soloed); + + // Undo restores the soloed state. + apply(&mut st, EditCommand::Undo, &g).unwrap(); + assert!(st.timeline.tracks[0].soloed); +} + +#[test] +fn toggle_solo_rejects_unknown_track() { + let mut st = state(vec![video_track("v0", true, vec![clip("a", 0, 30)])]); + let g = SeqIdGen::default(); + let err = apply( + &mut st, + EditCommand::ToggleSolo { track_index: 5 }, + &g, + ) + .unwrap_err(); + assert!(matches!(err, EditError::Invalid(_))); + assert_eq!(st.version(), 0); // unchanged + assert!(!st.timeline.tracks[0].soloed); +} + // ---- no-change command ---------------------------------------------------- #[test] diff --git a/crates/opentake-render/src/gpu/compositor.rs b/crates/opentake-render/src/gpu/compositor.rs index 7790111d..1eb9b9b6 100644 --- a/crates/opentake-render/src/gpu/compositor.rs +++ b/crates/opentake-render/src/gpu/compositor.rs @@ -271,20 +271,45 @@ impl Compositor { } } - /// Render one frame to an offscreen RGBA8 target and read it back. + /// Render one frame to an offscreen RGBA8 target and read it back + /// synchronously. Convenience wrapper around [`Self::begin_render`] + + /// [`PendingReadback::finish_blocking`] for the single-frame preview path + /// (paused / scrubbing) where the caller must get the frame immediately. + /// + /// Streaming/playback callers should use [`Self::begin_render`] directly + /// with [`PendingReadback::try_finish`] for non-blocking double-buffered + /// readback (Issue #202): while frame N renders on the GPU, frame N-1's + /// readback completes asynchronously, avoiding the per-frame + /// `device.poll(Maintain::Wait)` stall that capped playback at ~25.5fps. + pub fn render_to_rgba( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + size: RenderSize, + frame_plan: &FramePlan<'_>, + resolver: &mut dyn TextureResolver, + ) -> Result { + let pending = self.begin_render(device, queue, size, frame_plan, resolver)?; + pending.finish_blocking(device) + } + + /// Encode + submit one frame's render pass and return a [`PendingReadback`] + /// holding the staging buffer. The readback is NOT finished — the caller + /// finishes it via [`PendingReadback::finish_blocking`] (sync, preview) or + /// [`PendingReadback::try_finish`] (non-blocking, streaming). /// /// Clears to `frame_plan.clear_rgba` (opaque black), then composites each /// draw in order (later = on top). Draws whose texture can't be resolved are /// skipped (offline/unprocessable sources contribute nothing, mirroring /// upstream's offline handling). - pub fn render_to_rgba( + pub fn begin_render( &self, device: &wgpu::Device, queue: &wgpu::Queue, size: RenderSize, frame_plan: &FramePlan<'_>, resolver: &mut dyn TextureResolver, - ) -> Result { + ) -> Result { let rt = device.create_texture(&wgpu::TextureDescriptor { label: Some("opentake-render target"), size: wgpu::Extent3d { @@ -440,41 +465,130 @@ impl Compositor { } } - let frame = read_back(device, queue, &mut encoder, &rt, size)?; + let pending = read_back(device, queue, &mut encoder, &rt, size)?; queue.submit(Some(encoder.finish())); - // `read_back` mapped the staging buffer after submit via poll; finalize. - frame.finish(device) + Ok(pending) } } -/// Holds the staging buffer until its contents are mapped and copied out. -struct PendingReadback { +/// Holds the staging buffer for one frame's GPU readback. Created by +/// [`Compositor::begin_render`] after the render pass is encoded and submitted; +/// finished by one of two paths: +/// +/// - [`PendingReadback::finish_blocking`] — synchronous, polls the device with +/// `Maintain::Wait` until the readback completes. Use for the single-frame +/// preview path (paused / scrubbing) where the caller must get the frame. +/// - [`PendingReadback::try_finish`] — non-blocking, polls with `Maintain::Check` +/// and returns `Ok(None)` when the GPU hasn't finished yet. Use for the +/// streaming / playback path. +/// +/// # Double-buffered readback (Issue #202) +/// +/// The previous `finish` did a per-frame `device.poll(Maintain::Wait)`, blocking +/// the CPU on the GPU every frame and capping playback at ~25.5fps. The async +/// split below lets the caller overlap frame N's render with frame N-1's +/// readback: while frame N renders on the GPU, frame N-1's `map_async` callback +/// fires and `try_finish` harvests it without a blocking wait. +/// +/// Caller-side ring (one slot is enough — the GPU serializes work anyway): +/// +/// ```text +/// let mut prev = compositor.begin_render(frame_0, ...)?; // submit frame 0 +/// for f in 1..n { +/// let cur = compositor.begin_render(frame_f, ...)?; // submit frame N +/// match prev.try_finish(device)? { // finish N-1 (non-blocking) +/// Some(frame) => display(frame), +/// None => redisplay_previous(), // GPU not ready: reuse last +/// } +/// prev = cur; +/// } +/// let last = prev.finish_blocking(device)?; // drain final frame +/// ``` +pub struct PendingReadback { buffer: wgpu::Buffer, size: RenderSize, padded_bytes_per_row: u32, + /// Sender captured by the `map_async` callback; `None` once the map request + /// has been issued (`map_async` may only be called once per mapping, so the + /// first finish call — blocking or non-blocking — owns it). + tx: Option>>, + /// Receiver signaled by the `map_async` callback when the buffer is mapped. + rx: std::sync::mpsc::Receiver>, } impl PendingReadback { - fn finish(self, device: &wgpu::Device) -> Result { - let slice = self.buffer.slice(..); - let (tx, rx) = std::sync::mpsc::channel(); - slice.map_async(wgpu::MapMode::Read, move |res| { - let _ = tx.send(res); - }); + /// Issue the async map request if it hasn't been issued yet. Idempotent: + /// safe to call from both finish paths. `map_async` registers a callback + /// that fires once the GPU completes the `copy_texture_to_buffer` and the + /// buffer becomes mappable; the device must be polled to make progress. + fn issue_map(&mut self) { + if let Some(tx) = self.tx.take() { + let slice = self.buffer.slice(..); + slice.map_async(wgpu::MapMode::Read, move |res| { + let _ = tx.send(res); + }); + } + } + + /// Blocking finish: issue `map_async` then poll the device with + /// `Maintain::Wait` until the GPU completes the readback, and extract the + /// RGBA pixels. Use for the single-frame preview path where the caller must + /// get the frame immediately. This is the synchronous fallback for the + /// paused/scrub path (and for draining the last frame of a stream). + pub fn finish_blocking(mut self, device: &wgpu::Device) -> Result { + self.issue_map(); + // `Maintain::Wait` is the efficient single-frame wait on native backends + // (Metal/DX/Vulkan): it blocks the thread without busy-spinning. The + // streaming path uses `try_finish` (`Maintain::Check`) instead. device.poll(wgpu::Maintain::Wait); - rx.recv() + self.rx + .recv() .map_err(|_| RenderError::Readback("map channel closed".into()))? .map_err(|e| RenderError::Readback(e.to_string()))?; + self.extract_and_unmap() + } - let data = slice.get_mapped_range(); - let row_bytes = self.size.width as usize * 4; - let mut rgba = vec![0u8; row_bytes * self.size.height as usize]; - for y in 0..self.size.height as usize { - let src = y * self.padded_bytes_per_row as usize; - let dst = y * row_bytes; - rgba[dst..dst + row_bytes].copy_from_slice(&data[src..src + row_bytes]); + /// Non-blocking finish: issue `map_async` (once) and poll the device with + /// `Maintain::Check` (no wait). Returns `Ok(None)` when the GPU hasn't + /// finished the readback yet — the caller should reuse the previous frame. + /// The map request is issued on the first call; subsequent calls re-poll + /// without re-issuing. Once the readback is ready the buffer is unmapped and + /// the frame returned; the `PendingReadback` should then be dropped. + pub fn try_finish(&mut self, device: &wgpu::Device) -> Result, RenderError> { + self.issue_map(); + // `Maintain::Poll` makes one non-blocking poll pass: it processes + // completed GPU work (firing the map_async callback if the copy is done) + // without stalling the CPU. This is the key to the double-buffered + // overlap — the host stays free to prepare the next frame. (wgpu 23 + // names the non-blocking variant `Poll`, not `Check`.) + device.poll(wgpu::Maintain::Poll); + match self.rx.try_recv() { + Ok(Ok(())) => { + // Take the frame out without consuming `self` so the caller can + // keep the slot alive across retries. The buffer is unmapped + // here; a subsequent `try_finish` would find the channel empty. + let frame = self.extract_and_unmap()?; + Ok(Some(frame)) + } + Ok(Err(e)) => Err(RenderError::Readback(e.to_string())), + Err(std::sync::mpsc::TryRecvError::Empty) => Ok(None), + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + Err(RenderError::Readback("map channel closed".into())) + } } - drop(data); + } + + /// Copy the mapped staging rows into a tight RGBA buffer and unmap. The + /// caller must have already waited for `map_async` to complete (the channel + /// has yielded `Ok(())`). + fn extract_and_unmap(&self) -> Result { + // Scope the slice + mapped view so their borrows release before `unmap` + // (wgpu requires the buffer not be borrowed when `unmap` is called). + let rgba = { + let slice = self.buffer.slice(..); + let data = slice.get_mapped_range(); + extract_rgba(&data, self.size, self.padded_bytes_per_row) + }; self.buffer.unmap(); Ok(DecodedFrame::new( self.size.width, @@ -486,8 +600,21 @@ impl PendingReadback { } } +/// Copy 256-byte-aligned staging rows into a tight `width * 4` RGBA buffer. +fn extract_rgba(data: &[u8], size: RenderSize, padded_bytes_per_row: u32) -> Vec { + let row_bytes = size.width as usize * 4; + let mut rgba = vec![0u8; row_bytes * size.height as usize]; + for y in 0..size.height as usize { + let src = y * padded_bytes_per_row as usize; + let dst = y * row_bytes; + rgba[dst..dst + row_bytes].copy_from_slice(&data[src..src + row_bytes]); + } + rgba +} + /// Encode the RT -> buffer copy (256-aligned rows) and return a pending readback -/// to be finalized after `queue.submit`. +/// to be finalized after `queue.submit`. The staging buffer is created unmapped; +/// the `map_async` request is issued lazily by the first finish call. fn read_back( device: &wgpu::Device, _queue: &wgpu::Queue, @@ -525,9 +652,12 @@ fn read_back( depth_or_array_layers: 1, }, ); + let (tx, rx) = std::sync::mpsc::channel(); Ok(PendingReadback { buffer, size, padded_bytes_per_row: padded, + tx: Some(tx), + rx, }) } diff --git a/crates/opentake-render/src/gpu/lottie.rs b/crates/opentake-render/src/gpu/lottie.rs new file mode 100644 index 00000000..946bbc04 --- /dev/null +++ b/crates/opentake-render/src/gpu/lottie.rs @@ -0,0 +1,123 @@ +//! Lottie rasterization interface (Issue #65). Upstream bakes Lottie +//! (Bodymovin JSON) animations into an intermediate video via the +//! CoreAnimationTool; OpenTake rasterizes each Lottie clip to a sequence of +//! premultiplied-RGBA textures that composite like any other layer. +//! +//! This module defines the trait boundary + a null implementation that returns +//! `None` (never `todo!()` / `unimplemented!()`), so the compositor can route +//! Lottie clips and tests never trip an unimplemented panic. A real backend +//! (e.g. a `rlottie`/`vello` wrapper, or the `opentake-motion` crate's +//! `MotionClipSource` exposed as a `FrameProvider`) implements this trait and is +//! injected into the production `TextureResolver`. +//! +//! ## Integration points +//! +//! - The plan builder routes `ClipType::Lottie` → [`TextureSource::Lottie`] +//! (see `plan/build.rs::texture_source_for`). +//! - `source_frame_index` maps the timeline frame to a Lottie internal frame +//! (modulo `lottie_frame_count`, see `plan/build.rs`). +//! - The production resolver (`src-tauri/render.rs::MediaResolver`) calls the +//! injected `LottieRasterizer` for `TextureSource::Lottie`, uploading the +//! returned [`DecodedFrame`] as a texture (same path as image/text). +//! +//! ## Why a separate trait (not `FrameProvider::lottie_frame`)? +//! +//! `FrameProvider` is the contract for already-decoded sources (video/image +//! frames living in a codec). Lottie baking is a *rasterization* step (JSON → +//! pixels), mirroring `TextRasterizer` (style → pixels): both turn non-pixel +//! clip data into a texture on demand. Keeping the trait separate also lets the +//! resolver cache by `(media_ref, frame)` without entangling codec frame +//! providers, exactly as it caches text by `clip_id`. + +use crate::source::DecodedFrame; + +/// Inputs needed to rasterize one Lottie frame. +/// +/// `canvas` is the compositor's preview render size; a rasterizer may downscale +/// the Lottie's intrinsic composition to fit it (matching how text/image layers +/// respect the preview cap). The returned [`DecodedFrame`] carries its own +/// width/height so the uploader is size-agnostic. +#[derive(Clone, PartialEq, Debug)] +pub struct LottieRasterRequest<'a> { + /// The Lottie asset ref (resolves to a `.json`/`.lottie` file in the + /// caller's media manifest). + pub media_ref: &'a str, + /// Lottie internal frame index (already wrapped to `[0, frame_count)` by + /// `source_frame_index` when `lottie_frame_count` is known). + pub frame: i64, + /// Compositor canvas size — the rasterizer may cap the output to this size + /// to bound CPU/RAM (same rationale as the preview cap for video). + pub canvas: (u32, u32), +} + +/// Rasterizes one Lottie frame to a premultiplied-RGBA [`DecodedFrame`]. +/// +/// Implementations MUST be deterministic for cacheability: the same +/// `(media_ref, frame, canvas)` yields the same pixels (mirrors the motion +/// crate's content-hash contract). +pub trait LottieRasterizer { + /// Render the request, or `None` if Lottie baking is unavailable in this + /// build (the null backend), the asset is missing/corrupt, or the frame + /// index is out of range. Returning `None` makes the compositor skip the + /// layer (same graceful degradation as a failed video decode or an + /// un-rasterizable text clip). + fn rasterize(&self, request: &LottieRasterRequest<'_>) -> Option; + + /// The Lottie composition's internal frame count, if known. Used by the + /// plan builder to wrap the source-frame index modulo this count (SPEC + /// §4.3). `None` when unknown — the plan then clamps at 0 instead of + /// wrapping. Default `None` so a stub backend compiles without + /// introspecting the JSON. + fn frame_count(&self, _media_ref: &str) -> Option { + None + } +} + +/// Placeholder backend: produces no texture and reports no frame count. Lets +/// the pipeline compile, route Lottie clips, and run end-to-end without a +/// Lottie engine. Replaced by a real backend (rlottie / vello / motion crate) +/// in a later phase — inject it into `MediaResolver` when available. +/// +/// This is the Lottie analogue of [`crate::gpu::NullTextRasterizer`]. +#[derive(Clone, Copy, Debug, Default)] +pub struct NullLottieRasterizer; + +impl LottieRasterizer for NullLottieRasterizer { + fn rasterize(&self, _request: &LottieRasterRequest<'_>) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_rasterizer_returns_none_without_panicking() { + let r = NullLottieRasterizer; + let req = LottieRasterRequest { + media_ref: "anim.json", + frame: 0, + canvas: (1920, 1080), + }; + assert!(r.rasterize(&req).is_none()); + } + + #[test] + fn null_rasterizer_reports_no_frame_count() { + let r = NullLottieRasterizer; + assert_eq!(r.frame_count("anim.json"), None); + } + + #[test] + fn request_carries_media_ref_and_frame() { + let req = LottieRasterRequest { + media_ref: "intro.json", + frame: 12, + canvas: (1280, 720), + }; + assert_eq!(req.media_ref, "intro.json"); + assert_eq!(req.frame, 12); + assert_eq!(req.canvas, (1280, 720)); + } +} diff --git a/crates/opentake-render/src/gpu/mod.rs b/crates/opentake-render/src/gpu/mod.rs index 4b4b11b5..35e3b501 100644 --- a/crates/opentake-render/src/gpu/mod.rs +++ b/crates/opentake-render/src/gpu/mod.rs @@ -8,13 +8,15 @@ pub mod color; pub mod compositor; pub mod device; +pub mod lottie; pub mod text_engine; pub mod text_raster; pub mod texture; pub use color::{linear_to_srgb, srgb_to_linear}; -pub use compositor::{Compositor, TextureResolver}; +pub use compositor::{Compositor, PendingReadback, TextureResolver}; pub use device::RenderDevice; +pub use lottie::{LottieRasterRequest, LottieRasterizer, NullLottieRasterizer}; pub use text_engine::CosmicTextRasterizer; pub use text_raster::{NullTextRasterizer, TextRasterRequest, TextRasterizer}; pub use texture::{upload_rgba, GpuTexture, TextureCache}; diff --git a/crates/opentake-render/src/lib.rs b/crates/opentake-render/src/lib.rs index 893896a4..b96c38ba 100644 --- a/crates/opentake-render/src/lib.rs +++ b/crates/opentake-render/src/lib.rs @@ -21,6 +21,7 @@ pub use size::{even, export_render_size, ExportResolution}; pub use source::{DecodedFrame, FrameProvider, SourceMetrics}; pub use gpu::{ - Compositor, CosmicTextRasterizer, GpuTexture, NullTextRasterizer, RenderDevice, RenderError, + Compositor, CosmicTextRasterizer, GpuTexture, LottieRasterRequest, LottieRasterizer, + NullLottieRasterizer, NullTextRasterizer, PendingReadback, RenderDevice, RenderError, TextRasterRequest, TextRasterizer, TextureCache, TextureResolver, }; diff --git a/docs/visual-validation-checklist.md b/docs/visual-validation-checklist.md new file mode 100644 index 00000000..d7c7eff5 --- /dev/null +++ b/docs/visual-validation-checklist.md @@ -0,0 +1,50 @@ +# OpenTake 视觉验收 Checklist + +## 1. 预览渲染 +- [ ] 暂停态：GPU composite PNG 正确显示 +- [ ] 播放态：`