diff --git a/Cargo.lock b/Cargo.lock index 4d08eb8ddc6..a15ce87bc3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,6 +1830,7 @@ dependencies = [ "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "thiserror 1.0.69", + "uuid", "workspace-hack", ] diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 57e997d9471..3ecf2eff8a2 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -911,6 +911,7 @@ mod fixture { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index bc3f373e1e4..16b033742a8 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -1568,6 +1568,7 @@ dependencies = [ "objc2-foundation 0.2.2", "parakeet-rs", "raw-window-handle", + "relative-path", "reqwest 0.12.28", "resvg 0.45.1", "rfd", @@ -1587,6 +1588,7 @@ dependencies = [ "tracing-subscriber", "tray-icon", "unicode-segmentation", + "uuid", "wayland-client", "wgpu 25.0.2", "whisper-rs", @@ -1764,6 +1766,7 @@ dependencies = [ "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", "thiserror 1.0.69", + "uuid", "workspace-hack", ] diff --git a/apps/desktop-gpui/Cargo.toml b/apps/desktop-gpui/Cargo.toml index 5117c79ab5e..3e410fb1f3c 100644 --- a/apps/desktop-gpui/Cargo.toml +++ b/apps/desktop-gpui/Cargo.toml @@ -106,7 +106,9 @@ ffmpeg = { package = "ffmpeg-next", git = "https://github.com/CapSoftware/rust-f # `webp` is the config sidebar's four background-source illustrations, which # the app ships as webp; features are additive, so enabling it here does not # move gpui's own pin. -image = { version = "0.25.1", default-features = false, features = ["jpeg", "png", "webp"] } +image = { version = "0.25.1", default-features = false, features = ["jpeg", "png", "webp", "gif", "bmp", "tiff"] } +uuid = { version = "1.10.0", features = ["v4"] } +relative-path = "1.9.3" jpeg-decoder = "0.3" smallvec = "1" # Grapheme boundaries for `ui::TextInput`: Backspace deletes a user-perceived diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index 539e0dbffec..1ee0ecfdd5a 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -30,7 +30,7 @@ use crate::{ onboarding_window::{self, OnboardingWindow}, platform, recording::{RecordingMode, StartConfig, StudioFinalization}, - screenshot_editor::{self, ScreenshotEditorWindow}, + screenshot_editor, session::{Phase, RecordingSession, StudioEditorPresentation}, settings_window::{self, Page, SettingsWindow}, target_overlay::{AreaRect, HoveredWindow, OverlayWindow, TargetSelect}, @@ -207,9 +207,6 @@ pub struct AppWindows { pub editors: Vec<(PathBuf, WindowHandle)>, deleting_editors: HashSet, preparing_cleanup: crate::editor_preparing::PreparingCleanupRegistry, - /// One screenshot editor per `.cap` bundle -- the gpui spelling of - /// `ScreenshotEditorWindowIds`, keyed by the bundle directory. - pub screenshot_editors: Vec<(PathBuf, WindowHandle)>, /// `hasHiddenMainWindowForPicker` (`new-main/index.tsx:2016-2059`): the /// main window hides while the target picker is up, and comes back only on /// a dismissal that reveals ("cancelled" -- Escape, the overlay's close @@ -434,15 +431,17 @@ pub(crate) fn export_in_flight(cx: &App) -> bool { let windows = cx.global::(); windows.editors.iter().any(|(_, handle)| { - handle - .read(cx) - .ok() - .and_then(|editor| editor.export.as_ref()) + let Ok(editor) = handle.read(cx) else { + return false; + }; + editor + .export + .as_ref() .is_some_and(|export| export.phase.is_busy()) - }) || windows.screenshot_editors.iter().any(|(_, handle)| { - handle - .read(cx) - .is_ok_and(ScreenshotEditorWindow::export_in_flight) + || editor + .screenshot_workspace + .as_ref() + .is_some_and(|workspace| workspace.read(cx).export_in_flight()) }) } @@ -453,11 +452,6 @@ pub(crate) fn flush_pending_editor_saves(cx: &mut App) -> Result<(), String> { let windows = cx.global::(); let editors: Vec<_> = windows.editors.iter().map(|(_, handle)| *handle).collect(); - let screenshot_editors: Vec<_> = windows - .screenshot_editors - .iter() - .map(|(_, handle)| *handle) - .collect(); for handle in editors { if let Ok(result) = handle.update(cx, |editor, _, cx| editor.flush_pending_saves(cx)) { @@ -465,11 +459,6 @@ pub(crate) fn flush_pending_editor_saves(cx: &mut App) -> Result<(), String> { } } - for handle in screenshot_editors { - if let Ok(pending) = handle.update(cx, |editor, _, _| editor.pending_save()) { - pending.borrow_mut().flush(); - } - } Ok(()) } @@ -488,7 +477,6 @@ pub fn init(main: WindowHandle, session: Entity, c editors: Vec::new(), deleting_editors: HashSet::new(), preparing_cleanup: crate::editor_preparing::PreparingCleanupRegistry::default(), - screenshot_editors: Vec::new(), main_hidden_for_picker: false, editor_hidden_for_picker: None, camera_park: CameraPark::default(), @@ -668,11 +656,6 @@ pub fn broadcast_theme(cx: &mut App) { let camera = windows.camera; let overlays: Vec<_> = windows.overlays.iter().map(|(_, handle)| *handle).collect(); let editors: Vec<_> = windows.editors.iter().map(|(_, handle)| *handle).collect(); - let screenshot_editors: Vec<_> = windows - .screenshot_editors - .iter() - .map(|(_, handle)| *handle) - .collect(); let refresh = |window: &mut gpui::Window, cx: &mut gpui::App| { crate::theme::apply_native(window, cx); @@ -725,14 +708,11 @@ pub fn broadcast_theme(cx: &mut App) { }); } for handle in editors { - let _ = handle.update(cx, |_, window, cx| { - refresh(window, cx); - cx.notify(); - }); - } - for handle in screenshot_editors { - let _ = handle.update(cx, |_, window, cx| { + let _ = handle.update(cx, |editor, window, cx| { refresh(window, cx); + if let Some(workspace) = &editor.screenshot_workspace { + workspace.update(cx, |_, cx| cx.notify()); + } cx.notify(); }); } @@ -1011,9 +991,6 @@ pub fn handle_dock_reopen(cx: &mut App) { windows .editors .retain(|(_, handle)| live.contains(&handle.window_id())); - windows - .screenshot_editors - .retain(|(_, handle)| live.contains(&handle.window_id())); windows.settings = windows .settings .filter(|handle| live.contains(&handle.window_id())); @@ -1033,12 +1010,6 @@ pub fn handle_dock_reopen(cx: &mut App) { .editors .iter() .map(|(_, handle)| gpui::AnyWindowHandle::from(*handle)) - .chain( - windows - .screenshot_editors - .iter() - .map(|(_, handle)| (*handle).into()), - ) .chain(windows.settings.map(Into::into)) .collect::>(); let focus = first_registered_reopen_target(candidates, &live); @@ -2498,7 +2469,6 @@ pub enum OwnWindow { Teleprompter, TargetSelect, Editor, - ScreenshotEditor, Onboarding, } @@ -2507,7 +2477,7 @@ impl OwnWindow { /// (`WindowCaptureOccluder`, `CaptureArea`, `RecordingsOverlay`, `Upgrade`, /// `Debug`) have no counterpart in this app; their default rules are still /// honoured for *other* processes by `resolve_excluded_window_ids`. - pub const ALL: [Self; 10] = [ + pub const ALL: [Self; 9] = [ Self::Main, Self::Settings, Self::Controls, @@ -2516,7 +2486,6 @@ impl OwnWindow { Self::Teleprompter, Self::TargetSelect, Self::Editor, - Self::ScreenshotEditor, Self::Onboarding, ]; @@ -2530,7 +2499,6 @@ impl OwnWindow { Self::Teleprompter => "Cap Teleprompter", Self::TargetSelect => "Cap Target Select", Self::Editor => "Cap Editor", - Self::ScreenshotEditor => "Cap Screenshot Editor", Self::Onboarding => "Welcome to Cap", } } @@ -2640,7 +2608,6 @@ fn own_windows(cx: &mut App) -> Vec { editors, deleting_editors: _, preparing_cleanup: _, - screenshot_editors, main_hidden_for_picker: _, editor_hidden_for_picker: _, camera_park: _, @@ -2658,10 +2625,6 @@ fn own_windows(cx: &mut App) -> Vec { let teleprompter = *teleprompter; let overlays: Vec<_> = overlays.iter().map(|(_, handle)| *handle).collect(); let editors: Vec<_> = editors.iter().map(|(_, handle)| *handle).collect(); - let screenshot_editors: Vec<_> = screenshot_editors - .iter() - .map(|(_, handle)| *handle) - .collect(); let mut windows = Vec::new(); windows.extend(probe_own_window(OwnWindow::Main, main, cx)); @@ -2681,9 +2644,6 @@ fn own_windows(cx: &mut App) -> Vec { for handle in editors { windows.extend(probe_own_window(OwnWindow::Editor, handle, cx)); } - for handle in screenshot_editors { - windows.extend(probe_own_window(OwnWindow::ScreenshotEditor, handle, cx)); - } windows } @@ -4716,7 +4676,16 @@ fn open_editor_window( window.refresh(); }).ok(); } - load_editor_project(key, handle, finalization, cx); + let screenshot_workspace = handle + .update(cx, |editor, _window, _cx| { + editor.screenshot_workspace.is_some() + }) + .unwrap_or(false); + if screenshot_workspace { + screenshot_editor::load_screenshot_project_embedded(key, handle, cx); + } else { + load_editor_project(key, handle, finalization, cx); + } Some(handle.window_id()) } @@ -5779,7 +5748,7 @@ fn restore_after_editor_close(key: &Path, cx: &mut App) { } let windows = cx.global::(); - let editors_left = windows.editors.len() + windows.screenshot_editors.len(); + let editors_left = windows.editors.len(); let settings_open = windows.settings.is_some(); tracing::info!( path = %key.display(), @@ -6345,137 +6314,37 @@ pub fn screenshot_finished(captured: Option, cx: &mut App) { main.update(cx, |view, window, cx| view.refresh_open_library(window, cx)) .ok(); - // The editor owns the foreground now, the way a stopped studio recording - // hands off to the video editor; `screenshot_editor_closed` brings the - // main window back. cx.global_mut::().main_hidden_for_picker = false; open_screenshot_editor(png, cx); } -/// Open (or focus) the screenshot editor for a bundle -- the -/// `ShowCapWindow::ScreenshotEditor` arm: 1240x800, min 800x600, centered, -/// reused per path. Accepts the PNG or the `.cap` directory. -/// -/// Must be reached through `cx.defer` from anything inside an entity update: -/// opening a window paints it synchronously and would double-lease the caller. pub fn open_screenshot_editor(path: PathBuf, cx: &mut App) { - #[cfg(target_os = "linux")] - if defer_window_until_capture_safe(cx) { - return; - } let Some(bundle) = screenshot_editor::resolve_bundle(&path) else { tracing::error!(path = %path.display(), "not a screenshot bundle; not opening the editor"); return; }; - let key = editor_key(&bundle); - - if let Some(handle) = cx - .global::() - .screenshot_editors - .iter() - .find(|(existing, _)| existing == &key) - .map(|(_, handle)| *handle) - { - tracing::info!( - path = %key.display(), - "screenshot editor already open for this bundle; focusing it" - ); - let native = handle - .update(cx, |_, window, _| platform::native_window(window)) - .ok() - .flatten(); - cx.spawn(async move |_| { - if let Some(native) = &native { - platform::show_native(native); - } - }) - .detach(); - hide_main_window(cx); - return; - } - - let bounds = opening_window_bounds( - size( - px(screenshot_editor::SCREENSHOT_EDITOR_WIDTH), - px(screenshot_editor::SCREENSHOT_EDITOR_HEIGHT), - ), - cx, - ); - - let handle = cx.open_window( - WindowOptions { - window_bounds: Some(WindowBounds::Windowed(bounds)), - titlebar: Some(gpui::TitlebarOptions { - title: Some("Cap Screenshot Editor".into()), - appears_transparent: true, - traffic_light_position: None, - }), - kind: WindowKind::Normal, - focus: true, - show: true, - is_resizable: true, - is_minimizable: true, - window_min_size: Some(fitted_window_min_size( - size( - px(screenshot_editor::SCREENSHOT_EDITOR_MIN_WIDTH), - px(screenshot_editor::SCREENSHOT_EDITOR_MIN_HEIGHT), - ), - bounds, - )), - ..Default::default() - }, - { - let key = key.clone(); - move |window, cx| cx.new(|cx| ScreenshotEditorWindow::new(key, window, cx)) - }, - ); - - let handle = match handle { - Ok(handle) => handle, - Err(error) => { - tracing::error!("screenshot editor window failed to open: {error:#}"); - return; - } - }; - - cx.global_mut::() - .screenshot_editors - .push((key.clone(), handle)); - handle - .update(cx, |_, window, _| { - platform::kick_display_link(window); - tracing::info!( - number = platform::window_number(window), - path = %key.display(), - "screenshot editor window opened" - ); - }) - .ok(); - - hide_main_window(cx); - screenshot_editor::load_screenshot_project(key, handle, cx); + open_editor(bundle, cx); } -/// The screenshot editor's Delete finished: drop the window (its pending -/// write is for a bundle that no longer exists), refresh every surface that -/// lists screenshots, and run the ordinary closed bookkeeping. -pub fn close_screenshot_editor_after_delete(bundle: &Path, cx: &mut App) { +pub fn close_embedded_screenshot_after_delete(bundle: &Path, cx: &mut App) { let key = editor_key(bundle); let handle = cx .global::() - .screenshot_editors + .editors .iter() .find(|(path, _)| path == &key) .map(|(_, handle)| *handle); if let Some(handle) = handle { - if let Ok(pending) = handle.update(cx, |view, _window, _cx| view.pending_save()) { - pending.borrow_mut().discard(); - } handle - .update(cx, |_, window, _| window.remove_window()) + .update(cx, |editor, window, cx| { + if let Some(workspace) = &editor.screenshot_workspace { + workspace.read(cx).pending_save().borrow_mut().discard(); + } + window.remove_window(); + }) .ok(); + editor_closed(&key, handle.window_id(), cx); } - screenshot_editor_closed(&key, cx); refresh_screenshot_surfaces(cx); } @@ -6491,40 +6360,6 @@ pub fn refresh_screenshot_surfaces(cx: &mut App) { .ok(); } -/// A screenshot editor window is going away: flush its pending config write -/// and bring the main window back once the last editor of either kind closes -/// -- the same `Destroyed` arm `editor_closed` mirrors. -pub fn screenshot_editor_closed(bundle: &Path, cx: &mut App) { - let key = editor_key(bundle); - let handle = { - let editors = &mut cx.global_mut::().screenshot_editors; - let index = editors.iter().position(|(path, _)| path == &key); - index.map(|index| editors.remove(index).1) - }; - - if let Some(handle) = handle - && let Ok(pending) = handle.update(cx, |view, _window, _cx| view.pending_save()) - { - pending.borrow_mut().flush(); - } - - let windows = cx.global::(); - let editors_left = windows.editors.len() + windows.screenshot_editors.len(); - let settings_open = windows.settings.is_some(); - tracing::info!( - path = %key.display(), - editors_left, - settings_open, - "screenshot editor window closed" - ); - let idle = RecordingSession::global(cx).read(cx).phase == Phase::Idle; - if reveal_main_after_editor_close(editors_left, settings_open, idle) { - show_main_window(cx); - } else { - crate::menus::schedule_dock_sync(cx); - } -} - pub fn refresh_library_after_delete(cx: &mut App) { let main = cx.global::().main; let settings = cx.global::().settings; @@ -6706,9 +6541,8 @@ mod tests { #[test] fn dock_reopen_uses_registered_windows_and_falls_back_after_delete() { let editor = WindowHandle::::new(1_u64.into()); - let screenshot = WindowHandle::::new(2_u64.into()); let settings = WindowHandle::::new(3_u64.into()); - let candidates = [editor.into(), screenshot.into(), settings.into()]; + let candidates = [editor.into(), settings.into()]; let mut live = HashSet::from([editor.window_id(), settings.window_id()]); assert_eq!( first_registered_reopen_target(candidates, &live), @@ -7499,7 +7333,6 @@ mod tests { (OwnWindow::Teleprompter, "Cap Teleprompter"), (OwnWindow::TargetSelect, "Cap Target Select"), (OwnWindow::Editor, "Cap Editor"), - (OwnWindow::ScreenshotEditor, "Cap Screenshot Editor"), (OwnWindow::Onboarding, "Welcome to Cap"), ]; assert_eq!(OwnWindow::ALL.len(), expected.len()); @@ -7557,7 +7390,6 @@ mod tests { vec![ OwnWindow::TargetSelect, OwnWindow::Editor, - OwnWindow::ScreenshotEditor, OwnWindow::Onboarding, ] ); diff --git a/apps/desktop-gpui/src/assets.rs b/apps/desktop-gpui/src/assets.rs index 65eabb37589..c774bc26be7 100644 --- a/apps/desktop-gpui/src/assets.rs +++ b/apps/desktop-gpui/src/assets.rs @@ -85,7 +85,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "laptop.svg", "shuffle.svg", "gift.svg", - "history.svg", "hotkeys.svg", "image.svg", "info.svg", @@ -99,7 +98,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "message-square-plus.svg", "mic-off.svg", "microphone.svg", - "minimize.svg", "minus.svg", "monitor.svg", "more-vertical.svg", @@ -189,7 +187,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": // The main window's hand-drawn traffic lights: the x and expand glyphs // `CaptionControlsMacOS.tsx` inlines, shown while the group is hovered. "traffic-close.svg", - "traffic-zoom.svg", // The remaining settings pages (`settings_pages.rs`). `circle-check` is // Cap's own (`packages/ui-solid/icons/circle-check.svg`, hotkeys.tsx's // IconCapCircleCheck); the rest are the Lucide 24x24 originals the pages' @@ -291,6 +288,7 @@ mod tests { /// scan the same list. const ICON_SOURCES: &[&str] = &[ include_str!("main_window.rs"), + include_str!("ui/windows_caption.rs"), // Not a window: the Recents card's per-kind pill and fallback glyphs // are named on `MediaKind`, next to the `Recents.tsx` lines they come // from, so the table has to scan here too. diff --git a/apps/desktop-gpui/src/editor_canvas.rs b/apps/desktop-gpui/src/editor_canvas.rs index 14618f7bea7..7de38092ca9 100644 --- a/apps/desktop-gpui/src/editor_canvas.rs +++ b/apps/desktop-gpui/src/editor_canvas.rs @@ -448,6 +448,7 @@ pub enum CanvasSelection { Mask(usize), Text(usize), Image(usize), + Video(usize), } impl CanvasSelection { @@ -458,6 +459,7 @@ impl CanvasSelection { Self::Mask(_) => "Mask".into(), Self::Text(_) => "Text".into(), Self::Image(_) => "Image".into(), + Self::Video(_) => "Video".into(), } } @@ -468,6 +470,7 @@ impl CanvasSelection { Self::Mask(index) => format!("canvas-mask-{index}").into(), Self::Text(index) => format!("canvas-text-{index}").into(), Self::Image(index) => format!("canvas-image-{index}").into(), + Self::Video(index) => format!("canvas-video-{index}").into(), } } @@ -476,6 +479,7 @@ impl CanvasSelection { Self::Mask(index) => Some((TrackKind::Mask, index)), Self::Text(index) => Some((TrackKind::Text, index)), Self::Image(index) => Some((TrackKind::Image, index)), + Self::Video(index) => Some((TrackKind::Video, index)), _ => None, } } @@ -601,6 +605,24 @@ impl EditorWindow { } let t = self.preview_or_playhead(); if let Some(timeline) = self.project.timeline.as_ref() { + for (index, segment) in timeline.video_segments.iter().enumerate() { + if exclude == CanvasSelection::Video(index) + || !segment.is_active_at(t) + || segment.opacity <= 0. + { + continue; + } + if let Some(rect) = self.element_rect(CanvasSelection::Video(index)) { + rects.push(image_axis_bounds( + rect, + ( + f64::from(layout.output_size[0]), + f64::from(layout.output_size[1]), + ), + f64::from(segment.rotation), + )); + } + } for (index, segment) in timeline.image_segments.iter().enumerate() { if exclude == CanvasSelection::Image(index) || !segment.is_active_at(t) @@ -684,6 +706,7 @@ impl EditorWindow { | CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => true, + CanvasSelection::Video(_) => true, }; if !draggable { cx.notify(); @@ -717,7 +740,10 @@ impl EditorWindow { let resizable = match element { CanvasSelection::Display => self.display_draggable(), CanvasSelection::Camera => self.camera_resizable(), - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => true, + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => true, }; if !resizable { return; @@ -893,14 +919,28 @@ impl EditorWindow { cx, ); } - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { - let (rect, guides) = if let CanvasSelection::Image(index) = element { - let Some(segment) = self - .project - .timeline - .as_ref() - .and_then(|timeline| timeline.image_segments.get(index)) - else { + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => { + let (rect, guides) = if let CanvasSelection::Image(index) + | CanvasSelection::Video(index) = element + { + let Some(timeline) = self.project.timeline.as_ref() else { + return; + }; + let media = match element { + CanvasSelection::Image(_) => timeline + .image_segments + .get(index) + .map(|segment| (segment.rotation, segment.lock_aspect)), + CanvasSelection::Video(_) => timeline + .video_segments + .get(index) + .map(|segment| (segment.rotation, segment.lock_aspect)), + _ => None, + }; + let Some((rotation, lock_aspect)) = media else { return; }; ( @@ -909,8 +949,8 @@ impl EditorWindow { size, delta, (dir_x, dir_y), - f64::from(segment.rotation), - segment.lock_aspect, + f64::from(rotation), + lock_aspect, ), Vec::new(), ) @@ -928,14 +968,29 @@ impl EditorWindow { let (center, guides) = match element { CanvasSelection::Display => display_drag_center(start, size, delta, &targets, shift), CanvasSelection::Camera => camera_drag_center(start, size, delta, &targets, shift), - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { - let bounds = if let CanvasSelection::Image(index) = element { + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => { + let bounds = if let CanvasSelection::Image(index) | CanvasSelection::Video(index) = + element + { let rotation = self .project .timeline .as_ref() - .and_then(|timeline| timeline.image_segments.get(index)) - .map_or(0., |segment| f64::from(segment.rotation)); + .and_then(|timeline| match element { + CanvasSelection::Image(_) => timeline + .image_segments + .get(index) + .map(|segment| segment.rotation), + CanvasSelection::Video(_) => timeline + .video_segments + .get(index) + .map(|segment| segment.rotation), + _ => None, + }) + .map_or(0., f64::from); image_axis_bounds(start, size, rotation) } else { start @@ -958,7 +1013,10 @@ impl EditorWindow { self.canvas_drag_camera_rect = Some(optimistic); self.write_camera_position(center, cx); } - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => { self.canvas_overlay_rect = Some(optimistic); self.write_overlay_rect(element, optimistic, cx); } @@ -1010,6 +1068,9 @@ impl EditorWindow { CanvasSelection::Image(index) => { tracing::info!(index, "canvas image drag"); } + CanvasSelection::Video(index) => { + tracing::info!(index, "canvas video drag"); + } } } cx.notify(); @@ -1041,6 +1102,18 @@ impl EditorWindow { { return false; } + if let CanvasSelection::Video(index) = selected + && !self + .project + .timeline + .as_ref() + .and_then(|timeline| timeline.video_segments.get(index)) + .is_some_and(|segment| { + segment.is_active_at(self.preview_or_playhead()) && segment.opacity > 0. + }) + { + return false; + } let (Some(canvas), Some(rect)) = (self.canvas_bounds(), self.element_rect(selected)) else { return false; }; @@ -1059,9 +1132,10 @@ impl EditorWindow { let center = match selected { CanvasSelection::Display => display_nudge_center(rect, size, direction, shift), CanvasSelection::Camera => camera_nudge_center(rect, size, direction, shift), - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { - overlay_nudge_center(rect, direction, shift) - } + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => overlay_nudge_center(rect, direction, shift), }; let optimistic = NormRect { x: center.x - rect.w / 2., @@ -1077,7 +1151,10 @@ impl EditorWindow { self.canvas_drag_camera_rect = Some(optimistic); self.write_camera_position(center, cx); } - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => { self.canvas_overlay_rect = Some(optimistic); self.write_overlay_rect(selected, optimistic, cx); } @@ -1178,6 +1255,23 @@ impl EditorWindow { h: segment.size.y, }) } + CanvasSelection::Video(index) => { + if self + .canvas_drag + .as_ref() + .is_some_and(|drag| drag.element == element) + && let Some(rect) = self.canvas_overlay_rect + { + return Some(rect); + } + let segment = self.project.timeline.as_ref()?.video_segments.get(index)?; + Some(NormRect { + x: segment.center.x - segment.size.x / 2., + y: segment.center.y - segment.size.y / 2., + w: segment.size.x, + h: segment.size.y, + }) + } CanvasSelection::Mask(index) => { if self .canvas_drag @@ -1236,6 +1330,13 @@ impl EditorWindow { segment.center = center; segment.size = size; } + CanvasSelection::Video(index) => { + let Some(segment) = timeline.video_segments.get_mut(index) else { + return; + }; + segment.center = center; + segment.size = size; + } CanvasSelection::Mask(index) => { let Some(segment) = timeline.mask_segments.get_mut(index) else { return; @@ -1394,8 +1495,37 @@ impl EditorWindow { }) { if let Some(rect) = self.element_rect(CanvasSelection::Image(index)) { - layer = - layer.child(self.render_image_box(index, rect, (cw, ch), cx)); + layer = layer.child(self.render_media_box( + CanvasSelection::Image(index), + rect, + f64::from(timeline.image_segments[index].rotation), + (cw, ch), + cx, + )); + } + } + } + OverlayTrackKind::Video => { + for (index, segment) in + timeline + .video_segments + .iter() + .enumerate() + .filter(|(_, segment)| { + segment.track == track.track + && segment.is_active_at(time) + && segment.opacity > 0. + }) + { + let element = CanvasSelection::Video(index); + if let Some(rect) = self.element_rect(element) { + layer = layer.child(self.render_media_box( + element, + rect, + f64::from(segment.rotation), + (cw, ch), + cx, + )); } } } @@ -1472,9 +1602,10 @@ impl EditorWindow { self.camera_resizable(), (!self.camera_resizable()).then_some("Camera size follows the zoom — drag to move"), ), - CanvasSelection::Mask(_) | CanvasSelection::Text(_) | CanvasSelection::Image(_) => { - (true, true, None) - } + CanvasSelection::Mask(_) + | CanvasSelection::Text(_) + | CanvasSelection::Image(_) + | CanvasSelection::Video(_) => (true, true, None), }; let left = rect.x as f32 * canvas.0; @@ -2262,20 +2393,14 @@ fn image_resize_rect( } impl EditorWindow { - fn render_image_box( + fn render_media_box( &self, - index: usize, + element: CanvasSelection, rect: NormRect, + rotation: f64, canvas: (f32, f32), cx: &mut Context, ) -> AnyElement { - let element = CanvasSelection::Image(index); - let rotation = self - .project - .timeline - .as_ref() - .and_then(|timeline| timeline.image_segments.get(index)) - .map_or(0., |segment| f64::from(segment.rotation)); let size = (f64::from(canvas.0), f64::from(canvas.1)); let corners = image_corners(rect, size, rotation); let show = self.canvas_selection == Some(element) || self.hovered_canvas == Some(element); @@ -2343,7 +2468,8 @@ impl EditorWindow { layer = layer.child( div() .id(SharedString::from(format!( - "image-handle-{index}-{dx}-{dy}" + "{}-handle-{dx}-{dy}", + element.element_id() ))) .absolute() .left(px(x as f32 - 6.)) diff --git a/apps/desktop-gpui/src/editor_clips.rs b/apps/desktop-gpui/src/editor_clips.rs index 72f90015b6e..62b4ff6f6ee 100644 --- a/apps/desktop-gpui/src/editor_clips.rs +++ b/apps/desktop-gpui/src/editor_clips.rs @@ -2474,6 +2474,7 @@ fn ensure_project_timeline<'a>( camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }); } diff --git a/apps/desktop-gpui/src/editor_edits.rs b/apps/desktop-gpui/src/editor_edits.rs index 93e1889d773..3a0f3bcd636 100644 --- a/apps/desktop-gpui/src/editor_edits.rs +++ b/apps/desktop-gpui/src/editor_edits.rs @@ -358,7 +358,7 @@ pub fn min_segment_duration(kind: TrackKind, secs_per_pixel: f64) -> f64 { TrackKind::Zoom => (1., 40.), TrackKind::Scene => (1., 80.), TrackKind::ThreeD => (1., 40.), - TrackKind::Text | TrackKind::Style | TrackKind::Image => (1., 80.), + TrackKind::Text | TrackKind::Style | TrackKind::Image | TrackKind::Video => (1., 80.), TrackKind::Mask => (1., 80.), TrackKind::Audio => (0.5, 60.), TrackKind::Caption => (0.5, 40.), @@ -563,6 +563,34 @@ impl_track_segment!(MaskSegment, lane: track); impl_track_segment!(TextSegment, lane: track); impl_track_segment!(cap_project::StyleSegment, lane: track); impl_track_segment!(cap_project::ImageSegment, lane: track); +impl TrackSegmentOps for cap_project::VideoSegment { + fn start(&self) -> f64 { + self.start + } + + fn end(&self) -> f64 { + self.end + } + + fn set_start(&mut self, value: f64) { + self.start = value; + } + + fn set_end(&mut self, value: f64) { + self.end = value; + } + + fn lane(&self) -> u32 { + self.track + } + + fn split_tail(&self, at: f64) -> Self { + let mut tail = self.clone(); + tail.start += at; + tail.source_start += at; + tail + } +} impl TrackSegmentOps for CaptionTrackSegment { fn start(&self) -> f64 { @@ -763,6 +791,10 @@ macro_rules! with_track { let $segments = &mut $timeline.image_segments; $body } + TrackKind::Video => { + let $segments = &mut $timeline.video_segments; + $body + } TrackKind::Text => { let $segments = &mut $timeline.text_segments; $body @@ -798,6 +830,7 @@ pub fn segment_count(timeline: &TimelineConfiguration, kind: TrackKind) -> usize TrackKind::ThreeD => timeline.camera3d_segments.len(), TrackKind::Style => timeline.style_segments.len(), TrackKind::Image => timeline.image_segments.len(), + TrackKind::Video => timeline.video_segments.len(), TrackKind::Text => timeline.text_segments.len(), TrackKind::Mask => timeline.mask_segments.len(), TrackKind::Audio => timeline.audio_segments.len(), @@ -813,6 +846,22 @@ pub fn set_segment_start( index: usize, start: f64, ) -> bool { + if kind == TrackKind::Video { + let Some(segment) = timeline.video_segments.get_mut(index) else { + return false; + }; + let source_start = segment.source_start + start - segment.start; + if !source_start.is_finite() + || source_start < 0.0 + || start >= segment.end + || segment.start == start + { + return false; + } + segment.source_start = source_start; + segment.start = start; + return true; + } with_track!(timeline, kind, |segments| { let Some(segment) = segments.get_mut(index) else { return false; @@ -824,7 +873,7 @@ pub fn set_segment_start( return false; } segment.set_start(start); - if !matches!(kind, TrackKind::Style | TrackKind::Image) { + if !matches!(kind, TrackKind::Style | TrackKind::Image | TrackKind::Video) { sort_track(segments); } true @@ -838,6 +887,19 @@ pub fn set_segment_end( index: usize, end: f64, ) -> bool { + if kind == TrackKind::Video { + let Some(segment) = timeline.video_segments.get_mut(index) else { + return false; + }; + if end <= segment.start + || segment.source_start + end - segment.start > segment.source_duration + || segment.end == end + { + return false; + } + segment.end = end; + return true; + } with_track!(timeline, kind, |segments| { let Some(segment) = segments.get_mut(index) else { return false; @@ -846,7 +908,7 @@ pub fn set_segment_end( return false; } segment.set_end(end); - if !matches!(kind, TrackKind::Style | TrackKind::Image) { + if !matches!(kind, TrackKind::Style | TrackKind::Image | TrackKind::Video) { sort_track(segments); } true @@ -887,6 +949,7 @@ pub fn delete_segments( ) -> bool { match kind { TrackKind::Image => delete_indices(&mut timeline.image_segments, indices), + TrackKind::Video => delete_indices(&mut timeline.video_segments, indices), TrackKind::Style => { let deleted = delete_indices(&mut timeline.style_segments, indices); normalize_track(&mut timeline.style_segments, |segment, lane| { @@ -946,6 +1009,12 @@ pub fn delete_track_lane(timeline: &mut TimelineConfiguration, kind: TrackKind, |segment| segment.track, |segment, value| segment.track = value, ), + TrackKind::Video => apply( + &mut timeline.video_segments, + lane, + |segment| segment.track, + |segment, value| segment.track = value, + ), TrackKind::Text => apply( &mut timeline.text_segments, lane, @@ -1190,28 +1259,33 @@ fn ripple_deleted_bounds( (end > start).then_some((start, end)) } -fn ripple_relative_keyframes( - keyframes: &mut Vec, +#[derive(Clone, Copy)] +struct RippleKeyframeTimes { old_start: f64, new_start: f64, new_end: f64, cut_start: f64, cut_end: f64, shift: f64, +} + +fn ripple_relative_keyframes( + keyframes: &mut Vec, + times: RippleKeyframeTimes, time: impl for<'a> Fn(&'a mut T) -> &'a mut f64, ) { keyframes.retain_mut(|keyframe| { let value = time(keyframe); - let absolute = old_start + *value; - if absolute >= cut_start && absolute < cut_end { + let absolute = times.old_start + *value; + if absolute >= times.cut_start && absolute < times.cut_end { return false; } - let mapped = if absolute >= cut_end { - absolute - shift + let mapped = if absolute >= times.cut_end { + absolute - times.shift } else { absolute }; - *value = (mapped - new_start).clamp(0.0, new_end - new_start); + *value = (mapped - times.new_start).clamp(0.0, times.new_end - times.new_start); true }); } @@ -1229,36 +1303,23 @@ fn ripple_delete_mask_track( else { return false; }; - ripple_relative_keyframes( - &mut segment.keyframes.position, + let times = RippleKeyframeTimes { old_start, new_start, new_end, cut_start, cut_end, shift, - |keyframe| &mut keyframe.time, - ); - ripple_relative_keyframes( - &mut segment.keyframes.size, - old_start, - new_start, - new_end, - cut_start, - cut_end, - shift, - |keyframe| &mut keyframe.time, - ); - ripple_relative_keyframes( - &mut segment.keyframes.intensity, - old_start, - new_start, - new_end, - cut_start, - cut_end, - shift, - |keyframe| &mut keyframe.time, - ); + }; + ripple_relative_keyframes(&mut segment.keyframes.position, times, |keyframe| { + &mut keyframe.time + }); + ripple_relative_keyframes(&mut segment.keyframes.size, times, |keyframe| { + &mut keyframe.time + }); + ripple_relative_keyframes(&mut segment.keyframes.intensity, times, |keyframe| { + &mut keyframe.time + }); segment.start = new_start; segment.end = new_end; true @@ -1429,6 +1490,49 @@ fn ripple_delete_audio_track( *segments = next; } +fn ripple_delete_video_track( + segments: &mut Vec, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + let mut next = Vec::with_capacity(segments.len()); + for mut segment in std::mem::take(segments) { + if segment.end <= cut_start { + next.push(segment); + } else if segment.start >= cut_start && segment.end <= cut_end { + continue; + } else if segment.start >= cut_end { + segment.start -= shift; + segment.end -= shift; + next.push(segment); + } else if segment.start < cut_start && segment.end > cut_end { + let mut tail = segment.clone(); + let old_start = tail.start; + segment.end = cut_start; + tail.start = cut_end - shift; + tail.end -= shift; + tail.source_start += cut_end - old_start; + next.push(segment); + if tail.end > tail.start { + next.push(tail); + } + } else if segment.start < cut_start { + segment.end = cut_start; + next.push(segment); + } else { + let old_start = segment.start; + segment.start = cut_end - shift; + segment.end = (segment.end - shift).max(segment.start); + segment.source_start += cut_end - old_start; + if segment.end > segment.start { + next.push(segment); + } + } + } + *segments = next; +} + fn ripple_delete_output_tracks( timeline: &mut TimelineConfiguration, cut_start: f64, @@ -1437,6 +1541,7 @@ fn ripple_delete_output_tracks( ) { ripple_delete_track(&mut timeline.style_segments, cut_start, cut_end, shift); ripple_delete_track(&mut timeline.image_segments, cut_start, cut_end, shift); + ripple_delete_video_track(&mut timeline.video_segments, cut_start, cut_end, shift); ripple_delete_track(&mut timeline.zoom_segments, cut_start, cut_end, shift); ripple_delete_track(&mut timeline.scene_segments, cut_start, cut_end, shift); ripple_delete_camera3d_track(&mut timeline.camera3d_segments, cut_start, cut_end, shift); @@ -1824,6 +1929,7 @@ pub fn ensure_timeline(project: &mut ProjectConfiguration, clip_display_duration camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }); true } @@ -2377,6 +2483,10 @@ pub fn set_clip_segment_timescale( segment.start += shift(segment.start); segment.end += shift(segment.end); } + for segment in &mut timeline.video_segments { + segment.start += shift(segment.start); + segment.end += shift(segment.end); + } for segment in &mut timeline.zoom_segments { segment.start += shift(segment.start); segment.end += shift(segment.end); @@ -4316,6 +4426,8 @@ pub(crate) fn replace_image_asset( } segment.path = path; segment.name = name; + segment.source_path = None; + segment.annotations.clear(); true } @@ -4324,7 +4436,7 @@ mod style_image_replacement_tests { use super::*; #[test] fn style_image_replace_preserves_geometry_and_history_rejects_stale_target() { - let mut project: ProjectConfiguration = serde_json::from_value(serde_json::json!({"timeline":{"zoomSegments":[],"segments":[],"imageSegments":[{"start":2,"end":8,"track":3,"path":"content/images/old.png","name":"Old","center":{"x":0.3,"y":0.7},"size":{"x":0.2,"y":0.4},"rotation":35,"flipX":true,"opacity":0.6}]}})).unwrap(); + let mut project: ProjectConfiguration = serde_json::from_value(serde_json::json!({"timeline":{"zoomSegments":[],"segments":[],"imageSegments":[{"start":2,"end":8,"track":3,"path":"content/images/old.png","sourcePath":"content/images/source.png","annotations":[{"id":"old-drawing","type":"rectangle","x":10.0,"y":20.0,"width":30.0,"height":40.0,"strokeColor":"#f05656","strokeWidth":4.0,"fillColor":"transparent","opacity":1.0,"rotation":0.0}],"name":"Old","center":{"x":0.3,"y":0.7},"size":{"x":0.2,"y":0.4},"rotation":35,"flipX":true,"opacity":0.6}]}})).unwrap(); let before = serde_json::to_value(&project).unwrap(); let mut history = ProjectHistory::new(project.clone()); let fingerprint = @@ -4340,6 +4452,11 @@ mod style_image_replacement_tests { let mut expected = before.clone(); expected["timeline"]["imageSegments"][0]["path"] = "content/images/new.gif".into(); expected["timeline"]["imageSegments"][0]["name"] = "New".into(); + let image = expected["timeline"]["imageSegments"][0] + .as_object_mut() + .unwrap(); + assert!(image.remove("sourcePath").is_some()); + assert!(image.remove("annotations").is_some()); assert_eq!(serde_json::to_value(&project).unwrap(), expected); assert!(!replace_image_asset( &mut project, diff --git a/apps/desktop-gpui/src/editor_panels.rs b/apps/desktop-gpui/src/editor_panels.rs index 4ad482e3035..2b1cc539b0d 100644 --- a/apps/desktop-gpui/src/editor_panels.rs +++ b/apps/desktop-gpui/src/editor_panels.rs @@ -3015,6 +3015,11 @@ segment_editor!( image_segments, cap_project::ImageSegment ); +segment_editor!( + edit_video_segment, + video_segments, + cap_project::VideoSegment +); segment_editor!(edit_audio_segment, audio_segments, AudioTrackSegment); impl EditorWindow { @@ -3826,12 +3831,22 @@ impl EditorWindow { cx, |this, index, cx| this.render_style_panel(index, cx), ), - TrackKind::Image => self.stacked_panel( - "image", - "image", - count(timeline.image_segments.len()), + TrackKind::Image => { + let indices = count(timeline.image_segments.len()); + if indices.len() == 1 { + self.render_image_panel(indices[0], cx) + } else { + self.stacked_panel("image", "image", indices, cx, |this, index, cx| { + this.render_image_panel(index, cx) + }) + } + } + TrackKind::Video => self.stacked_panel( + "video", + "video", + count(timeline.video_segments.len()), cx, - |this, index, cx| this.render_image_panel(index, cx), + |this, index, cx| this.render_video_panel(index, cx), ), TrackKind::Zoom => { let indices = count(timeline.zoom_segments.len()); @@ -8715,17 +8730,209 @@ impl ImageProperty { } impl EditorWindow { + fn render_video_panel(&self, index: usize, cx: &mut Context) -> AnyElement { + let Some(segment) = self + .timeline() + .and_then(|timeline| timeline.video_segments.get(index)) + else { + return div().into_any_element(); + }; + let mut panel = div() + .flex() + .flex_col() + .gap(px(16.)) + .child( + self.labelled_small( + "Video", + div() + .text_size(px(13.)) + .child(segment.name.clone()) + .into_any_element(), + ), + ) + .child( + div() + .text_size(px(12.)) + .text_color(Hsla::from(self.theme.gray_10)) + .child("Drag the video on the canvas to move it. Pull a corner to resize it."), + ); + for (key, label, value) in [ + (0, "Enabled", segment.enabled), + (1, "Mute audio", segment.muted), + (2, "Lock aspect ratio", segment.lock_aspect), + (3, "Flip horizontally", segment.flip_x), + (4, "Flip vertically", segment.flip_y), + ] { + panel = panel.child( + ui::Subfield::plain(&self.theme, label).child( + ui::Toggle::plain( + &self.theme, + SharedString::from(format!("video-{index}-{key}")), + value, + ) + .on_click(cx.listener(move |this, _, window, cx| { + this.edit_video_segment( + "video-toggle", + index, + window, + cx, + move |segment| { + match key { + 0 => segment.enabled = !value, + 1 => segment.muted = !value, + 2 => segment.lock_aspect = !value, + 3 => segment.flip_x = !value, + _ => segment.flip_y = !value, + } + true + }, + ); + })), + ), + ); + } + panel.into_any_element() + } + fn render_image_panel(&self, index: usize, cx: &mut Context) -> AnyElement { + use crate::screenshot_annotations::Tool; + use crate::screenshot_editor::ImageEditAction; + let Some(segment) = self .timeline() .and_then(|timeline| timeline.image_segments.get(index)) else { return div().into_any_element(); }; + let theme = self.theme; + let screenshot = div() + .flex() + .flex_col() + .gap(px(12.)) + .child( + div() + .text_size(px(14.)) + .font_weight(FontWeight::SEMIBOLD) + .text_color(Hsla::from(theme.editor.text_1)) + .child("Image editing"), + ) + .child( + ui::Button::plain( + &self.theme, + SharedString::from(format!("edit-screenshot-{index}")), + ui::ButtonVariant::Gray, + ui::ButtonSize::Md, + ) + .icon("icons/pencil.svg") + .label("Open canvas") + .on_click(cx.listener(move |this, _, window, cx| { + this.open_image_drawing(index, ImageEditAction::Tool(Tool::Select), window, cx) + })), + ) + .child( + div() + .text_size(px(12.)) + .font_weight(FontWeight::MEDIUM) + .text_color(Hsla::from(theme.editor.text_2)) + .child("Annotate"), + ) + .child( + div() + .flex() + .flex_row() + .flex_wrap() + .gap(px(8.)) + .children(Tool::ALL.map(|tool| { + div() + .id(SharedString::from(format!( + "image-tool-{index}-{}", + tool.label() + ))) + .tab_index(0) + .flex() + .flex_col() + .items_center() + .justify_center() + .gap(px(4.)) + .w(px(84.)) + .h(px(68.)) + .rounded(px(12.)) + .bg(Hsla::from(theme.editor.ctl)) + .cursor_pointer() + .hover(move |style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .child( + svg() + .path(tool.icon()) + .size(px(20.)) + .text_color(Hsla::from(theme.editor.text_2)), + ) + .child(div().text_size(px(11.)).truncate().child(tool.label())) + .tooltip(move |_window, cx| { + ui::Tooltip::new(&theme, tool.label()).view(cx) + }) + .on_click(cx.listener(move |this, _, window, cx| { + this.open_image_drawing( + index, + ImageEditAction::Tool(tool), + window, + cx, + ) + })) + })), + ) + .child( + div() + .text_size(px(12.)) + .font_weight(FontWeight::MEDIUM) + .text_color(Hsla::from(theme.editor.text_2)) + .child("Appearance"), + ) + .child( + div().flex().flex_row().flex_wrap().gap(px(8.)).children( + [ + ("Aspect", "icons/layout.svg", ImageEditAction::Aspect), + ("Crop", "icons/crop.svg", ImageEditAction::Crop), + ("Background", "icons/image.svg", ImageEditAction::Background), + ("Padding", "icons/padding.svg", ImageEditAction::Padding), + ("Corners", "icons/corners.svg", ImageEditAction::Rounding), + ("Shadow", "icons/shadow.svg", ImageEditAction::Shadow), + ("Border", "icons/square.svg", ImageEditAction::Border), + ] + .map(|(label, icon, action)| { + div() + .id(SharedString::from(format!( + "image-appearance-{index}-{label}" + ))) + .tab_index(0) + .flex() + .flex_col() + .items_center() + .justify_center() + .gap(px(4.)) + .w(px(84.)) + .h(px(68.)) + .rounded(px(12.)) + .bg(Hsla::from(theme.editor.ctl)) + .cursor_pointer() + .hover(move |style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .child( + svg() + .path(icon) + .size(px(20.)) + .text_color(Hsla::from(theme.editor.text_2)), + ) + .child(div().text_size(px(11.)).truncate().child(label)) + .tooltip(move |_window, cx| ui::Tooltip::new(&theme, label).view(cx)) + .on_click(cx.listener(move |this, _, window, cx| { + this.open_image_drawing(index, action, window, cx) + })) + }), + ), + ); let mut panel = div() .flex() .flex_col() - .gap(px(16.)) + .gap(px(12.)) .child(self.labelled_small( "Name", self.render_field_input(FieldKey::ImageName(index), None), @@ -8853,7 +9060,24 @@ impl EditorWindow { ), ); } - panel.into_any_element() + let layer = self.sidebar.section(PanelSection::ImageLayer); + div() + .flex() + .flex_col() + .gap(px(20.)) + .child(screenshot) + .child(crate::editor_sidebar::disclosure_row( + &theme, + "image-layer-settings", + "Layer settings", + layer.is_open(), + cx.listener(|this, _, window, cx| { + this.sidebar.section(PanelSection::ImageLayer).toggle(); + this.animate_collapsibles(window, cx); + }), + )) + .child(collapsible(&layer, panel.into_any_element())) + .into_any_element() } fn render_style_panel(&self, index: usize, cx: &mut Context) -> AnyElement { diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index 339dea45ab1..038e5d6286f 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -1025,6 +1025,7 @@ pub enum PanelSection { /// "Timing & advanced". Camera3DAdvanced, ZoomHelper, + ImageLayer, } // --------------------------------------------------------------------------- @@ -2529,11 +2530,62 @@ impl EditorWindow { }) .collect(); - let rail = ui::TabRail::editor(&theme, "sidebar-tabs", self.panel_bg(), items) + let tab_rail = ui::TabRail::editor(&theme, "sidebar-tabs", self.panel_bg(), items) .height(px(crate::editor_window::SIDEBAR_TAB_BAR_HEIGHT)) .on_select(cx.listener(|this, index: &usize, window, cx| { this.select_sidebar_tab(*index, window, cx); })); + let rail = if selection.as_ref().is_some_and(|selection| { + selection.track == TrackKind::Image && selection.indices.len() == 1 + }) { + div() + .flex() + .flex_row() + .items_center() + .gap(px(8.)) + .h(px(crate::editor_window::SIDEBAR_TAB_BAR_HEIGHT)) + .px(px(12.)) + .flex_none() + .border_b_1() + .border_color(self.card_line()) + .child( + div() + .id("image-sidebar-back") + .flex() + .flex_row() + .items_center() + .justify_center() + .gap(px(4.)) + .h(px(32.)) + .px(px(8.)) + .rounded(px(8.)) + .cursor_pointer() + .hover(|style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .child( + svg() + .path("icons/arrow-left.svg") + .size(px(16.)) + .text_color(Hsla::from(theme.editor.text_2)), + ) + .child( + div() + .text_size(px(11.)) + .text_color(Hsla::from(theme.editor.text_2)) + .child("Back"), + ) + .on_click(cx.listener(|this, _, _window, cx| this.set_selection(None, cx))), + ) + .child( + div() + .text_size(px(13.)) + .font_weight(FontWeight::SEMIBOLD) + .text_color(Hsla::from(theme.editor.text_1)) + .child("Edit image"), + ) + .into_any_element() + } else { + tab_rail.into_any_element() + }; div() .w(px(crate::editor_window::SIDEBAR_WIDTH)) diff --git a/apps/desktop-gpui/src/editor_timeline.rs b/apps/desktop-gpui/src/editor_timeline.rs index 987bb776ee2..24ec0671452 100644 --- a/apps/desktop-gpui/src/editor_timeline.rs +++ b/apps/desktop-gpui/src/editor_timeline.rs @@ -31,7 +31,7 @@ //! A segment wider than the viewport has its true centre off screen, so //! [`visible_box`] clamps it (`useSegmentVisibleBox`, `TL/Track.tsx:147-181`). -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; mod playback_follow; @@ -176,6 +176,7 @@ pub const MINIMAP_MAX_WIDTH: f32 = 78.; /// (`apps/desktop/src/styles/theme.css`). pub mod track_color { pub const CLIP: u32 = 0x3b82f6; + pub const VIDEO: u32 = 0x2563eb; pub const ZOOM: u32 = 0x64748b; pub const CAPTION: u32 = 0x0ea5e9; pub const KEYBOARD: u32 = 0xf97316; @@ -578,23 +579,6 @@ pub fn waveform_path( if peaks.is_empty() || scale <= 0. { return None; } - let duration = (range.1 - range.0).max(WAVEFORM_SAMPLE_STEP); - if !duration.is_finite() || duration <= 0. { - return None; - } - - let native_samples = (duration / WAVEFORM_SAMPLE_STEP).ceil() as usize + 1; - let num_samples = target_samples - .clamp(50, MAX_WAVEFORM_SAMPLES) - .min(native_samples); - if num_samples == 0 { - return None; - } - let time_step = duration / num_samples as f64; - - // `sourceTimeAt` (`TL/ClipTrack.tsx:185-193`): output time back to - // recording time, or `null` inside a hold -- the mixer renders silence - // there, so the waveform drops to the baseline. let source_time_at = |output_time: f64| -> Option { let mut held = 0.; for (start, end) in holds { @@ -608,6 +592,40 @@ pub fn waveform_path( } Some(segment_start + output_time - held) }; + build_waveform_path( + range, + target_samples, + origin, + size, + scale, + clip_bounds, + |time, _| waveform_amplitude(peaks, source_time_at(time)), + ) +} + +#[allow(clippy::too_many_arguments)] +fn build_waveform_path( + range: (f64, f64), + target_samples: usize, + origin: gpui::Point, + size: gpui::Size, + scale: f64, + clip_bounds: gpui::Bounds, + amplitude_at: impl Fn(f64, f64) -> f64, +) -> Option> { + let duration = (range.1 - range.0).max(WAVEFORM_SAMPLE_STEP); + if !duration.is_finite() || duration <= 0. { + return None; + } + + let native_samples = (duration / WAVEFORM_SAMPLE_STEP).ceil() as usize + 1; + let num_samples = target_samples + .clamp(50, MAX_WAVEFORM_SAMPLES) + .min(native_samples); + if num_samples == 0 { + return None; + } + let time_step = duration / num_samples as f64; let width = f32::from(size.width) as f64; let height = f32::from(size.height) as f64; @@ -640,8 +658,8 @@ pub fn waveform_path( let normalized_x = (time - range.0) / duration; let prev_time = time - time_step; let prev_x = ((prev_time - range.0) / duration).max(0.); - let y = 1. - waveform_amplitude(peaks, source_time_at(time)); - let prev_y = 1. - waveform_amplitude(peaks, source_time_at(prev_time)); + let y = 1. - amplitude_at(time, time_step); + let prev_y = 1. - amplitude_at(prev_time, time_step); let cp_x1 = prev_x + control_step / 2.; let cp_x2 = normalized_x - control_step / 2.; builder.cubic_bezier_to(map(normalized_x, y), map(cp_x1, prev_y), map(cp_x2, y)); @@ -669,6 +687,88 @@ pub fn waveform_path( builder.build().ok() } +#[derive(Debug)] +pub struct ImportedWaveform { + levels: Vec>, +} + +impl ImportedWaveform { + pub fn new(peaks: Arc<[u8]>) -> Self { + let mut levels = vec![peaks]; + while levels.last().is_some_and(|level| level.len() > 1) { + let previous = levels.last().unwrap(); + let mut next = Vec::with_capacity(previous.len().div_ceil(2)); + for pair in previous.chunks(2) { + next.push(pair.iter().copied().max().unwrap_or(0)); + } + levels.push(next.into()); + } + Self { levels } + } + + pub fn is_empty(&self) -> bool { + self.levels[0].is_empty() + } + + fn range_max(&self, start: f64, end: f64) -> f64 { + let length = self.levels[0].len(); + let first = (start.floor().max(0.) as usize).min(length); + if first >= length { + return 0.; + } + let last = (end.ceil().max(0.) as usize).min(length).max(first + 1); + let mut peak = 0u8; + let mut left = first; + let mut right = last; + for level in &self.levels { + if left >= right { + break; + } + if !left.is_multiple_of(2) { + peak = peak.max(level.get(left).copied().unwrap_or(0)); + left += 1; + } + if !right.is_multiple_of(2) { + right -= 1; + peak = peak.max(level.get(right).copied().unwrap_or(0)); + } + left /= 2; + right /= 2; + } + f64::from(peak) / 255. + } +} + +#[allow(clippy::too_many_arguments)] +fn imported_waveform_path( + waveform: &ImportedWaveform, + range: (f64, f64), + target_samples: usize, + source_start: f64, + origin: gpui::Point, + size: gpui::Size, + scale: f64, + clip_bounds: gpui::Bounds, +) -> Option> { + if waveform.is_empty() || scale <= 0. { + return None; + } + build_waveform_path( + range, + target_samples, + origin, + size, + scale, + clip_bounds, + |time, step| { + waveform.range_max( + (source_start + time) * 10., + (source_start + time + step) * 10., + ) + }, + ) +} + fn rounded_corner_inset(x: f64, width: f64, radius: f64) -> f64 { let distance = x.min(width - x).clamp(0., radius); if distance >= radius { @@ -701,6 +801,7 @@ pub fn waveform_color(color: Hsla) -> Hsla { pub enum TrackKind { Style, Image, + Video, Clip, Caption, Keyboard, @@ -716,11 +817,10 @@ impl TrackKind { /// `trackDefinitions` (`TL/index.tsx:89-144`) and `trackIcons` (`:70-80`). pub fn label(self) -> &'static str { match self { - // The clip row's gutter label is "Video", not the definition's - // "Clip" (`TL/index.tsx:1334`). - Self::Clip => "Video", + Self::Clip => "Recording", Self::Style => "Style", Self::Image => "Image", + Self::Video => "Video", Self::Caption => "Captions", Self::Keyboard => "Keyboard", Self::Text => "Text", @@ -737,6 +837,7 @@ impl TrackKind { Self::Clip => "icons/clapperboard.svg", Self::Style => "icons/palette.svg", Self::Image => "icons/image.svg", + Self::Video => "icons/video.svg", Self::Caption => "icons/captions.svg", Self::Keyboard => "icons/keyboard.svg", Self::Text => "icons/type.svg", @@ -751,6 +852,7 @@ impl TrackKind { pub fn color(self) -> Hsla { gpui::rgb(match self { Self::Clip => track_color::CLIP, + Self::Video => track_color::VIDEO, Self::Style => track_color::STYLE, Self::Image => track_color::IMAGE, Self::Caption => track_color::CAPTION, @@ -768,6 +870,7 @@ impl TrackKind { pub fn picker_label(self) -> &'static str { match self { Self::Clip => "Clip", + Self::Video => "Video", other => other.label(), } } @@ -775,6 +878,7 @@ impl TrackKind { pub fn picker_description(self) -> &'static str { match self { Self::Clip => "Your recorded screen footage.", + Self::Video => "Add a video file to the timeline.", Self::Style => "Change background, camera and cursor settings over time.", Self::Image => "Add an image to your recording.", Self::Zoom => "Smooth zoom-ins that follow the action.", @@ -798,7 +902,7 @@ impl TrackKind { pub fn supports_multiple(self) -> bool { matches!( self, - Self::Text | Self::Mask | Self::Audio | Self::Style | Self::Image + Self::Text | Self::Mask | Self::Audio | Self::Style | Self::Image | Self::Video ) } @@ -806,6 +910,7 @@ impl TrackKind { let kind = match self { Self::Mask => OverlayTrackKind::Mask, Self::Image => OverlayTrackKind::Image, + Self::Video => OverlayTrackKind::Video, Self::Text => OverlayTrackKind::Text, _ => return None, }; @@ -816,6 +921,7 @@ impl TrackKind { pub const ADD_TRACK_OPTIONS: &[TrackKind] = &[ TrackKind::Style, TrackKind::Image, + TrackKind::Video, TrackKind::Caption, TrackKind::Keyboard, TrackKind::Text, @@ -829,6 +935,7 @@ pub const ADD_TRACK_OPTIONS: &[TrackKind] = &[ pub struct TrackLanes { pub style: u32, pub image: u32, + pub video: u32, pub caption: bool, pub keyboard: bool, pub scene: bool, @@ -857,6 +964,9 @@ impl TrackLanes { image: timeline.map_or(0, |timeline| { used_config_lane_count(timeline.image_segments.iter().map(|segment| segment.track)) }), + video: timeline.map_or(0, |timeline| { + used_config_lane_count(timeline.video_segments.iter().map(|segment| segment.track)) + }), caption: config .captions .as_ref() @@ -890,6 +1000,7 @@ impl TrackLanes { TrackKind::ThreeD => self.three_d, TrackKind::Style => self.style > 0, TrackKind::Image => self.image > 0, + TrackKind::Video => self.video > 0, TrackKind::Text => self.text > 0, TrackKind::Mask => self.mask > 0, TrackKind::Audio => self.audio > 0, @@ -901,6 +1012,7 @@ impl TrackLanes { match kind { TrackKind::Style => self.style, TrackKind::Image => self.image, + TrackKind::Video => self.video, TrackKind::Text => self.text, TrackKind::Mask => self.mask, TrackKind::Audio => self.audio, @@ -938,6 +1050,14 @@ pub enum SegmentDetail { name: SharedString, enabled: bool, }, + Video { + name: SharedString, + enabled: bool, + path: SharedString, + source_start: f64, + muted: bool, + volume_db: f64, + }, /// `TL/ClipTrack.tsx`. `start`/`end` above are the **output-time** box; /// these carry the recording-domain numbers the label reads. Clip { @@ -989,6 +1109,9 @@ pub enum SegmentDetail { Audio { name: SharedString, enabled: bool, + path: SharedString, + trim_start: f64, + volume_db: f64, fade_in: f64, fade_out: f64, }, @@ -1027,6 +1150,7 @@ impl TrackRow { let kind = match track.kind { OverlayTrackKind::Mask => TrackKind::Mask, OverlayTrackKind::Image => TrackKind::Image, + OverlayTrackKind::Video => TrackKind::Video, OverlayTrackKind::Text => TrackKind::Text, }; Self { @@ -1041,6 +1165,7 @@ impl TrackRow { pub struct TimelineModel { pub style: Vec, pub image: Vec, + pub video: Vec, pub rows: Vec, pub clips: Vec, pub zoom: Vec, @@ -1063,6 +1188,7 @@ pub struct TimelineModel { /// (`TL/ClipTrack.tsx:713-730`). pub mic_waveforms: Vec>>, pub system_waveforms: Vec>>, + pub imported_waveforms: HashMap>, /// The span a live ghost trim is removing, in output time. Drawn as a gap /// with a red duration badge, the way Blip's ghost resize marks the cut. pub clip_ghost_gap: Option<(f64, f64)>, @@ -1084,6 +1210,7 @@ impl TimelineModel { match kind { TrackKind::Style => &self.style, TrackKind::Image => &self.image, + TrackKind::Video => &self.video, TrackKind::Clip => &self.clips, TrackKind::Caption => &self.caption, TrackKind::Keyboard => &self.keyboard, @@ -1205,6 +1332,9 @@ impl TimelineModel { .filter(|name| !name.is_empty()) .map_or_else(|| SharedString::new_static("Audio"), SharedString::from), enabled: segment.enabled, + path: segment.path.clone().into(), + trim_start: segment.trim_start, + volume_db: f64::from(segment.volume_db), fade_in: segment.fade_in, fade_out: segment.fade_out, }, @@ -1273,9 +1403,27 @@ impl TimelineModel { }, }) .collect(); + let video = timeline + .video_segments + .iter() + .map(|segment| Segment { + start: segment.start, + end: segment.end, + lane: segment.track, + detail: SegmentDetail::Video { + name: segment.name.clone().into(), + enabled: segment.enabled, + path: segment.path.clone().into(), + source_start: segment.source_start, + muted: segment.muted, + volume_db: f64::from(segment.volume_db), + }, + }) + .collect(); let mut model = Self { style, image, + video, rows: Vec::new(), clips, zoom, @@ -1292,6 +1440,7 @@ impl TimelineModel { system_volume_db: config.audio.system_volume_db as f64, mic_waveforms: Vec::new(), system_waveforms: Vec::new(), + imported_waveforms: HashMap::new(), clip_ghost_gap: None, }; model.rows = build_rows( @@ -1331,10 +1480,13 @@ fn build_rows( has_camera: bool, lanes: &TrackLanes, ) -> Vec { - let mut rows = vec![TrackRow { - kind: TrackKind::Clip, - lane: 0, - }]; + let mut rows = Vec::new(); + if !model.clips.is_empty() { + rows.push(TrackRow { + kind: TrackKind::Clip, + lane: 0, + }); + } if lanes.caption { rows.push(TrackRow { kind: TrackKind::Caption, @@ -1357,6 +1509,7 @@ fn build_rows( for (kind, segments, count) in [ (TrackKind::Text, &model.text, lanes.text), (TrackKind::Image, &model.image, lanes.image), + (TrackKind::Video, &model.video, lanes.video), (TrackKind::Mask, &model.mask, lanes.mask), ] { overlay_tracks.extend( @@ -1377,10 +1530,12 @@ fn build_rows( lane, }); } - rows.push(TrackRow { - kind: TrackKind::Zoom, - lane: 0, - }); + if !model.clips.is_empty() { + rows.push(TrackRow { + kind: TrackKind::Zoom, + lane: 0, + }); + } if lanes.three_d { rows.push(TrackRow { kind: TrackKind::ThreeD, @@ -2469,6 +2624,7 @@ fn render_segment( SegmentDetail::Text { enabled, .. } | SegmentDetail::Style { enabled, .. } | SegmentDetail::Image { enabled, .. } + | SegmentDetail::Video { enabled, .. } if !enabled => { Some(0.6) @@ -2533,6 +2689,39 @@ fn render_segment( } } + let imported = match &segment.detail { + SegmentDetail::Video { + path, + source_start, + volume_db, + enabled: true, + muted: false, + .. + } => Some((path.as_ref(), *source_start, *volume_db)), + SegmentDetail::Audio { + path, + trim_start, + volume_db, + enabled: true, + .. + } => Some((path.as_ref(), *trim_start, *volume_db)), + _ => None, + }; + if let Some((path, source_start, volume_db)) = imported + && let Some(waveform) = model.imported_waveforms.get(path) + { + fill = fill.child(render_imported_waveform( + waveform.clone(), + segment, + source_start, + volume_db, + view, + width, + height, + color, + )); + } + fill = fill.child(render_label( theme, color, @@ -3093,6 +3282,68 @@ fn render_waveform( .into_any_element() } +#[allow(clippy::too_many_arguments)] +fn render_imported_waveform( + waveform: Arc, + segment: &Segment, + source_start: f64, + volume_db: f64, + view: TimelineView, + width: f32, + height: f32, + color: Hsla, +) -> impl IntoElement { + let timeline_start = segment.start; + let duration = (segment.end - segment.start).max(0.0001); + let transform = view.transform; + let full_width = width.max(1.) as f64; + let wave_height = height.min(WAVEFORM_MAX_HEIGHT); + let wave_color = waveform_color(color); + let scale = gain_to_scale(volume_db); + gpui::canvas( + |bounds, _window, _cx| bounds, + move |_, bounds, window, _cx| { + let visible_start = transform.position.max(timeline_start) - timeline_start; + let visible_end = (transform.position + transform.zoom).min(timeline_start + duration) + - timeline_start; + if visible_end <= visible_start { + return; + } + let pixels_per_second = full_width / duration; + let origin = gpui::point( + bounds.origin.x + px((visible_start * pixels_per_second) as f32), + bounds.origin.y, + ); + let slice_width = ((visible_end - visible_start) * pixels_per_second) as f32; + let size = gpui::size(px(slice_width), px(wave_height)); + let samples = waveform_sample_count(f64::from(slice_width)); + if let Some(path) = imported_waveform_path( + &waveform, + (visible_start, visible_end), + samples, + source_start, + origin, + size, + scale, + gpui::Bounds { + origin: gpui::point(bounds.origin.x, bounds.bottom() - px(height)), + size: gpui::size(px(width), px(height)), + }, + ) { + window.with_content_mask(Some(gpui::ContentMask { bounds }), |window| { + window.paint_path(path, wave_color) + }); + } + }, + ) + .absolute() + .bottom_0() + .left_0() + .w(px(width)) + .h(px(wave_height)) + .into_any_element() +} + /// `SegmentLabel` (`TL/Track.tsx:186-220`): full, compact and glyph tiers, /// anchored to the visible box and left-aligned inside the segment's own /// `0 10px 0 13px` content padding. @@ -3207,7 +3458,9 @@ fn label_body( ) -> Option { Some(match (&segment.detail, tier) { ( - SegmentDetail::Style { name, .. } | SegmentDetail::Image { name, .. }, + SegmentDetail::Style { name, .. } + | SegmentDetail::Image { name, .. } + | SegmentDetail::Video { name, .. }, LabelTier::Full | LabelTier::Compact, ) => label_row() .child(label_primary(theme, color).child(name.clone())) @@ -3218,6 +3471,9 @@ fn label_body( (SegmentDetail::Image { .. }, LabelTier::Glyph) => { label_glyph(theme, color, "icons/image.svg", 12.) } + (SegmentDetail::Video { .. }, LabelTier::Glyph) => { + label_glyph(theme, color, "icons/video.svg", 12.) + } // -- Clip (`TL/ClipTrack.tsx:1255-1279`) -------------------------- ( diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 5cbec1e496e..3738128d4e7 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -37,12 +37,12 @@ use std::{ cell::RefCell, - collections::HashMap, + collections::{HashMap, HashSet, VecDeque}, path::PathBuf, rc::Rc, sync::{ Arc, Mutex, - atomic::{AtomicU32, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }, time::{Duration, Instant}, }; @@ -59,9 +59,9 @@ use core_foundation::base::TCFType; #[cfg(target_os = "macos")] use core_video::pixel_buffer::{CVPixelBuffer, CVPixelBufferRef}; use gpui::{ - Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, FocusHandle, - FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent, - MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString, + Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, ExternalPaths, + FocusHandle, FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, + MouseMoveEvent, MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString, StatefulInteractiveElement as _, StyleRefinement, Styled, Subscription, WeakEntity, Window, div, point, prelude::FluentBuilder, px, svg, }; @@ -369,6 +369,7 @@ pub fn preflight(path: &std::path::Path) -> Result { return Err("Cannot edit non-studio recordings".to_string()); }; + let still_image = cap_rendering::media_project::still_image_path(studio); let (segment_count, has_camera, multiple_recording_segments) = match studio.as_ref() { StudioRecordingMeta::SingleSegment { segment } => (1, segment.camera.is_some(), false), StudioRecordingMeta::MultipleSegments { inner } => ( @@ -377,6 +378,11 @@ pub fn preflight(path: &std::path::Path) -> Result { inner.segments.len() > 1, ), }; + let segment_count = if still_image.is_some() { + 0 + } else { + segment_count + }; // `hasMicrophone` reads `audio` on a single-segment recording and `mic` on // a multi-segment one; `hasSystemAudio` is a multi-segment concept only @@ -390,21 +396,33 @@ pub fn preflight(path: &std::path::Path) -> Result { } }; - if segment_count == 0 { + if segment_count == 0 + && !matches!( + studio.status(), + cap_project::StudioRecordingStatus::Complete + ) + { return Err("Recording has no segments. It may need to be recovered first.".to_string()); } // The panicking call, contained. `AssertUnwindSafe` because neither // borrow escapes the closure and nothing is left half-mutated by an // unwind here -- the value is constructed and dropped inside it. - let owned_path = path.to_path_buf(); - let recordings = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - ProjectRecordingsMeta::new(&owned_path, studio.as_ref()) - })) - .map_err(|_| { - "This recording's video tracks could not be opened. The bundle looks damaged.".to_string() - })? - .map_err(|error| format!("Failed to read this recording's media: {error}"))?; + let recordings = if segment_count == 0 { + ProjectRecordingsMeta { + segments: Vec::new(), + } + } else { + let owned_path = path.to_path_buf(); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + ProjectRecordingsMeta::new(&owned_path, studio.as_ref()) + })) + .map_err(|_| { + "This recording's video tracks could not be opened. The bundle looks damaged." + .to_string() + })? + .map_err(|error| format!("Failed to read this recording's media: {error}"))? + }; let recordings = Arc::new(recordings); // `RecordingMeta::project_config()` loads `project-config.json` (falling @@ -412,6 +430,9 @@ pub fn preflight(path: &std::path::Path) -> Result { // `EditorInstance::new` starts from, so the timeline shown here is the one // that will be rendered. let mut config = meta.project_config(); + if let Some(image_path) = &still_image { + cap_rendering::media_project::add_still_image_to_timeline(&mut config, image_path); + } // With no persisted timeline `EditorInstance::new` synthesises one from // the per-segment display durations (`editor_instance.rs:210-230`) and // writes it back. Synthesise the same shape here so the strip is not empty @@ -434,8 +455,6 @@ pub fn preflight(path: &std::path::Path) -> Result { volume: None, }) .collect(), - // `TimelineConfiguration` has no `Default`, so the eight other - // track vectors are spelled out empty. transitions: Vec::new(), zoom_segments: Vec::new(), scene_segments: Vec::new(), @@ -447,6 +466,7 @@ pub fn preflight(path: &std::path::Path) -> Result { camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }); } @@ -1371,6 +1391,10 @@ struct ClipReleaseAnim { pub struct EditorWindow { pub(crate) theme: Theme, pub(crate) project_path: PathBuf, + pub(crate) screenshot_workspace: + Option>, + pub(crate) image_drawing_workspace: + Option>, state: LoadState, preparing_consumer: Option, preparing_candidate_frame: Option, @@ -1429,6 +1453,8 @@ pub struct EditorWindow { // -- Timeline ----------------------------------------------------------- /// Every track the strip draws. timeline: TimelineModel, + waveform_pending: HashSet, + waveform_cancellation: Arc, /// The viewport, the hover ghost and the hovered track. view: TimelineView, playback_follow: timeline::PlaybackFollow, @@ -1582,6 +1608,9 @@ pub struct EditorWindow { preview_quality: crate::store::EditorPreviewQuality, pub(crate) tracks: TrackLanes, + media_drop_queue: VecDeque, + media_drop_active: bool, + video_picker_active: bool, toolbar_menu: Option, frame_controls: frame::FrameControls, add_track: Option, @@ -1619,6 +1648,12 @@ pub struct EditorWindow { pub(crate) clips: crate::editor_clips::ClipsState, } +impl Drop for EditorWindow { + fn drop(&mut self) { + self.waveform_cancellation.store(true, Ordering::Relaxed); + } +} + struct PresetsMenu { origin: gpui::Point, store: crate::presets::PresetsStore, @@ -2044,7 +2079,6 @@ impl EditorWindow { }); let timeline_view = cx.new(move |cx| EditorSectionView::new(&editor, EditorSection::Timeline, cx)); - Self { // No material and no transparency: `applyMacOSWindowMaterial` runs // in the `(window-chrome)` layout and `/editor` is not one of its @@ -2055,6 +2089,8 @@ impl EditorWindow { .ok() .and_then(|meta| meta.sharing), project_path, + screenshot_workspace: None, + image_drawing_workspace: None, state: LoadState::Loading, preparing_consumer: None, preparing_candidate_frame: None, @@ -2088,6 +2124,8 @@ impl EditorWindow { stats: None, play_mark: None, timeline: TimelineModel::default(), + waveform_pending: HashSet::new(), + waveform_cancellation: Arc::new(AtomicBool::new(false)), view: TimelineView::default(), playback_follow: timeline::PlaybackFollow::default(), fitted: false, @@ -2149,6 +2187,9 @@ impl EditorWindow { snap_guides: Vec::new(), preview_quality: crate::store::GeneralSettings::load().editor_preview_quality, tracks: TrackLanes::from_project(&ProjectConfiguration::default(), false), + media_drop_queue: VecDeque::new(), + media_drop_active: false, + video_picker_active: false, toolbar_menu: None, frame_controls: frame::FrameControls::default(), add_track: None, @@ -2202,6 +2243,10 @@ impl EditorWindow { } pub(crate) fn flush_pending_saves(&mut self, cx: &mut Context) -> Result<(), String> { + if let Some(workspace) = &self.screenshot_workspace { + workspace.read(cx).pending_save().borrow_mut().flush(); + return Ok(()); + } self.commit_pretty_name(cx)?; self.pending_save.borrow_mut().try_flush().map_err(|error| { format!( @@ -2224,7 +2269,106 @@ impl EditorWindow { } pub fn focus_root(&self, window: &mut Window, cx: &mut Context) { - window.focus(&self.focus, cx); + let focus = self + .image_drawing_workspace + .as_ref() + .or(self.screenshot_workspace.as_ref()) + .map(|workspace| workspace.read(cx).focus.clone()) + .unwrap_or_else(|| self.focus.clone()); + window.focus(&focus, cx); + } + + pub(crate) fn open_image_drawing( + &mut self, + index: usize, + action: crate::screenshot_editor::ImageEditAction, + window: &mut Window, + cx: &mut Context, + ) { + if self.image_drawing_workspace.is_some() { + return; + } + let Some(segment) = self + .project + .timeline + .as_ref() + .and_then(|timeline| timeline.image_segments.get(index)) + else { + return; + }; + let source = segment + .source_path + .as_deref() + .unwrap_or(&segment.path) + .to_string(); + if let Err(error) = self.flush_pending_saves(cx) { + tracing::error!(%error, "could not save project before image drawing"); + return; + } + self.stop_playback(cx); + let project_path = self.project_path.clone(); + let workspace = cx.new(|cx| { + crate::screenshot_editor::ScreenshotEditorWindow::new_image_drawing( + project_path.clone(), + index, + source, + action, + window, + cx, + ) + }); + self.image_drawing_workspace = Some(workspace); + self.focus_root(window, cx); + if let Some(handle) = window.window_handle().downcast::() { + cx.defer(move |cx| { + crate::screenshot_editor::load_image_drawing_project_embedded( + project_path, + index, + handle, + cx, + ); + }); + } + cx.notify(); + window.refresh(); + } + + pub(crate) fn close_image_drawing(&mut self, window: &mut Window, cx: &mut Context) { + self.image_drawing_workspace = None; + self.focus_root(window, cx); + cx.notify(); + window.refresh(); + } + + pub(crate) fn commit_image_drawing( + &mut self, + index: usize, + source_relative: &str, + path: String, + annotations: Vec, + window: &mut Window, + cx: &mut Context, + ) -> bool { + let changed = self.edit( + |timeline| { + let Some(segment) = timeline.image_segments.get_mut(index) else { + return false; + }; + if segment.source_path.as_deref().unwrap_or(&segment.path) != source_relative { + return false; + } + segment.source_path = Some(source_relative.to_string()); + segment.path = path; + segment.annotations = annotations; + true + }, + window, + cx, + ); + if changed { + self.close_image_drawing(window, cx); + } + changed } pub fn set_summary( @@ -2276,6 +2420,7 @@ impl EditorWindow { self.history = ProjectHistory::new(self.project.clone()); self.tracks = TrackLanes::from_project(&self.project, self.has_camera); self.rebuild_timeline(); + self.queue_imported_waveforms(window, cx); // The sidebar's own signals are seeded from the config the instance // actually loaded, not the pre-flight's: `backgroundSourceTab`'s // initial value reads `background.padding`/`rounding` (`CS:1799-1802`). @@ -2293,6 +2438,7 @@ impl EditorWindow { dismiss_indexed_sidebar_menu(&mut self.sidebar.menu); let mic = std::mem::take(&mut self.timeline.mic_waveforms); let system = std::mem::take(&mut self.timeline.system_waveforms); + let imported = std::mem::take(&mut self.timeline.imported_waveforms); self.timeline = TimelineModel::build_with_lanes( &self.project, self.has_camera, @@ -2301,6 +2447,7 @@ impl EditorWindow { ); self.timeline.mic_waveforms = mic; self.timeline.system_waveforms = system; + self.timeline.imported_waveforms = imported; if self.timeline.total_duration > 0.0 { self.total = self.timeline.total_duration; } @@ -2323,6 +2470,65 @@ impl EditorWindow { window.refresh(); } + fn queue_imported_waveforms(&mut self, window: &mut Window, cx: &mut Context) { + let Some(timeline) = self.project.timeline.as_ref() else { + return; + }; + let paths = timeline + .audio_segments + .iter() + .map(|segment| segment.path.as_str()) + .chain( + timeline + .video_segments + .iter() + .map(|segment| segment.path.as_str()), + ) + .filter(|path| !path.is_empty()) + .map(str::to_string) + .collect::>(); + for path in paths { + if self.waveform_pending.contains(&path) + || self.timeline.imported_waveforms.contains_key(&path) + { + continue; + } + self.waveform_pending.insert(path.clone()); + let project_path = self.project_path.clone(); + let cancellation = self.waveform_cancellation.clone(); + let pending_path = path.clone(); + cx.spawn_in(window, async move |this, cx| { + let loaded = cx + .background_executor() + .spawn(async move { + let _slot = cap_audio::imported_waveform_slots() + .acquire() + .await + .map_err(|error| format!("Waveform worker unavailable: {error}"))?; + cap_audio::imported_waveform(&project_path, &path, cancellation) + .map(|peaks| (path, peaks)) + }) + .await; + this.update_in(cx, |this, window, cx| match loaded { + Ok((path, peaks)) => { + this.waveform_pending.remove(&pending_path); + this.timeline + .imported_waveforms + .insert(path, Arc::new(timeline::ImportedWaveform::new(peaks))); + cx.notify(); + window.refresh(); + } + Err(error) => { + this.waveform_pending.remove(&pending_path); + tracing::warn!(%error, "imported waveform unavailable"); + } + }) + .ok(); + }) + .detach(); + } + } + // -- Editing: the write path --------------------------------------------- /// Every mutation goes through here. @@ -2436,6 +2642,7 @@ impl EditorWindow { self.synchronize_caption_track(false); self.history.record(&self.project); self.rebuild_timeline(); + self.queue_imported_waveforms(window, cx); self.publish_project(); self.schedule_save(window, cx); cx.notify(); @@ -2953,6 +3160,7 @@ impl EditorWindow { request_frame(instance, initial_frame, self.preview_resolution()); } } + self.process_next_media_drop(window, cx); cx.notify(); window.refresh(); } @@ -3970,6 +4178,7 @@ impl EditorWindow { let index = selection.indices[0]; match selection.track { TrackKind::Image => Some(crate::editor_canvas::CanvasSelection::Image(index)), + TrackKind::Video => Some(crate::editor_canvas::CanvasSelection::Video(index)), TrackKind::Text => Some(crate::editor_canvas::CanvasSelection::Text(index)), TrackKind::Mask => Some(crate::editor_canvas::CanvasSelection::Mask(index)), _ => None, @@ -4281,6 +4490,7 @@ impl EditorWindow { kind, TrackKind::Style | TrackKind::Image + | TrackKind::Video | TrackKind::Text | TrackKind::Mask | TrackKind::Audio @@ -4403,9 +4613,11 @@ impl EditorWindow { return; } let valid_target = self.lane_reorder_position_valid(current.source, x, y); - let target_index = valid_target - .then(|| self.lane_reorder_target_index(current.source, y)) - .unwrap_or(current.target_index); + let target_index = if valid_target { + self.lane_reorder_target_index(current.source, y) + } else { + current.target_index + }; if promote { self.history.pause(); } @@ -5134,6 +5346,33 @@ impl EditorWindow { type GhostClipLayout = (Vec<(f64, f64)>, Option<(f64, f64)>); +enum ImportedDroppedMedia { + Image(crate::import::ImportedEditorImage), + Video(cap_media_info::video_import::ImportedVideo), +} + +fn spawn_editor_media_import( + project_path: PathBuf, + source: PathBuf, + video: bool, +) -> Result>, String> { + let (tx, rx) = flume::bounded(1); + std::thread::Builder::new() + .name("cap-editor-media-import".into()) + .spawn(move || { + let imported = if video { + cap_media_info::video_import::import_video(&project_path, &source) + .map(ImportedDroppedMedia::Video) + } else { + crate::import::import_editor_image(&project_path, &source) + .map(ImportedDroppedMedia::Image) + }; + let _ = tx.send(imported); + }) + .map_err(|error| format!("Cannot start media import worker: {error}"))?; + Ok(rx) +} + /// One transient line under the player. The editor has no toast host -- the /// screenshot editor's bubbles are its own -- and exactly one message at a /// time is all any editor action needs to say. @@ -6056,6 +6295,7 @@ impl EditorWindow { let count = match kind { TrackKind::Style => &mut self.tracks.style, TrackKind::Image => &mut self.tracks.image, + TrackKind::Video => &mut self.tracks.video, TrackKind::Text => &mut self.tracks.text, TrackKind::Mask => &mut self.tracks.mask, TrackKind::Audio => &mut self.tracks.audio, @@ -6065,6 +6305,7 @@ impl EditorWindow { let used = match (kind, self.project.timeline.as_ref()) { (TrackKind::Style, Some(timeline)) => edits::used_lane_count(&timeline.style_segments), (TrackKind::Image, Some(timeline)) => edits::used_lane_count(&timeline.image_segments), + (TrackKind::Video, Some(timeline)) => edits::used_lane_count(&timeline.video_segments), (TrackKind::Text, Some(timeline)) => edits::used_lane_count(&timeline.text_segments), (TrackKind::Mask, Some(timeline)) => edits::used_lane_count(&timeline.mask_segments), (TrackKind::Audio, Some(timeline)) => edits::used_lane_count(&timeline.audio_segments), @@ -6074,6 +6315,7 @@ impl EditorWindow { match kind { TrackKind::Style => self.tracks.style = next, TrackKind::Image => self.tracks.image = next, + TrackKind::Video => self.tracks.video = next, TrackKind::Text => self.tracks.text = next, TrackKind::Mask => self.tracks.mask = next, TrackKind::Audio => self.tracks.audio = next, @@ -6943,6 +7185,11 @@ impl EditorWindow { ) } else if self.clips.is_importing() { Some("Wait for the clip import to finish before closing or deleting this recording.") + } else if self.media_drop_active + || !self.media_drop_queue.is_empty() + || self.video_picker_active + { + Some("Wait for the media import to finish before closing or deleting this project.") } else { None } @@ -7258,6 +7505,7 @@ impl EditorWindow { self.rebuild_timeline(); self.open_audio_picker(lane, cx); } + TrackKind::Video => self.pick_timeline_video(window, cx), TrackKind::Text | TrackKind::Mask => { self.add_overlay_segment(kind, window, cx); } @@ -7580,6 +7828,176 @@ impl EditorWindow { cx.notify(); } + fn on_external_paths_drop( + &mut self, + paths: &ExternalPaths, + window: &mut Window, + cx: &mut Context, + ) { + let mut accepted = 0; + for path in &paths.0 { + if crate::import::is_supported_image_import_path(path) + || cap_media_info::video_import::is_supported_video_path(path) + { + self.media_drop_queue.push_back(path.clone()); + accepted += 1; + } + } + if accepted == 0 { + self.show_notice( + "Choose an image or video file to add to this editor", + window, + cx, + ); + return; + } + self.process_next_media_drop(window, cx); + } + + fn pick_timeline_video(&mut self, window: &mut Window, cx: &mut Context) { + if self.video_picker_active { + return; + } + self.video_picker_active = true; + let project_path = self.project_path.clone(); + let time = self.playhead; + cx.spawn_in(window, async move |this, cx| { + #[cfg(target_os = "linux")] + let source = crate::platform::open_file_panel_async( + &[("Videos", cap_media_info::video_import::VIDEO_EXTENSIONS)], + None, + cx, + ) + .await; + #[cfg(not(target_os = "linux"))] + let source = crate::import::pick_import_file( + &[("Videos", cap_media_info::video_import::VIDEO_EXTENSIONS)], + cx, + ) + .await; + let imported = match source { + Some(source) => Some( + match spawn_editor_media_import(project_path, source, true) { + Ok(receiver) => receiver.recv_async().await.unwrap_or_else(|error| { + Err(format!("Media import worker stopped: {error}")) + }), + Err(error) => Err(error), + }, + ), + None => None, + }; + this.update_in(cx, |this, window, cx| { + this.video_picker_active = false; + match imported { + Some(Ok(ImportedDroppedMedia::Video(imported))) => { + this.commit_video_import(imported, time, window, cx) + } + Some(Ok(ImportedDroppedMedia::Image(_))) => {} + Some(Err(error)) => this.show_notice(error, window, cx), + None => cx.notify(), + } + }) + .ok(); + }) + .detach(); + cx.notify(); + } + + fn process_next_media_drop(&mut self, window: &mut Window, cx: &mut Context) { + if self.media_drop_active || !self.project_ready() { + return; + } + let Some(source) = self.media_drop_queue.pop_front() else { + return; + }; + self.media_drop_active = true; + let project_path = self.project_path.clone(); + let time = self.playhead; + let video = cap_media_info::video_import::is_supported_video_path(&source); + let receiver = match spawn_editor_media_import(project_path, source, video) { + Ok(receiver) => receiver, + Err(error) => { + self.media_drop_active = false; + self.show_notice(error, window, cx); + self.process_next_media_drop(window, cx); + return; + } + }; + cx.spawn_in(window, async move |this, cx| { + let imported = receiver + .recv_async() + .await + .unwrap_or_else(|error| Err(format!("Media import worker stopped: {error}"))); + this.update_in(cx, |this, window, cx| { + this.media_drop_active = false; + match imported { + Ok(ImportedDroppedMedia::Image(imported)) => { + this.commit_image_import(this.tracks.image, time, imported, window, cx); + if let Some(error) = this.sidebar.image_import_error.clone() { + this.show_notice(error, window, cx); + } + } + Ok(ImportedDroppedMedia::Video(imported)) => { + this.commit_video_import(imported, time, window, cx); + } + Err(error) => this.show_notice(error, window, cx), + } + this.process_next_media_drop(window, cx); + }) + .ok(); + }) + .detach(); + cx.notify(); + } + + fn commit_video_import( + &mut self, + imported: cap_media_info::video_import::ImportedVideo, + time: f64, + window: &mut Window, + cx: &mut Context, + ) { + if !edits::ensure_timeline(&mut self.project, &self.clip_display_durations) { + self.show_notice("The editor timeline is unavailable", window, cx); + return; + } + let output = self + .frame_layout + .map_or([1920, 1080], |layout| layout.output_size); + let ratio = f64::from(imported.width) / f64::from(imported.height) * f64::from(output[1]) + / f64::from(output[0]); + let size = if ratio >= 1.0 { + XY::new(1.0, 1.0 / ratio) + } else { + XY::new(ratio, 1.0) + }; + let start = time.max(0.0); + let lane = self.tracks.video; + let mut index = 0; + if self.edit( + |timeline| { + timeline.video_segments.push(cap_project::VideoSegment { + start, + end: start + imported.duration, + track: lane, + path: imported.path, + name: imported.name, + source_duration: imported.duration, + size, + ..Default::default() + }); + index = timeline.video_segments.len() - 1; + true + }, + window, + cx, + ) { + self.tracks.video = self.tracks.video.max(lane + 1); + self.set_selection(Some(Selection::single(TrackKind::Video, index)), cx); + self.seek_to_time(start, cx); + } + } + fn commit_image_import( &mut self, mut lane: u32, @@ -7596,15 +8014,7 @@ impl EditorWindow { cx.notify(); return; } - let total = self.total_duration(); - if total <= 0.0 { - self.sidebar.image_import_error = Some( - "The recording is no longer available for this image. Please reopen the project." - .into(), - ); - cx.notify(); - return; - } + let total = self.total_duration().max(time.max(0.0) + 5.0); let Some(timeline) = self.project.timeline.as_ref() else { self.sidebar.image_import_error = Some( "The recording is no longer available for this image. Please reopen the project." @@ -10571,7 +10981,8 @@ impl EditorWindow { | TrackKind::Mask | TrackKind::Audio | TrackKind::Style - | TrackKind::Image => Some("Delete"), + | TrackKind::Image + | TrackKind::Video => Some("Delete"), TrackKind::Zoom | TrackKind::ThreeD | TrackKind::Scene => { (!model.segments(kind).is_empty()).then_some("Clear all") } @@ -10930,7 +11341,8 @@ impl EditorWindow { | TrackKind::Mask | TrackKind::Audio | TrackKind::Style - | TrackKind::Image => { + | TrackKind::Image + | TrackKind::Video => { this.delete_track_lane(kind, lane, window, cx); } TrackKind::Zoom @@ -11145,6 +11557,12 @@ fn is_playback_shortcut( impl Render for EditorWindow { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + if let Some(workspace) = &self.image_drawing_workspace { + return div().size_full().child(workspace.clone()); + } + if let Some(workspace) = &self.screenshot_workspace { + return div().size_full().child(workspace.clone()); + } self.sync_appearance(window, cx); let title_disabled = !self.visual_ready(); if self.name_input.read(cx).is_disabled() != title_disabled { @@ -11223,6 +11641,7 @@ impl Render for EditorWindow { .bg(self.root_bg()) .text_color(Hsla::from(theme.gray_12)) .track_focus(&self.focus) + .on_drop(cx.listener(Self::on_external_paths_drop)) .capture_key_down(cx.listener(Self::capture_playback_key)) .on_key_down(cx.listener(Self::on_key)) // Only the cropper needs key-*up*: its nudge loop runs until every @@ -11985,6 +12404,7 @@ mod tests { camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }), captions: Some(cap_project::CaptionsData { segments: vec![cap_project::CaptionSegment { @@ -12051,6 +12471,7 @@ mod tests { camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }), captions: Some(cap_project::CaptionsData { segments: vec![cap_project::CaptionSegment { @@ -12280,6 +12701,7 @@ mod tests { timeline: Some(TimelineConfiguration { style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), segments: Vec::new(), transitions: Vec::new(), zoom_segments: Vec::new(), @@ -12480,6 +12902,41 @@ mod tests { assert!(error.contains("recording meta"), "{error}"); } + #[test] + fn preflight_opens_screenshot_and_blank_media_projects_without_recording_clips() { + for screenshot in [true, false] { + let dir = std::env::temp_dir().join(format!( + "cap-gpui-media-preflight-{}.cap", + uuid::Uuid::new_v4() + )); + std::fs::create_dir_all(&dir).unwrap(); + if screenshot { + image::RgbaImage::from_pixel(80, 45, image::Rgba([12, 34, 56, 255])) + .save(dir.join("original.png")) + .unwrap(); + } + let meta = if screenshot { + serde_json::json!({ + "pretty_name": "Screenshot", + "display": { "path": "original.png", "fps": 0 }, + "camera": null, "audio": null, "cursor": null + }) + } else { + serde_json::json!({ + "pretty_name": "Media project", + "segments": [], + "status": { "status": "Complete" } + }) + }; + std::fs::write(dir.join("recording-meta.json"), meta.to_string()).unwrap(); + let summary = preflight(&dir).unwrap(); + assert!(summary.recordings.segments.is_empty()); + assert!(summary.timeline.clips.is_empty()); + assert_eq!(summary.duration, if screenshot { 5.0 } else { 0.0 }); + std::fs::remove_dir_all(dir).unwrap(); + } + } + // -- Playback ------------------------------------------------------------ /// `isAtEnd()` is `total > 0 && total - playbackTime <= 0.1` diff --git a/apps/desktop-gpui/src/import.rs b/apps/desktop-gpui/src/import.rs index 46c37db8302..8faf337bd1b 100644 --- a/apps/desktop-gpui/src/import.rs +++ b/apps/desktop-gpui/src/import.rs @@ -1,27 +1,11 @@ -//! Media import -- the gpui port of the Tauri binary's `import.rs`: a picked -//! video is transcoded into a fresh `.cap` studio bundle, a picked image -//! becomes a screenshot bundle, and progress is reported through a global the -//! main window's library panel draws. -//! -//! The Tauri version encodes through `cap-enc-ffmpeg` -//! (`H264EncoderBuilder` / `OpusEncoder`), which is not a dependency of this -//! standalone workspace -- the narrow slices of it the import path actually -//! exercises are transcribed here onto the same `ffmpeg-next` this app already -//! builds (each function cites its source). Progress travels the tray-channel -//! shape: the worker thread owns a `flume::Sender` and a foreground task -//! drains it into [`ActiveImports`] with a clean gpui borrow. - use std::{ + io::ErrorKind, path::{Path, PathBuf}, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, time::Duration, }; -use cap_project::{ - AudioMeta, Cursors, MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, - RecordingMeta, RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, - VideoMeta, -}; +use cap_project::ProjectConfiguration; use ffmpeg::{ChannelLayout, codec as avcodec, format as avformat}; use gpui::{App, Global}; @@ -35,8 +19,8 @@ const MEDIA_IMPORT_EXTENSIONS: &[&str] = &[ "mp4", "mov", "avi", "mkv", "webm", "wmv", "m4v", "flv", "png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff", ]; -pub(crate) const OVERLAY_IMAGE_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "webp", "gif", "bmp"]; -const MAX_IMAGE_DIMENSION: u32 = 16_384; +pub(crate) const OVERLAY_IMAGE_EXTENSIONS: &[&str] = + &["png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff"]; static ACTIVE_IMPORT_WORKERS: AtomicUsize = AtomicUsize::new(0); #[cfg(test)] @@ -79,73 +63,12 @@ fn generate_project_name(source_path: &Path, fallback: &str) -> String { format!("{stem} {}", now.format("%Y-%m-%d at %H.%M.%S")) } -/// `import.rs:125-132`. -fn sanitize_filename(name: &str) -> String { - name.chars() - .map(|c| match c { - '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', - _ => c, - }) - .collect() -} - -/// The `(1)`-suffix uniquing loop from `start_video_import` (`import.rs:1370-1376`). -fn unique_project_path(recordings_dir: &Path, sanitized_name: &str) -> PathBuf { - let mut project_path = recordings_dir.join(format!("{sanitized_name}.cap")); - let mut counter = 1; - while project_path.exists() { - project_path = recordings_dir.join(format!("{sanitized_name} ({counter}).cap")); - counter += 1; - } - project_path -} - fn check_project_exists(project_path: &Path) -> bool { project_path.exists() && project_path.join("recording-meta.json").exists() } /// The bundle both metas describe: `content/segments/segment-0/display.mp4` /// plus an optional sibling `audio.ogg` (`import.rs:1413-1439` and `1506-1536`). -fn imported_video_meta( - project_path: &Path, - pretty_name: &str, - fps: u32, - has_audio: bool, - status: StudioRecordingStatus, -) -> RecordingMeta { - RecordingMeta { - platform: Some(Platform::default()), - project_path: project_path.to_path_buf(), - pretty_name: pretty_name.to_string(), - sharing: None, - inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { - inner: MultipleSegments { - segments: vec![MultipleSegment { - display: VideoMeta { - path: "content/segments/segment-0/display.mp4".into(), - fps, - start_time: Some(0.0), - device_id: None, - }, - camera: None, - mic: None, - system_audio: has_audio.then(|| AudioMeta { - path: "content/segments/segment-0/audio.ogg".into(), - start_time: Some(0.0), - device_id: None, - gap_summary: None, - }), - cursor: None, - keyboard: None, - display_notch: None, - }], - cursors: Cursors::default(), - status: Some(status), - }, - })), - upload: None, - } -} // --------------------------------------------------------------------------- // Progress state -- what the main window's library panel draws @@ -163,7 +86,6 @@ pub enum ImportKind { pub enum ImportStage { Probing, Converting, - Finalizing, Complete, Failed, } @@ -240,15 +162,10 @@ fn apply_progress(update: ImportProgress, cx: &mut App) { if complete { refresh_libraries(cx); - // `importVideoPath` opens the editor on the imported bundle; the - // Tauri version opens it up front and shows `ImportProgress.tsx` - // inside, this app opens it once the bundle is Complete (deviation: - // the gpui editor has no importing screen). `importImagePath` ends in - // `ShowCapWindow::ScreenshotEditor` the same way. if cx.has_global::() { match kind { ImportKind::Video => crate::app_windows::open_editor(project_path, cx), - ImportKind::Image => crate::app_windows::open_screenshot_editor(project_path, cx), + ImportKind::Image => crate::app_windows::open_editor(project_path, cx), } } } @@ -463,173 +380,75 @@ impl ProgressSink<'_> { } } -/// `start_video_import` (`import.rs:1361-1599`), linearised: the Tauri command -/// does the probe inline and spawns the transcode; here the whole pipeline is -/// already on a worker thread. fn run_video_import(source_path: &Path, tx: &flume::Sender) { - let recordings_dir = crate::recording::recordings_dir(); + run_video_import_in(&crate::recording::recordings_dir(), source_path, tx); +} + +fn run_video_import_in(base: &Path, source_path: &Path, tx: &flume::Sender) { let project_name = generate_project_name(source_path, "Imported Video"); - let project_path = unique_project_path(&recordings_dir, &sanitize_filename(&project_name)); + let project_path = match cap_project::create_media_project(base, &project_name) { + Ok(path) => path, + Err(error) => { + let _ = tx.send(ImportProgress { + kind: ImportKind::Video, + project_path: source_path.to_path_buf(), + pretty_name: project_name, + stage: ImportStage::Failed, + progress: 0.0, + message: error, + }); + return; + } + }; let sink = ProgressSink { tx, kind: ImportKind::Video, project_path: &project_path, pretty_name: &project_name, }; - - sink.send(ImportStage::Probing, 0.0, "Analyzing video file..."); - match probe_video_can_decode(source_path) { - Ok(true) => {} - Ok(false) => { - sink.send( - ImportStage::Failed, - 0.0, - "Video format not supported or file is corrupted", - ); - return; + sink.send(ImportStage::Probing, 0.0, "Importing video..."); + let result = (|| { + let imported = cap_media_info::video_import::import_video(&project_path, source_path)?; + let asset_path = project_path.join(&imported.path); + let mut config = ProjectConfiguration::load(&project_path) + .map_err(|error| format!("Cannot load media timeline: {error}"))?; + config + .timeline + .get_or_insert_with(cap_project::TimelineConfiguration::default) + .video_segments + .push(cap_project::VideoSegment { + end: imported.duration, + path: imported.path, + name: imported.name, + source_duration: imported.duration, + muted: !imported.has_audio, + ..Default::default() + }); + config + .write(&project_path) + .map_err(|error| format!("Cannot save imported video timeline: {error}"))?; + let thumbnail = crate::library::bundle_thumbnail_path(&project_path); + if let Err(error) = crate::library::create_screenshot(&asset_path, &thumbnail, None) { + tracing::warn!(%error, "could not create imported video thumbnail"); } - Err(error) => { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Cannot decode video: {error}"), - ); - return; - } - } - - let segment_dir = project_path - .join("content") - .join("segments") - .join("segment-0"); - if let Err(error) = std::fs::create_dir_all(&segment_dir) { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to create project directory: {error}"), - ); - return; - } - - let output_video_path = segment_dir.join("display.mp4"); - let output_audio_path = segment_dir.join("audio.ogg"); - - // The InProgress meta first, so the library lists the bundle with its - // "In progress" badge while the conversion runs. - if let Err(error) = imported_video_meta( - &project_path, - &project_name, - 30, - false, - StudioRecordingStatus::InProgress, - ) - .save_for_project() - { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to save initial metadata: {error:?}"), - ); - return; - } - - sink.send(ImportStage::Converting, 0.0, "Starting conversion..."); - let result = transcode_video( - source_path, - &output_video_path, - Some(&output_audio_path), - &project_path, - &|progress| { - sink.send( - ImportStage::Converting, - progress, - &format!("Converting video... {}%", (progress * 100.0) as u32), - ); - }, - None, - ); - - let (fps, sample_rate) = match result { - Ok(result) => result, - Err(error) => { - if error == IMPORT_CANCELLED { - tracing::info!("video import cancelled"); - } else { - tracing::error!("video import transcode failed: {error}"); - // The Tauri version leaves the InProgress meta behind, which - // reads as a recording that never finishes; a Failed status - // gives the library its "Recording failed" badge instead. - if check_project_exists(&project_path) - && let Err(save_error) = imported_video_meta( - &project_path, - &project_name, - 30, - false, - StudioRecordingStatus::Failed { - error: error.clone(), - }, - ) - .save_for_project() - { - tracing::warn!("could not mark the import failed: {save_error:?}"); - } - } - sink.send(ImportStage::Failed, 0.0, &error); - return; - } - }; - - sink.send( - ImportStage::Finalizing, - 0.95, - "Creating project metadata...", - ); - - // `import.rs:1490-1504`: an Opus file this small is headers with no - // samples, so the meta must not point playback at it. - const MIN_VALID_AUDIO_SIZE: u64 = 1000; - let audio_file_size = std::fs::metadata(&output_audio_path) - .map(|metadata| metadata.len()) - .unwrap_or(0); - let has_audio = sample_rate.is_some() && audio_file_size > MIN_VALID_AUDIO_SIZE; - - if let Err(error) = imported_video_meta( - &project_path, - &project_name, - fps, - has_audio, - StudioRecordingStatus::Complete, - ) - .save_for_project() - { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to save metadata: {error:?}"), - ); + Ok::<(), String>(()) + })(); + if let Err(error) = result { + let _ = std::fs::remove_dir_all(&project_path); + sink.send(ImportStage::Failed, 0.0, &error); return; } - - // Written before Complete rather than fire-and-forget as the Tauri spawn - // does, so the refresh that Complete triggers already finds the file. - let thumbnail = crate::library::bundle_thumbnail_path(&project_path); - if let Err(error) = crate::library::create_screenshot(&output_video_path, &thumbnail, None) { - tracing::warn!("could not write the imported video's thumbnail: {error}"); - } - sink.send(ImportStage::Complete, 1.0, "Import complete!"); tracing::info!(path = %project_path.display(), "video import complete"); } -/// `start_image_import` (`import.rs:1899-2017`). fn run_image_import(source_path: &Path, tx: &flume::Sender) { - let screenshots_dir = crate::library::screenshots_dir(); - let project_name = generate_project_name(source_path, "Imported Image"); - // `import.rs:1947-1948`: `:` becomes `.` the way recording bundles spell - // timestamps, then the reserved characters go. - let bundle_name = format!("{}.cap", sanitize_filename(&project_name.replace(':', "."))); + run_image_import_in(&crate::recording::recordings_dir(), source_path, tx); +} - let placeholder_path = screenshots_dir.join(&bundle_name); +fn run_image_import_in(base: &Path, source_path: &Path, tx: &flume::Sender) { + let project_name = generate_project_name(source_path, "Imported Image"); + let placeholder_path = base.join(format!("{}.pending", uuid::Uuid::new_v4())); let early = ProgressSink { tx, kind: ImportKind::Image, @@ -641,17 +460,8 @@ fn run_image_import(source_path: &Path, tx: &flume::Sender) { early.send(ImportStage::Failed, 0.0, "Image file does not exist"); return; } - if let Err(error) = std::fs::create_dir_all(&screenshots_dir) { - early.send( - ImportStage::Failed, - 0.0, - &format!("Failed to create screenshots directory: {error}"), - ); - return; - } - - let project_path = match cap_utils::ensure_unique_filename(&bundle_name, &screenshots_dir) { - Ok(name) => screenshots_dir.join(name), + let project_path = match cap_project::create_media_project(base, &project_name) { + Ok(path) => path, Err(error) => { early.send(ImportStage::Failed, 0.0, &error); return; @@ -666,149 +476,74 @@ fn run_image_import(source_path: &Path, tx: &flume::Sender) { sink.send(ImportStage::Probing, 0.0, "Importing image..."); - let (width, height, rgba) = match decode_image_rgba(source_path) { - Ok(decoded) => decoded, - Err(error) => { - sink.send(ImportStage::Failed, 0.0, &error); - return; - } - }; - - if let Err(error) = std::fs::create_dir_all(&project_path) { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to create screenshot project directory: {error}"), - ); - return; - } - - let image_path = project_path.join("original.png"); - if let Err(error) = write_png(&image_path, width, height, &rgba) { + let result = (|| { + let imported = import_editor_image(&project_path, source_path)?; + let mut config = ProjectConfiguration::load(&project_path) + .map_err(|error| format!("Cannot load media timeline: {error}"))?; + config + .timeline + .get_or_insert_with(cap_project::TimelineConfiguration::default) + .image_segments + .push(cap_project::ImageSegment { + end: 5.0, + path: imported.path.clone(), + size: cap_project::XY::new(1.0, 1.0), + ..Default::default() + }); + config + .write(&project_path) + .map_err(|error| format!("Cannot save imported image timeline: {error}"))?; + create_image_thumbnail( + &project_path.join(imported.path), + &project_path.join("screenshots/display.jpg"), + ) + })(); + if let Err(error) = result { + let _ = std::fs::remove_dir_all(&project_path); sink.send(ImportStage::Failed, 0.0, &error); return; } - // The bundle shape `list_screenshots` scans for: a `.cap` directory whose - // meta parses, holding a PNG (`import.rs:1981-2009`). - let meta = RecordingMeta { - platform: Some(Platform::default()), - project_path: project_path.clone(), - pretty_name: project_name.clone(), - sharing: None, - inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::SingleSegment { - segment: SingleSegment { - display: VideoMeta { - path: "original.png".into(), - fps: 0, - start_time: Some(0.0), - device_id: None, - }, - camera: None, - audio: None, - cursor: None, - }, - })), - upload: None, - }; - if let Err(error) = meta.save_for_project() { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to save screenshot metadata: {error:?}"), - ); - return; - } - if let Err(error) = ProjectConfiguration::default().write(&project_path) { - sink.send( - ImportStage::Failed, - 0.0, - &format!("Failed to save screenshot project config: {error}"), - ); - return; - } - sink.send(ImportStage::Complete, 1.0, "Import complete!"); tracing::info!(path = %project_path.display(), "image import complete"); } +fn create_image_thumbnail(path: &Path, thumbnail: &Path) -> Result<(), String> { + use image::ImageDecoder; + let mut reader = image::ImageReader::open(path) + .map_err(|error| format!("Cannot open imported image thumbnail: {error}"))? + .with_guessed_format() + .map_err(|error| format!("Cannot identify imported image thumbnail: {error}"))?; + let mut limits = image::Limits::default(); + limits.max_alloc = Some(128 * 1024 * 1024); + limits.max_image_width = Some(32_768); + limits.max_image_height = Some(32_768); + reader.limits(limits); + let mut decoder = reader + .into_decoder() + .map_err(|error| format!("Cannot decode imported image thumbnail: {error}"))?; + let orientation = decoder + .orientation() + .map_err(|error| format!("Cannot orient imported image thumbnail: {error}"))?; + let mut image = image::DynamicImage::from_decoder(decoder) + .map_err(|error| format!("Cannot read imported image thumbnail: {error}"))?; + image.apply_orientation(orientation); + std::fs::create_dir_all( + thumbnail + .parent() + .ok_or("Imported image thumbnail has no directory")?, + ) + .map_err(|error| format!("Cannot create imported image thumbnail directory: {error}"))?; + image + .thumbnail(400, 225) + .save_with_format(thumbnail, image::ImageFormat::Jpeg) + .map_err(|error| format!("Cannot save imported image thumbnail: {error}")) +} + // --------------------------------------------------------------------------- // Probing // --------------------------------------------------------------------------- -/// `probe_video_can_decode` (`crates/enc-ffmpeg/src/remux.rs:322-390`), minus -/// the log suppression that crate wraps around it. -fn probe_video_can_decode(path: &Path) -> Result { - let input = avformat::input(path).map_err(|e| format!("Failed to open file: {e}"))?; - - let input_stream = input - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or_else(|| "No video stream found".to_string())?; - - let decoder_ctx = avcodec::Context::from_parameters(input_stream.parameters()) - .map_err(|e| format!("Failed to create decoder context: {e}"))?; - let mut decoder = decoder_ctx - .decoder() - .video() - .map_err(|e| format!("Failed to create video decoder: {e}"))?; - - let stream_index = input_stream.index(); - - let mut input = avformat::input(path).map_err(|e| format!("Failed to reopen file: {e}"))?; - - let mut frame = ffmpeg::frame::Video::empty(); - let mut packets_tried = 0; - const MAX_PACKETS: usize = 100; - - for (stream, packet) in input.packets() { - if stream.index() != stream_index { - continue; - } - - packets_tried += 1; - - if let Err(e) = decoder.send_packet(&packet) { - if packets_tried >= MAX_PACKETS { - return Err(format!( - "Failed to send packet after {packets_tried} attempts: {e}" - )); - } - continue; - } - - match decoder.receive_frame(&mut frame) { - Ok(()) => return Ok(true), - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => continue, - Err(ffmpeg::Error::Eof) => break, - Err(e) => { - if packets_tried >= MAX_PACKETS { - return Err(format!( - "Failed to decode frame after {packets_tried} packets: {e}" - )); - } - continue; - } - } - } - - if let Err(e) = decoder.send_eof() { - return Err(format!("Failed to send EOF: {e}")); - } - - loop { - match decoder.receive_frame(&mut frame) { - Ok(()) => return Ok(true), - Err(ffmpeg::Error::Eof) => break, - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => continue, - Err(e) => return Err(format!("Failed to receive frame after EOF: {e}")), - } - } - - Ok(false) -} - -/// `get_media_duration` (`remux.rs:543-557`). fn media_duration(path: &Path) -> Option { let input = avformat::input(path).ok()?; let duration = input.duration(); @@ -1745,142 +1480,96 @@ impl OpusOutput { } } -// --------------------------------------------------------------------------- -// Image decode + PNG encode -// --------------------------------------------------------------------------- - -fn check_image_dimensions(width: u32, height: u32) -> Result<(), String> { - if width > MAX_IMAGE_DIMENSION || height > MAX_IMAGE_DIMENSION { - return Err(format!("Image dimensions exceed maximum: {width}x{height}")); - } - if width - .checked_mul(height) - .and_then(|pixels| pixels.checked_mul(4)) - .is_none() - { - return Err(format!("Image dimensions overflow: {width}x{height}")); - } - Ok(()) -} +#[cfg(test)] +mod tests { + use super::*; + use cap_project::{RecordingMeta, StudioRecordingMeta}; -/// `start_image_import`'s decode (`import.rs:1908-1933`), with an ffmpeg -/// fallback: this workspace's `image` build only carries png/jpeg/webp -/// (Cargo.toml pins the features), so gif/bmp/tiff decode through the same -/// ffmpeg stack the video path uses. -fn decode_image_rgba(source_path: &Path) -> Result<(u32, u32, Vec), String> { - match decode_image_with_image_crate(source_path) { - Ok(decoded) => Ok(decoded), - Err(image_error) => decode_image_with_ffmpeg(source_path) - .map_err(|ffmpeg_error| format!("{image_error} ({ffmpeg_error})")), + #[test] + fn direct_video_import_creates_media_timeline_with_source_audio_and_preserves_input() { + let root = temp_dir("direct-video"); + let base = root.join("library"); + let source = root.join("source.mp4"); + let original = + include_bytes!("../../media-server/src/__tests__/fixtures/test-with-audio.mp4"); + std::fs::write(&source, original).unwrap(); + let source_permissions = std::fs::metadata(&source).unwrap().permissions(); + let mut read_only_permissions = source_permissions.clone(); + read_only_permissions.set_readonly(true); + std::fs::set_permissions(&source, read_only_permissions).unwrap(); + let (tx, rx) = flume::unbounded(); + run_video_import_in(&base, &source, &tx); + let events: Vec<_> = rx.try_iter().collect(); + let completed = events + .iter() + .find(|event| event.stage == ImportStage::Complete) + .unwrap_or_else(|| panic!("video import failed: {events:?}")); + let project_path = &completed.project_path; + let meta = RecordingMeta::load_for_project(project_path).unwrap(); + assert!(matches!( + meta.studio_meta().unwrap(), + StudioRecordingMeta::MultipleSegments { inner } if inner.segments.is_empty() + )); + let config = ProjectConfiguration::load(project_path).unwrap(); + let timeline = config.timeline.unwrap(); + assert_eq!(timeline.video_segments.len(), 1); + let video = &timeline.video_segments[0]; + assert!(video.end > 0.0 && !video.muted); + assert!(crate::library::bundle_thumbnail_path(project_path).is_file()); + assert_eq!( + std::fs::read(project_path.join(&video.path)).unwrap(), + original + ); + assert_eq!(std::fs::read(&source).unwrap(), original); + assert!(std::fs::metadata(&source).unwrap().permissions().readonly()); + std::fs::set_permissions(&source, source_permissions).unwrap(); + let damaged = root.join("damaged.mp4"); + std::fs::write(&damaged, b"invalid").unwrap(); + run_video_import_in(&base, &damaged, &tx); + assert_eq!(std::fs::read_dir(&base).unwrap().count(), 1); + std::fs::remove_dir_all(root).unwrap(); } -} -fn decode_image_with_image_crate(source_path: &Path) -> Result<(u32, u32, Vec), String> { - let image = image::ImageReader::open(source_path) - .map_err(|e| format!("Failed to open image: {e}"))? - .with_guessed_format() - .map_err(|e| format!("Failed to detect image format: {e}"))? - .decode() - .map_err(|e| format!("Failed to decode image: {e}"))?; - - let (width, height) = (image.width(), image.height()); - check_image_dimensions(width, height)?; - Ok((width, height, image.to_rgba8().into_raw())) -} - -fn decode_image_with_ffmpeg(source_path: &Path) -> Result<(u32, u32, Vec), String> { - let mut input = - avformat::input(source_path).map_err(|e| format!("Failed to open image: {e}"))?; - let stream_index = input - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("Failed to decode image: no image stream")? - .index(); - let mut decoder = avcodec::Context::from_parameters( - input - .stream(stream_index) - .ok_or("Failed to decode image: no image stream")? - .parameters(), - ) - .map_err(|e| format!("Failed to decode image: {e}"))? - .decoder() - .video() - .map_err(|e| format!("Failed to decode image: {e}"))?; - - let mut frame = ffmpeg::frame::Video::empty(); - let mut decoded = false; - for (stream, packet) in input.packets() { - if stream.index() != stream_index { - continue; - } - if decoder.send_packet(&packet).is_err() { - continue; - } - if decoder.receive_frame(&mut frame).is_ok() { - decoded = true; - break; - } - } - if !decoded { - decoder - .send_eof() - .map_err(|e| format!("Failed to decode image: {e}"))?; - decoded = decoder.receive_frame(&mut frame).is_ok(); - } - if !decoded { - return Err("Failed to decode image".to_string()); + #[test] + fn direct_image_import_creates_media_timeline_and_preserves_input() { + let root = temp_dir("direct-image"); + let base = root.join("library"); + let source = root.join("source.png"); + image::RgbaImage::from_pixel(24, 16, image::Rgba([25, 80, 210, 255])) + .save(&source) + .unwrap(); + let original = std::fs::read(&source).unwrap(); + let (tx, rx) = flume::unbounded(); + run_image_import_in(&base, &source, &tx); + let events: Vec<_> = rx.try_iter().collect(); + let completed = events + .iter() + .find(|event| event.stage == ImportStage::Complete) + .unwrap(); + let project_path = &completed.project_path; + let meta = RecordingMeta::load_for_project(project_path).unwrap(); + assert!(matches!( + meta.studio_meta().unwrap(), + StudioRecordingMeta::MultipleSegments { inner } if inner.segments.is_empty() + )); + let config = ProjectConfiguration::load(project_path).unwrap(); + let timeline = config.timeline.unwrap(); + assert_eq!(timeline.image_segments.len(), 1); + let image = &timeline.image_segments[0]; + assert_eq!(image.end, 5.0); + assert!(crate::library::bundle_thumbnail_path(project_path).is_file()); + assert_eq!( + std::fs::read(project_path.join(&image.path)).unwrap(), + original + ); + assert_eq!(std::fs::read(&source).unwrap(), original); + let damaged = root.join("damaged.png"); + std::fs::write(&damaged, b"invalid").unwrap(); + run_image_import_in(&base, &damaged, &tx); + assert_eq!(std::fs::read_dir(&base).unwrap().count(), 1); + std::fs::remove_dir_all(root).unwrap(); } - let (width, height) = (frame.width(), frame.height()); - check_image_dimensions(width, height)?; - - let mut scaler = ffmpeg::software::scaling::Context::get( - frame.format(), - width, - height, - avformat::Pixel::RGBA, - width, - height, - ffmpeg::software::scaling::Flags::BILINEAR, - ) - .map_err(|e| format!("Failed to decode image: {e}"))?; - let mut rgba_frame = ffmpeg::frame::Video::empty(); - scaler - .run(&frame, &mut rgba_frame) - .map_err(|e| format!("Failed to decode image: {e}"))?; - - // Row by row: the scaler pads rows to its own stride, the buffer is tight - // (the `create_screenshot` copy, `library.rs:779-787`). - let width_usize = width as usize; - let height_usize = height as usize; - let src_stride = rgba_frame.stride(0); - let row_bytes = width_usize * 4; - let mut buffer = vec![0u8; height_usize * row_bytes]; - for y in 0..height_usize { - let src = &rgba_frame.data(0)[y * src_stride..y * src_stride + row_bytes]; - buffer[y * row_bytes..(y + 1) * row_bytes].copy_from_slice(src); - } - - Ok((width, height, buffer)) -} - -/// The PNG write (`import.rs:1960-1977`). -fn write_png(path: &Path, width: u32, height: u32, rgba: &[u8]) -> Result<(), String> { - let file = std::fs::File::create(path) - .map_err(|e| format!("Failed to create imported image file: {e}"))?; - let encoder = image::codecs::png::PngEncoder::new_with_quality( - std::io::BufWriter::new(file), - image::codecs::png::CompressionType::Default, - image::codecs::png::FilterType::Adaptive, - ); - image::ImageEncoder::write_image(encoder, rgba, width, height, image::ColorType::Rgba8.into()) - .map_err(|e| format!("Failed to encode imported image: {e}")) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] fn editor_import_cancellation_interrupts_a_real_conversion() { ffmpeg::init().unwrap(); @@ -1982,15 +1671,6 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } - #[test] - fn sanitize_filename_replaces_reserved_characters() { - assert_eq!( - sanitize_filename(r#"a/b\c:d*e?f"gi|j"#), - "a_b_c_d_e_f_g_h_i_j" - ); - assert_eq!(sanitize_filename("My Clip 2026"), "My Clip 2026"); - } - #[test] fn project_names_carry_the_source_stem_and_a_timestamp() { let name = generate_project_name(Path::new("/tmp/My Clip.mp4"), "Imported Video"); @@ -2001,24 +1681,6 @@ mod tests { assert!(fallback.starts_with("Imported Video "), "{fallback}"); } - #[test] - fn unique_project_path_suffixes_like_the_tauri_import() { - let dir = temp_dir("unique"); - - let first = unique_project_path(&dir, "Video 2026-01-01 at 10.00.00"); - assert!(first.ends_with("Video 2026-01-01 at 10.00.00.cap")); - std::fs::create_dir_all(&first).unwrap(); - - let second = unique_project_path(&dir, "Video 2026-01-01 at 10.00.00"); - assert!(second.ends_with("Video 2026-01-01 at 10.00.00 (1).cap")); - std::fs::create_dir_all(&second).unwrap(); - - let third = unique_project_path(&dir, "Video 2026-01-01 at 10.00.00"); - assert!(third.ends_with("Video 2026-01-01 at 10.00.00 (2).cap")); - - std::fs::remove_dir_all(&dir).ok(); - } - #[test] fn opus_output_rate_selection_matches_the_tauri_encoder() { let supported = [8_000, 12_000, 16_000, 24_000, 48_000]; @@ -2089,116 +1751,152 @@ pub(crate) struct ImportedEditorImage { pub height: u32, } +fn reject_linked_image_asset_directories( + project_path: &Path, + directory: &Path, +) -> Result<(), String> { + for candidate in [project_path.join("content"), directory.to_path_buf()] { + match std::fs::symlink_metadata(&candidate) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("Image assets cannot use linked project directories".into()); + } + Ok(_) => {} + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(format!("Cannot inspect image assets: {error}")), + } + } + Ok(()) +} + +fn check_image_asset_directory(project_path: &Path, directory: &Path) -> Result<(), String> { + reject_linked_image_asset_directories(project_path, directory)?; + let root = project_path + .canonicalize() + .map_err(|error| format!("Cannot inspect editor project: {error}"))?; + let resolved = directory + .canonicalize() + .map_err(|error| format!("Cannot inspect image assets: {error}"))?; + if !resolved.starts_with(root) { + return Err("Image assets must stay inside the editor project".into()); + } + Ok(()) +} + pub(crate) fn import_editor_image( project_path: &Path, source: &Path, ) -> Result { use image::ImageDecoder; - use std::{ - hash::BuildHasher, - io::{Read, Write}, - }; + use std::io::Read; const MAX_BYTES: u64 = 64 * 1024 * 1024; if !project_path.is_dir() || !has_supported_extension(source, OVERLAY_IMAGE_EXTENSIONS) { - return Err("Choose a PNG, JPEG, WebP, GIF or BMP image for this project".into()); + return Err("Choose a PNG, JPEG, WebP, GIF, BMP or TIFF image for this project".into()); } let file = std::fs::File::open(source).map_err(|error| error.to_string())?; let metadata = file.metadata().map_err(|error| error.to_string())?; if !metadata.is_file() || metadata.len() > MAX_BYTES { return Err("Image files must be 64 MiB or smaller".into()); } - let mut encoded = Vec::new(); - file.take(MAX_BYTES + 1) - .read_to_end(&mut encoded) - .map_err(|error| error.to_string())?; - if encoded.len() as u64 > MAX_BYTES { - return Err("Image files must be 64 MiB or smaller".into()); - } - let mut reader = image::ImageReader::new(std::io::Cursor::new(&encoded)) + let id = uuid::Uuid::new_v4(); + let directory = project_path.join("content/images"); + reject_linked_image_asset_directories(project_path, &directory)?; + std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?; + check_image_asset_directory(project_path, &directory)?; + let temporary = directory.join(format!(".{id}.import")); + let result = (|| { + let mut temporary_file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| error.to_string())?; + let copied = std::io::copy(&mut file.take(MAX_BYTES + 1), &mut temporary_file) + .map_err(|error| format!("Failed to copy image: {error}"))?; + if copied > MAX_BYTES { + return Err("Image files must be 64 MiB or smaller".into()); + } + temporary_file + .sync_all() + .map_err(|error| format!("Failed to save image: {error}"))?; + drop(temporary_file); + let mut reader = image::ImageReader::new(std::io::BufReader::new( + std::fs::File::open(&temporary).map_err(|error| error.to_string())?, + )) .with_guessed_format() .map_err(|error| error.to_string())?; - let extension = match reader.format() { - Some(image::ImageFormat::Png) => "png", - Some(image::ImageFormat::Jpeg) => "jpg", - Some(image::ImageFormat::WebP) => "webp", - Some(image::ImageFormat::Gif) => "gif", - Some(image::ImageFormat::Bmp) => "bmp", - _ => return Err("Choose a PNG, JPEG, WebP, GIF or BMP image".into()), - }; - let mut limits = image::Limits::default(); - limits.max_alloc = Some(128 * 1024 * 1024); - limits.max_image_width = Some(32_768); - limits.max_image_height = Some(32_768); - reader.limits(limits); - let mut decoder = reader.into_decoder().map_err(|error| { - format!("Cannot decode image (maximum 32,768 pixels per side): {error}") - })?; - let (source_width, source_height) = decoder.dimensions(); - if source_width == 0 - || source_height == 0 - || u64::from(source_width) * u64::from(source_height) > 16_777_216 - || decoder.total_bytes() > 128 * 1024 * 1024 - { - return Err("Images must have at most 16,777,216 pixels (32,768 per side) and decode to at most 128 MiB".into()); - } - let orientation = decoder.orientation().map_err(|error| error.to_string())?; - let mut decoded = image::DynamicImage::from_decoder(decoder) - .map_err(|error| format!("Cannot decode image: {error}"))?; - decoded.apply_orientation(orientation); - let (width, height) = (decoded.width(), decoded.height()); - drop(decoded); - let mut bytes = [0u8; 16]; - for chunk in bytes.chunks_exact_mut(8) { - chunk.copy_from_slice( - &std::collections::hash_map::RandomState::new() - .hash_one(source) - .to_be_bytes(), - ); + let extension = match reader.format() { + Some(image::ImageFormat::Png) => "png", + Some(image::ImageFormat::Jpeg) => "jpg", + Some(image::ImageFormat::WebP) => "webp", + Some(image::ImageFormat::Gif) => "gif", + Some(image::ImageFormat::Bmp) => "bmp", + Some(image::ImageFormat::Tiff) => "tiff", + _ => return Err("Choose a PNG, JPEG, WebP, GIF, BMP or TIFF image".into()), + }; + let mut limits = image::Limits::default(); + limits.max_alloc = Some(128 * 1024 * 1024); + limits.max_image_width = Some(32_768); + limits.max_image_height = Some(32_768); + reader.limits(limits); + let mut decoder = reader + .into_decoder() + .map_err(|error| format!("Cannot decode image: {error}"))?; + let (source_width, source_height) = decoder.dimensions(); + if source_width == 0 + || source_height == 0 + || u64::from(source_width) * u64::from(source_height) > 16_777_216 + || decoder.total_bytes() > 128 * 1024 * 1024 + { + return Err("Images must have at most 16,777,216 pixels (32,768 per side) and decode to at most 128 MiB".into()); + } + let orientation = decoder.orientation().map_err(|error| error.to_string())?; + let mut decoded = image::DynamicImage::from_decoder(decoder) + .map_err(|error| format!("Cannot decode image: {error}"))?; + decoded.apply_orientation(orientation); + let (width, height) = (decoded.width(), decoded.height()); + let relative = format!("content/images/{id}.{extension}"); + check_image_asset_directory(project_path, &directory)?; + std::fs::rename(&temporary, project_path.join(&relative)) + .map_err(|error| format!("Failed to save image: {error}"))?; + Ok(ImportedEditorImage { + path: relative, + name: source + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("Image") + .to_string(), + width, + height, + }) + })(); + if result.is_err() { + let _ = std::fs::remove_file(temporary); } - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); - let id = format!( - "{}-{}-{}-{}-{}", - &hex[..8], - &hex[8..12], - &hex[12..16], - &hex[16..20], - &hex[20..] - ); - let relative = format!("content/images/{id}.{extension}"); - let destination = project_path.join(&relative); - std::fs::create_dir_all(project_path.join("content/images")) - .map_err(|error| error.to_string())?; - let mut destination_file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&destination) - .map_err(|error| error.to_string())?; - if let Err(error) = destination_file - .write_all(&encoded) - .and_then(|()| destination_file.sync_all()) - { - drop(destination_file); - let _ = std::fs::remove_file(&destination); - return Err(format!("Failed to save image: {error}")); - } - Ok(ImportedEditorImage { - path: relative, - name: source - .file_stem() - .and_then(|name| name.to_str()) - .unwrap_or("Image") - .to_string(), - width, - height, - }) + result } #[cfg(test)] mod style_image_tests { use super::*; + #[cfg(unix)] + #[test] + fn style_image_import_rejects_linked_asset_directory_without_writing_outside() { + let root = std::env::temp_dir().join(format!("cap-overlay-link-{}", uuid::Uuid::new_v4())); + let project = root.join("project"); + let outside = root.join("outside"); + std::fs::create_dir_all(project.join("content")).unwrap(); + std::fs::create_dir(&outside).unwrap(); + std::os::unix::fs::symlink(&outside, project.join("content/images")).unwrap(); + let source = root.join("source.png"); + image::RgbaImage::from_pixel(4, 3, image::Rgba([25, 80, 210, 255])) + .save(&source) + .unwrap(); + + assert!(import_editor_image(&project, &source).is_err()); + assert_eq!(std::fs::read_dir(&outside).unwrap().count(), 0); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn style_image_import_rotates_exif_copies_source_and_keeps_relative_unique_assets() { let dir = std::env::temp_dir().join(format!( diff --git a/apps/desktop-gpui/src/presets.rs b/apps/desktop-gpui/src/presets.rs index 1ca45d8150e..6bfa6b3ead8 100644 --- a/apps/desktop-gpui/src/presets.rs +++ b/apps/desktop-gpui/src/presets.rs @@ -247,6 +247,7 @@ mod tests { camera3d_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), }), overlay_order: vec![cap_project::OverlayTrack { kind: cap_project::OverlayTrackKind::Image, diff --git a/apps/desktop-gpui/src/screenshot_editor.rs b/apps/desktop-gpui/src/screenshot_editor.rs index 223d67da613..6a4c1adef74 100644 --- a/apps/desktop-gpui/src/screenshot_editor.rs +++ b/apps/desktop-gpui/src/screenshot_editor.rs @@ -20,14 +20,16 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; +use std::io::Read; use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::Arc; use std::time::Duration; use cap_project::{ - BackgroundSource, BorderConfiguration, CornerStyle, ProjectConfiguration, RecordingMeta, - RecordingMetaInner, ShadowConfiguration, StudioRecordingMeta, + AspectRatio, BackgroundConfiguration, BackgroundSource, BorderConfiguration, CornerStyle, + ProjectConfiguration, RecordingMeta, RecordingMetaInner, ShadowConfiguration, SingleSegment, + StudioRecordingMeta, VideoMeta, }; use cap_rendering::{ DecodedFrame, DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RenderOptions, @@ -40,22 +42,19 @@ use gpui::{ StatefulInteractiveElement as _, Styled, StyledImage as _, Window, WindowHandle, canvas, div, img, linear_color_stop, linear_gradient, prelude::FluentBuilder as _, px, svg, }; +use relative_path::RelativePathBuf; +use serde::{Deserialize, Serialize}; use crate::editor_edits::ProjectHistory; use crate::editor_sidebar::{ self, BACKGROUND_COLORS, BACKGROUND_IMAGE_EXTENSIONS, BACKGROUND_THEMES, DEFAULT_GRADIENT_FROM, DEFAULT_GRADIENT_TO, GRADIENT_PRESETS, color_to_hsla, hex_to_rgb, }; +use crate::editor_window::EditorWindow; use crate::screenshot_annotations::{self as annotations, AnnotationState, Tool}; use crate::theme::Theme; use crate::ui; -/// `ShowCapWindow::ScreenshotEditor`: 1240x800, min 800x600, resizable. -pub const SCREENSHOT_EDITOR_WIDTH: f32 = 1240.; -pub const SCREENSHOT_EDITOR_HEIGHT: f32 = 800.; -pub const SCREENSHOT_EDITOR_MIN_WIDTH: f32 = 800.; -pub const SCREENSHOT_EDITOR_MIN_HEIGHT: f32 = 600.; - /// `MAX_DIMENSION` (`screenshot_editor.rs:38` over there). const MAX_DIMENSION: u32 = 16_384; @@ -64,11 +63,7 @@ const MAX_DIMENSION: u32 = 16_384; const SAVE_DEBOUNCE: Duration = Duration::from_millis(1000); /// `Header.tsx:112` -- `h-14`. -const HEADER_HEIGHT: f32 = 56.; -/// `AnnotationConfig.tsx:44` -- `h-11`. -const CONFIG_BAR_HEIGHT: f32 = 44.; -/// `LayersPanel.tsx:203` -- `w-56`. -const LAYERS_PANEL_WIDTH: f32 = 224.; +const HEADER_HEIGHT: f32 = 52.; /// `Preview.tsx:52`. const PREVIEW_PADDING: f32 = 20.; /// `clampZoom` (`Preview.tsx:193`). @@ -163,6 +158,355 @@ pub fn load_source(bundle: &Path) -> Result { }) } +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImageAppearance { + version: u8, + background: BackgroundConfiguration, + aspect_ratio: Option, +} + +fn image_appearance_file(project_path: &Path, image_path: &str) -> Option { + let relative = Path::new(image_path); + if relative.parent()? != Path::new("content/images") || relative.extension()?.to_str()? != "png" + { + return None; + } + let stem = relative.file_stem()?.to_str()?; + if stem.len() != 40 + || !stem.starts_with("drawing-") + || !stem[8..] + .chars() + .all(|character| character.is_ascii_hexdigit()) + { + return None; + } + Some( + project_path + .join("content/images") + .join(format!("{stem}.style.json")), + ) +} + +fn portable_project_path(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(root) + .map_err(|_| "Screenshot background is outside the project".to_string())?; + let mut parts = Vec::new(); + for component in relative.components() { + let std::path::Component::Normal(part) = component else { + return Err("Screenshot background path is invalid".to_string()); + }; + parts.push(part.to_string_lossy().into_owned()); + } + Ok(parts.join("/")) +} + +fn load_image_appearance( + project_path: &Path, + image_path: &str, +) -> Result, String> { + let Some(file) = image_appearance_file(project_path, image_path) else { + return Ok(None); + }; + if !file.exists() { + return Ok(None); + } + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let canonical = file.canonicalize().map_err(|error| error.to_string())?; + if !canonical.starts_with(&root) { + return Err("Screenshot appearance escapes the project".to_string()); + } + let mut input = std::fs::File::open(&canonical).map_err(|error| error.to_string())?; + let mut bytes = Vec::new(); + input + .by_ref() + .take(65_537) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 65_536 { + return Err("Screenshot appearance exceeds the size limit".to_string()); + } + let mut appearance: ImageAppearance = + serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if appearance.version != 1 { + return Err("Screenshot appearance version is unsupported".to_string()); + } + match &mut appearance.background.source { + BackgroundSource::Wallpaper { path } | BackgroundSource::Image { path } => { + if let Some(relative) = path { + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err("Screenshot background path is invalid".to_string()); + } + let source = project_path + .join(relative_path) + .canonicalize() + .map_err(|error| error.to_string())?; + if !source.starts_with(&root) || !source.is_file() { + return Err("Screenshot background escapes the project".to_string()); + } + *relative = source.to_string_lossy().into_owned(); + } + } + _ => {} + } + Ok(Some(appearance)) +} + +fn save_image_appearance( + project_path: &Path, + image_path: &str, + mut appearance: ImageAppearance, +) -> Result, String> { + let file = image_appearance_file(project_path, image_path) + .ok_or("Screenshot appearance output path is invalid")?; + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let images_dir = project_path.join("content/images"); + let canonical_images = images_dir + .canonicalize() + .map_err(|error| error.to_string())?; + if !canonical_images.starts_with(&root) { + return Err("Screenshot appearance directory escapes the project".to_string()); + } + let mut created = Vec::new(); + let result = (|| -> Result<(), String> { + match &mut appearance.background.source { + BackgroundSource::Wallpaper { path } | BackgroundSource::Image { path } => { + if let Some(source_path) = path { + let source = Path::new(source_path) + .canonicalize() + .map_err(|error| error.to_string())?; + let extension = source + .extension() + .and_then(|part| part.to_str()) + .ok_or("Screenshot background type is unsupported")? + .to_ascii_lowercase(); + if !matches!( + extension.as_str(), + "png" | "jpg" | "jpeg" | "webp" | "gif" | "bmp" | "tif" | "tiff" + ) { + return Err("Screenshot background type is unsupported".to_string()); + } + let portable = if source.starts_with(&root) { + portable_project_path(&root, &source)? + } else { + let mut input = + std::fs::File::open(&source).map_err(|error| error.to_string())?; + let size = input.metadata().map_err(|error| error.to_string())?.len(); + if size == 0 || size > 64 * 1024 * 1024 { + return Err("Screenshot background exceeds the size limit".to_string()); + } + let name = format!( + "screenshot-background-{}.{}", + uuid::Uuid::new_v4().simple(), + extension + ); + let output = canonical_images.join(&name); + let mut target = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .map_err(|error| error.to_string())?; + created.push(output); + std::io::copy(&mut input, &mut target) + .map_err(|error| error.to_string())?; + target.sync_all().map_err(|error| error.to_string())?; + format!("content/images/{name}") + }; + *source_path = portable; + } + } + _ => {} + } + let bytes = serde_json::to_vec(&appearance).map_err(|error| error.to_string())?; + if bytes.len() > 65_536 { + return Err("Screenshot appearance exceeds the size limit".to_string()); + } + let temp = canonical_images.join(format!( + ".screenshot-style-{}.tmp", + uuid::Uuid::new_v4().simple() + )); + let mut target = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(|error| error.to_string())?; + created.push(temp.clone()); + std::io::Write::write_all(&mut target, &bytes).map_err(|error| error.to_string())?; + target.sync_all().map_err(|error| error.to_string())?; + std::fs::rename(&temp, &file).map_err(|error| error.to_string())?; + let _ = created.pop(); + created.push(file); + Ok(()) + })(); + if let Err(error) = result { + for path in &created { + let _ = std::fs::remove_file(path); + } + return Err(error); + } + Ok(created) +} + +pub fn load_image_drawing_source( + bundle: &Path, + image_index: usize, +) -> Result { + let project = ProjectConfiguration::load(bundle).map_err(|error| error.to_string())?; + let segment = project + .timeline + .as_ref() + .and_then(|timeline| timeline.image_segments.get(image_index)) + .ok_or("Image track item not found")?; + let relative = segment.source_path.as_deref().unwrap_or(&segment.path); + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path.components().any(|component| { + matches!( + component, + std::path::Component::ParentDir | std::path::Component::Prefix(_) + ) + }) + { + return Err("Image source path escapes the project".to_string()); + } + let root = bundle.canonicalize().map_err(|error| error.to_string())?; + let source_path = bundle + .join(relative_path) + .canonicalize() + .map_err(|error| error.to_string())?; + if !source_path.starts_with(&root) + || !crate::import::is_supported_image_import_path(&source_path) + { + return Err("Image source is unavailable inside the project".to_string()); + } + let image = + image::open(&source_path).map_err(|error| format!("Failed to open image: {error}"))?; + let (width, height) = (image.width(), image.height()); + if width == 0 + || height == 0 + || width > MAX_DIMENSION + || height > MAX_DIMENSION + || u64::from(width) * u64::from(height) > 64_000_000 + { + return Err("Image dimensions exceed the editor limit".to_string()); + } + let filename = source_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or("Image source has an invalid name")?; + let video_meta = VideoMeta { + path: RelativePathBuf::from(filename), + fps: 30, + start_time: Some(0.0), + device_id: None, + }; + let studio_meta = StudioRecordingMeta::SingleSegment { + segment: SingleSegment { + display: video_meta, + camera: None, + audio: None, + cursor: None, + }, + }; + let meta = RecordingMeta { + platform: None, + project_path: bundle.to_path_buf(), + pretty_name: segment.name.clone(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(studio_meta.clone())), + upload: None, + }; + let mut config = ProjectConfiguration::default(); + config.background.source = BackgroundSource::Color { + value: [255, 255, 255], + alpha: 0, + }; + config.background.padding = 0.0; + config.background.shadow = 0.0; + if let Some(appearance) = load_image_appearance(bundle, &segment.path)? { + config.background = appearance.background; + config.aspect_ratio = appearance.aspect_ratio; + } else if segment.path == "original.png" { + config.background = project.background.clone(); + config.aspect_ratio = project.aspect_ratio.clone(); + } + config.annotations = segment.annotations.clone(); + Ok(LoadedSource { + rgba: image.to_rgba8().into_raw(), + width, + height, + pretty_name: segment.name.clone(), + config, + meta, + studio_meta, + }) +} + +fn save_image_drawing_asset( + bundle: &Path, + bytes: &[u8], + appearance: ImageAppearance, +) -> Result<(String, Vec), String> { + if bytes.is_empty() || bytes.len() > 64 * 1024 * 1024 { + return Err("Drawing image exceeds the editor size limit".to_string()); + } + let (width, height) = + image::ImageReader::with_format(std::io::Cursor::new(bytes), image::ImageFormat::Png) + .into_dimensions() + .map_err(|error| format!("Drawing PNG is invalid: {error}"))?; + if width == 0 + || height == 0 + || width > MAX_DIMENSION + || height > MAX_DIMENSION + || u64::from(width) * u64::from(height) > 64_000_000 + { + return Err("Drawing image dimensions exceed the editor limit".to_string()); + } + let images_dir = bundle.join("content/images"); + std::fs::create_dir_all(&images_dir).map_err(|error| error.to_string())?; + let root = bundle.canonicalize().map_err(|error| error.to_string())?; + let canonical_images = images_dir + .canonicalize() + .map_err(|error| error.to_string())?; + if !canonical_images.starts_with(root) { + return Err("Drawing asset directory escapes the project".to_string()); + } + let filename = format!("drawing-{}.png", uuid::Uuid::new_v4().simple()); + let relative = format!("content/images/{filename}"); + let output = canonical_images.join(&filename); + let temp = canonical_images.join(format!(".{filename}.tmp")); + let write_result = (|| -> Result<(), std::io::Error> { + let mut file = std::fs::File::create(&temp)?; + std::io::Write::write_all(&mut file, bytes)?; + file.sync_all()?; + std::fs::rename(&temp, &output) + })(); + if let Err(error) = write_result { + let _ = std::fs::remove_file(&temp); + return Err(format!("Cannot save drawing asset: {error}")); + } + let style_files = match save_image_appearance(bundle, &relative, appearance) { + Ok(files) => files, + Err(error) => { + let _ = std::fs::remove_file(&output); + return Err(error); + } + }; + let mut files = vec![output]; + files.extend(style_files); + Ok((relative, files)) +} + // --------------------------------------------------------------------------- // The still renderer // --------------------------------------------------------------------------- @@ -507,6 +851,18 @@ enum Popover { Border, } +#[derive(Clone, Copy)] +pub(crate) enum ImageEditAction { + Tool(Tool), + Aspect, + Crop, + Background, + Padding, + Rounding, + Shadow, + Border, +} + impl Popover { fn anchor(self) -> Anchor { match self { @@ -923,6 +1279,9 @@ pub struct ScreenshotEditorWindow { previous_transform: Option<((f32, f32), annotations::ImageTransform)>, /// `isRenderReady`: the skeleton stands in until the first frame lands. ready: bool, + image_drawing_index: Option, + image_drawing_source: Option, + pending_image_action: Option, pending_save: Rc>, save_task: Option>, @@ -971,14 +1330,7 @@ impl ScreenshotEditorWindow { self.exporting } - pub fn new(bundle: PathBuf, window: &mut Window, cx: &mut Context) -> Self { - let close_bundle = bundle.clone(); - window.on_window_should_close(cx, move |_window, cx| { - let bundle = close_bundle.clone(); - cx.defer(move |cx| crate::app_windows::screenshot_editor_closed(&bundle, cx)); - true - }); - + pub fn new_embedded(bundle: PathBuf, window: &mut Window, cx: &mut Context) -> Self { let pretty_name = bundle .file_stem() .and_then(|stem| stem.to_str()) @@ -1003,6 +1355,9 @@ impl ScreenshotEditorWindow { image_size: None, previous_transform: None, ready: false, + image_drawing_index: None, + image_drawing_source: None, + pending_image_action: None, pending_save: Rc::new(RefCell::new(PendingConfigSave::default())), save_task: None, @@ -1096,7 +1451,9 @@ impl ScreenshotEditorWindow { self.image_size = Some(image_size); self.config_tx = Some(config_tx); self.export_tx = Some(export_tx); - self.pending_save.borrow_mut().path = Some(self.bundle.clone()); + if self.image_drawing_index.is_none() { + self.pending_save.borrow_mut().path = Some(self.bundle.clone()); + } if self.bg_tab == BgTab::Wallpaper { self.ensure_wallpapers(cx); } @@ -1132,6 +1489,67 @@ impl ScreenshotEditorWindow { if let Some(previous) = self.frame.replace(image) { let _ = window.drop_image(previous); } + if let Some(action) = self.pending_image_action.take() { + match action { + ImageEditAction::Tool(tool) => self.set_tool(tool, cx), + ImageEditAction::Crop => self.open_crop_dialog(window, cx), + _ => { + let anchor = match action { + ImageEditAction::Aspect => Anchor::Aspect, + ImageEditAction::Background => Anchor::Background, + ImageEditAction::Padding => Anchor::Padding, + ImageEditAction::Rounding => Anchor::Rounding, + ImageEditAction::Shadow => Anchor::Shadow, + ImageEditAction::Border => Anchor::Border, + ImageEditAction::Tool(_) | ImageEditAction::Crop => unreachable!(), + }; + cx.spawn_in(window, async move |this, cx| { + for _ in 0..15 { + cx.background_executor() + .timer(Duration::from_millis(16)) + .await; + let applied = this.update_in(cx, |this, window, cx| { + if this.image_drawing_index.is_none() + || this.anchor(anchor).get().is_none() + { + return false; + } + match action { + ImageEditAction::Aspect => { + this.toggle_menu(MenuKind::Aspect, anchor, window, cx); + } + ImageEditAction::Background => { + this.active_popover = Some(Popover::Background); + } + ImageEditAction::Padding => { + this.active_popover = Some(Popover::Padding); + } + ImageEditAction::Rounding => { + this.active_popover = Some(Popover::Rounding); + } + ImageEditAction::Shadow => { + this.active_popover = Some(Popover::Shadow); + } + ImageEditAction::Border => { + this.active_popover = Some(Popover::Border); + } + ImageEditAction::Tool(_) | ImageEditAction::Crop => { + unreachable!() + } + } + cx.notify(); + true + }); + match applied { + Ok(true) | Err(_) => return, + Ok(false) => {} + } + } + }) + .detach(); + } + } + } // The content rect moves with the frame, so every mask is re-clipped // into it (`AnnotationLayer.tsx:88-141`) before the overlays resample. if self.clamp_masks() { @@ -1146,6 +1564,21 @@ impl ScreenshotEditorWindow { self.pending_save.clone() } + pub fn new_image_drawing( + bundle: PathBuf, + image_index: usize, + source_relative: String, + action: ImageEditAction, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let mut workspace = Self::new_embedded(bundle, window, cx); + workspace.image_drawing_index = Some(image_index); + workspace.image_drawing_source = Some(source_relative); + workspace.pending_image_action = Some(action); + workspace + } + /// The context's resize effect (`context.tsx:474-547`): a frame that has /// changed size by more than a pixel moves every annotation across from /// the old content rect to the new one. @@ -1212,6 +1645,9 @@ impl ScreenshotEditorWindow { } pub(crate) fn schedule_save(&mut self, window: &mut Window, cx: &mut Context) { + if self.image_drawing_index.is_some() { + return; + } self.pending_save.borrow_mut().config = Some(self.project.clone()); let pending = self.pending_save.clone(); self.save_task = Some(cx.spawn_in(window, async move |_, cx| { @@ -1894,6 +2330,138 @@ impl ScreenshotEditorWindow { /// the encoded bytes to their destination -- `exportImage`'s Copy and Save /// arms (`useScreenshotExport.ts:139-253`). The composite and the encode /// both run on the background executor. + fn cancel_image_drawing(&mut self, window: &mut Window, cx: &mut Context) { + let Some(parent) = window.window_handle().downcast::() else { + return; + }; + cx.defer(move |cx| { + parent + .update(cx, |editor, window, cx| { + editor.close_image_drawing(window, cx) + }) + .ok(); + }); + } + + fn apply_image_drawing(&mut self, window: &mut Window, cx: &mut Context) { + if self.exporting { + return; + } + let (Some(index), Some(source_relative), Some(export_tx), Some(parent)) = ( + self.image_drawing_index, + self.image_drawing_source.clone(), + self.export_tx.clone(), + window.window_handle().downcast::(), + ) else { + return; + }; + let config = self.project.clone(); + let appearance = ImageAppearance { + version: 1, + background: config.background.clone(), + aspect_ratio: config.aspect_ratio.clone(), + }; + if let Err(error) = config.validate() { + self.toast_error(error.to_string(), window, cx); + return; + } + let bundle = self.bundle.clone(); + self.exporting = true; + self.export_status = ExportStatus::Rendering; + cx.notify(); + cx.spawn_in(window, async move |this, cx| { + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + let request = ExportRequest::Render { + config: config.clone(), + reply: reply_tx, + }; + let rendered = if export_tx.send(request).await.is_ok() { + match reply_rx.await { + Ok(result) => result, + Err(_) => Err("The renderer stopped before the drawing finished".into()), + } + } else { + Err("The image renderer is not running".into()) + }; + let encoded = match rendered { + Ok(raw) => { + let composite_config = config.clone(); + cx.background_executor() + .spawn(async move { + let scale_x = f64::from(raw.width) / f64::from(raw.base_width.max(1)); + let scale_y = f64::from(raw.height) / f64::from(raw.base_height.max(1)); + let scaled = crate::screenshot_export::scale_annotations( + &composite_config.annotations, + scale_x, + scale_y, + ); + let bounds = crate::screenshot_export::export_bounds( + &scaled, + (raw.width, raw.height), + ); + if bounds.width > MAX_DIMENSION + || bounds.height > MAX_DIMENSION + || u64::from(bounds.width) * u64::from(bounds.height) > 64_000_000 + { + return Err( + "Drawing image dimensions exceed the editor limit".to_string() + ); + } + let output = + crate::screenshot_export::composite(&raw, &composite_config); + crate::screenshot_export::encode_for_save(&output) + }) + .await + } + Err(error) => Err(error), + }; + let saved = match encoded { + Ok(bytes) => { + cx.background_executor() + .spawn(async move { save_image_drawing_asset(&bundle, &bytes, appearance) }) + .await + } + Err(error) => Err(error), + }; + match saved { + Ok((path, files)) => { + let committed = parent + .update(cx, |editor, window, cx| { + editor.commit_image_drawing( + index, + &source_relative, + path, + config.annotations.clone(), + window, + cx, + ) + }) + .unwrap_or(false); + if !committed { + for file in files { + let _ = std::fs::remove_file(file); + } + this.update_in(cx, |view, window, cx| { + view.toast_error("The image track changed while drawing", window, cx) + }) + .ok(); + } + } + Err(error) => { + this.update_in(cx, |view, window, cx| view.toast_error(error, window, cx)) + .ok(); + } + } + this.update_in(cx, |view, _window, cx| { + view.exporting = false; + view.export_status = ExportStatus::Idle; + cx.notify(); + }) + .ok(); + }) + .detach(); + } + fn export_image( &mut self, destination: ExportDestination, @@ -2347,7 +2915,7 @@ impl ScreenshotEditorWindow { tracing::error!(path = %bundle.display(), "deleting the screenshot failed: {error}"); return; } - cx.update(|_, cx| crate::app_windows::close_screenshot_editor_after_delete(&bundle, cx)) + cx.update(|_, cx| crate::app_windows::close_embedded_screenshot_after_delete(&bundle, cx)) .ok(); }) .detach(); @@ -2613,7 +3181,7 @@ impl ScreenshotEditorWindow { } "c" => { cx.stop_propagation(); - if !self.copy_selected_annotation(cx) { + if !self.copy_selected_annotation(cx) && self.image_drawing_index.is_none() { self.export_image(ExportDestination::Clipboard, window, cx); } } @@ -2623,7 +3191,11 @@ impl ScreenshotEditorWindow { } "s" => { cx.stop_propagation(); - self.export_image(ExportDestination::File, window, cx); + if self.image_drawing_index.is_some() { + self.apply_image_drawing(window, cx); + } else { + self.export_image(ExportDestination::File, window, cx); + } } "-" => { cx.stop_propagation(); @@ -2703,8 +3275,9 @@ impl ScreenshotEditorWindow { div() .flex() .flex_row() + .flex_wrap() .items_center() - .gap(px(4.)) + .gap(px(8.)) .child(tool_button( &theme, "screenshot-layers-toggle", @@ -2717,7 +3290,6 @@ impl ScreenshotEditorWindow { this.toggle_layers_panel(cx); }), )) - .child(divider(&theme, 16.)) .children(Tool::ALL.map(|tool| { tool_button( &theme, @@ -2735,99 +3307,8 @@ impl ScreenshotEditorWindow { } /// `Header.tsx:109-216`. - fn render_header(&self, _window: &Window, cx: &mut Context) -> impl IntoElement { + fn render_header(&self, window: &Window, _cx: &mut Context) -> impl IntoElement { let theme = self.theme; - let crop_enabled = self.image_size.is_some(); - let exporting = self.exporting; - let share_tooltip = match self.export_status { - ExportStatus::Rendering => "Rendering screenshot", - ExportStatus::Encoding => "Preparing upload", - ExportStatus::Uploading => "Uploading screenshot", - ExportStatus::Idle => "Create shareable link", - }; - - let tools = div() - .flex() - .flex_row() - .items_center() - .justify_center() - .gap(px(8.)) - .child( - self.anchored( - Anchor::Aspect, - ui::EditorButton::plain(&theme, "screenshot-aspect") - .width(px(80.)) - .left_icon("icons/layout.svg") - .icon_size(px(16.)) - .label(self.aspect_label()) - .right_icon("icons/chevron-down.svg") - .right_icon_end(true) - .pressed( - self.menu - .as_ref() - .is_some_and(|(kind, _)| *kind == MenuKind::Aspect), - ) - .tooltip(&theme, "Aspect Ratio") - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.toggle_menu(MenuKind::Aspect, Anchor::Aspect, window, cx); - })), - ), - ) - .child( - ui::EditorButton::plain(&theme, "screenshot-crop") - .left_icon("icons/crop.svg") - .icon_size(px(16.)) - .disabled(!crop_enabled) - .tooltip(&theme, "Crop Image") - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.open_crop_dialog(window, cx); - })), - ) - .child(divider(&theme, 24.)) - .child(self.render_annotation_tools(cx)) - .child(divider(&theme, 24.)) - .children( - [ - Popover::Background, - Popover::Padding, - Popover::Rounding, - Popover::Shadow, - Popover::Border, - ] - .map(|popover| { - let button = ui::EditorButton::plain(&theme, popover.id()) - .left_icon(popover.icon()) - .icon_size(px(16.)) - .pressed(self.active_popover == Some(popover)) - .on_click(cx.listener(move |this, _, window, cx| { - cx.stop_propagation(); - this.toggle_popover(popover, window, cx); - })); - // The `kbd` prop rides the tooltip, which - // `ui::EditorButton` does not carry, so the wrapper the - // popover anchors against owns it instead. - self.anchored( - popover.anchor(), - kbd_tooltip(&theme, popover.tooltip(), popover.keys(), button), - ) - .into_any_element() - }), - ); - #[cfg(not(target_os = "windows"))] - let tools = tools.absolute().top_0().left_0().size_full(); - #[cfg(target_os = "windows")] - let tools = div() - .id("screenshot-header-tools") - .flex() - .flex_1() - .min_w_0() - .h(px(32.)) - .overflow_x_scroll() - .occlude() - .child(tools.flex_shrink_0().mx_auto()); - let header = div() .relative() .flex() @@ -2837,12 +3318,6 @@ impl ScreenshotEditorWindow { .w_full() .h(px(HEADER_HEIGHT)) .px(px(16.)) - .when(cfg!(target_os = "windows"), |header| { - header - .pr_0() - .gap(px(8.)) - .window_control_area(gpui::WindowControlArea::Drag) - }) .flex_shrink_0() .border_b_1() .border_color(theme.gray_3) @@ -2851,90 +3326,43 @@ impl ScreenshotEditorWindow { } else { theme.gray_1 }) - // The inset traffic lights' spacer (`Header.tsx:115`). - .when(!cfg!(target_os = "windows"), |header| { - header.child(div().flex().items_center().child(div().w(px(56.)))) + .when(cfg!(target_os = "windows"), |header| { + header.window_control_area(gpui::WindowControlArea::Drag) }) - // `absolute left-1/2 -translate-x-1/2` -- a full-width centred row - // is the same placement without a transform, and it is not - // interactive itself, so the right cluster painted after it still - // takes its own clicks. - .child(tools) .child( div() .flex() .flex_row() .items_center() - .gap(px(8.)) - .h_full() - .pr(px(8.)) - .when(cfg!(target_os = "windows"), |actions| { - actions.h(px(32.)).flex_shrink_0().occlude() + .gap(px(6.)) + .when(!cfg!(target_os = "windows"), |title| { + title.child(div().w(px(76.))) }) - .child(divider(&theme, 24.)) + .text_size(px(14.)) + .font_weight(FontWeight::MEDIUM) + .child(self.pretty_name.clone()) .child( - ui::EditorButton::plain(&theme, "screenshot-copy") - .left_icon("icons/copy.svg") - .icon_size(px(16.)) - .disabled(exporting) - .tooltip(&theme, "Copy to Clipboard") - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.export_image(ExportDestination::Clipboard, window, cx); - })), - ) - .child( - ui::EditorButton::plain(&theme, "screenshot-save") - .left_icon("icons/save.svg") - .icon_size(px(16.)) - .disabled(exporting) - .tooltip(&theme, "Save") - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.export_image(ExportDestination::File, window, cx); - })), - ) - .child( - ui::EditorButton::plain(&theme, "screenshot-share") - .left_icon("icons/link.svg") - .icon_size(px(16.)) - .disabled(exporting) - .tooltip(&theme, share_tooltip) - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.share_screenshot(window, cx); - })), - ) - .child( - self.anchored( - Anchor::More, - ui::EditorButton::plain(&theme, "screenshot-more") - .left_icon("icons/more-horizontal.svg") - .icon_size(px(16.)) - .disabled(exporting) - .pressed( - self.menu - .as_ref() - .is_some_and(|(kind, _)| *kind == MenuKind::More), - ) - .tooltip(&theme, "More Actions") - .on_click(cx.listener(|this, _, window, cx| { - cx.stop_propagation(); - this.toggle_menu(MenuKind::More, Anchor::More, window, cx); - })), - ), + div() + .text_color(Hsla::from(theme.editor.text_3)) + .child(".cap"), ), + ) + .child( + div() + .text_size(px(12.)) + .text_color(Hsla::from(theme.editor.text_3)) + .child("Image editing"), ); - #[cfg(target_os = "windows")] let header = header.child(ui::windows_caption_controls( theme, - _window.is_window_active(), - _window.is_maximized(), + window.is_window_active(), + window.is_maximized(), true, true, )); - + #[cfg(not(target_os = "windows"))] + let _ = window; header } @@ -2946,11 +3374,11 @@ impl ScreenshotEditorWindow { div() .flex() .flex_col() - .h_full() - .w(px(LAYERS_PANEL_WIDTH)) - .flex_shrink_0() - .border_r_1() - .border_color(theme.gray_3) + .h(px(240.)) + .w_full() + .rounded(px(8.)) + .border_1() + .border_color(theme.gray_4) .bg(if theme.is_dark() { theme.gray_2 } else { @@ -3035,34 +3463,21 @@ impl ScreenshotEditorWindow { let theme = self.theme; Some( div() - // The bar floats over the preview, and gpui hitboxes do not - // occlude by default: without this a press on a swatch would - // also land on the annotation underneath it. .occlude() - .absolute() - .top(px(HEADER_HEIGHT)) - .left(px(if self.layers_panel_open { - LAYERS_PANEL_WIDTH - } else { - 0. - })) - .right_0() - .h(px(CONFIG_BAR_HEIGHT)) - .border_b_1() - .border_color(theme.gray_3) + .w_full() + .rounded(px(8.)) + .border_1() + .border_color(theme.gray_4) .bg(if theme.is_dark() { theme.gray_2 } else { theme.gray_1 }) .flex() - .flex_row() - .items_center() - .justify_center() - .gap(px(24.)) - .px(px(16.)) + .flex_col() + .gap(px(12.)) + .p(px(12.)) .children(self.render_annotation_config_controls(cx)) - .child(divider(&theme, 20.)) .child( div() .id("screenshot-annotation-done") @@ -3997,11 +4412,11 @@ impl ScreenshotEditorWindow { .flex_1() .min_w_0() .overflow_hidden() - .bg(if theme.is_dark() { - theme.gray_2 - } else { - theme.gray_1 - }) + .rounded(px(12.)) + .border_1() + .border_color(Hsla::from(theme.editor.line)) + .shadow(theme.editor.card_shadow()) + .bg(Hsla::from(theme.editor.card_2)) .child( div() .id("screenshot-preview-area") @@ -4400,6 +4815,301 @@ impl ScreenshotEditorWindow { } } +impl ScreenshotEditorWindow { + fn render_screenshot_sidebar(&self, cx: &mut Context) -> impl IntoElement { + let theme = self.theme; + let crop_enabled = self.image_size.is_some(); + let appearance = div() + .flex() + .flex_row() + .flex_wrap() + .gap(px(8.)) + .child(self.anchored( + Anchor::Aspect, + appearance_tile( + &theme, + "screenshot-aspect", + "icons/layout.svg", + "Aspect", + Some(self.aspect_label()), + false, + true, + cx.listener(|this, _, window, cx| { + cx.stop_propagation(); + this.toggle_menu(MenuKind::Aspect, Anchor::Aspect, window, cx); + }), + ), + )) + .child(appearance_tile( + &theme, + "screenshot-crop", + "icons/crop.svg", + "Crop", + None, + false, + crop_enabled, + cx.listener(|this, _, window, cx| { + cx.stop_propagation(); + this.open_crop_dialog(window, cx); + }), + )) + .children( + [ + Popover::Background, + Popover::Padding, + Popover::Rounding, + Popover::Shadow, + Popover::Border, + ] + .map(|popover| { + let label = match popover { + Popover::Background => "Background", + Popover::Padding => "Padding", + Popover::Rounding => "Corners", + Popover::Shadow => "Shadow", + Popover::Border => "Border", + }; + let button = appearance_tile( + &theme, + popover.id(), + popover.icon(), + label, + None, + self.active_popover == Some(popover), + true, + cx.listener(move |this, _, window, cx| { + cx.stop_propagation(); + this.toggle_popover(popover, window, cx); + }), + ); + self.anchored( + popover.anchor(), + kbd_tooltip(&theme, popover.tooltip(), popover.keys(), button), + ) + .into_any_element() + }), + ); + let body = div() + .id("screenshot-sidebar-scroll") + .flex() + .flex_col() + .flex_1() + .min_h_0() + .overflow_y_scroll() + .gap(px(16.)) + .p(px(16.)) + .child( + div() + .flex() + .flex_col() + .gap(px(8.)) + .child( + div() + .text_size(px(11.)) + .text_color(theme.gray_11) + .child("Annotate"), + ) + .child(self.render_annotation_tools(cx)), + ) + .children(self.render_annotation_config_bar(cx)) + .when(self.layers_panel_open, |body| { + body.child(self.render_layers_panel(cx)) + }) + .child( + div() + .flex() + .flex_col() + .gap(px(8.)) + .child( + div() + .text_size(px(11.)) + .text_color(theme.gray_11) + .child("Appearance"), + ) + .child(appearance), + ); + let footer = div() + .flex() + .flex_row() + .items_center() + .gap(px(8.)) + .p(px(16.)) + .border_t_1() + .border_color(Hsla::from(theme.editor.line)) + .children(self.render_image_drawing_actions(cx)) + .when(self.image_drawing_index.is_none(), |footer| { + footer + .child( + ui::EditorButton::plain(&theme, "screenshot-copy") + .width(px(64.)) + .left_icon("icons/copy.svg") + .icon_size(px(16.)) + .label("Copy") + .disabled(self.exporting) + .on_click(cx.listener(|this, _, window, cx| { + this.export_image(ExportDestination::Clipboard, window, cx); + })), + ) + .child( + ui::EditorButton::plain(&theme, "screenshot-save") + .width(px(64.)) + .left_icon("icons/save.svg") + .icon_size(px(16.)) + .label("Save") + .disabled(self.exporting) + .on_click(cx.listener(|this, _, window, cx| { + this.export_image(ExportDestination::File, window, cx); + })), + ) + .child( + ui::EditorButton::plain(&theme, "screenshot-share") + .width(px(64.)) + .left_icon("icons/link.svg") + .icon_size(px(16.)) + .label("Share") + .disabled(self.exporting) + .on_click(cx.listener(|this, _, window, cx| { + this.share_screenshot(window, cx); + })), + ) + .child( + self.anchored( + Anchor::More, + ui::EditorButton::plain(&theme, "screenshot-more") + .width(px(64.)) + .left_icon("icons/more-horizontal.svg") + .icon_size(px(16.)) + .label("More") + .disabled(self.exporting) + .on_click(cx.listener(|this, _, window, cx| { + this.toggle_menu(MenuKind::More, Anchor::More, window, cx); + })), + ), + ) + }); + let back = self.image_drawing_index.is_some(); + div() + .flex() + .flex_col() + .h_full() + .w(px(416.)) + .flex_shrink_0() + .overflow_hidden() + .rounded(px(12.)) + .border_1() + .border_color(Hsla::from(theme.editor.line)) + .shadow(theme.editor.card_shadow()) + .bg(Hsla::from(theme.editor.card)) + .child( + div() + .flex() + .flex_row() + .items_center() + .gap(px(8.)) + .h(px(46.)) + .px(px(12.)) + .flex_none() + .border_b_1() + .border_color(Hsla::from(theme.editor.line)) + .children(back.then(|| { + div() + .id("screenshot-sidebar-back") + .flex() + .flex_row() + .items_center() + .justify_center() + .gap(px(4.)) + .h(px(32.)) + .px(px(8.)) + .rounded(px(8.)) + .cursor_pointer() + .hover(|style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .child( + svg() + .path("icons/arrow-left.svg") + .size(px(16.)) + .text_color(Hsla::from(theme.editor.text_2)), + ) + .child( + div() + .text_size(px(11.)) + .text_color(Hsla::from(theme.editor.text_2)) + .child("Back"), + ) + .on_click(cx.listener(|this, _, window, cx| { + if !this.exporting { + this.cancel_image_drawing(window, cx); + } + })) + })) + .child( + div() + .flex() + .flex_col() + .min_w_0() + .child( + div() + .text_size(px(13.)) + .font_weight(FontWeight::SEMIBOLD) + .text_color(Hsla::from(theme.editor.text_1)) + .child("Edit image"), + ) + .child( + div() + .text_size(px(11.)) + .text_color(Hsla::from(theme.editor.text_3)) + .truncate() + .child(self.pretty_name.clone()), + ), + ), + ) + .child(body) + .child(footer) + } + + fn render_image_drawing_actions(&self, cx: &mut Context) -> Option { + self.image_drawing_index?; + let theme = self.theme; + Some( + div() + .flex() + .flex_row() + .gap(px(8.)) + .child( + ui::Button::plain( + &theme, + "cancel-image-drawing", + ui::ButtonVariant::Gray, + ui::ButtonSize::Md, + ) + .label("Cancel") + .disabled(self.exporting) + .on_click( + cx.listener(|this, _, window, cx| this.cancel_image_drawing(window, cx)), + ), + ) + .child( + ui::Button::plain( + &theme, + "apply-image-drawing", + ui::ButtonVariant::Blue, + ui::ButtonSize::Md, + ) + .label(if self.exporting { + "Applying…" + } else { + "Apply changes" + }) + .disabled(self.exporting || !self.ready) + .on_click( + cx.listener(|this, _, window, cx| this.apply_image_drawing(window, cx)), + ), + ) + .into_any_element(), + ) + } +} + impl Render for ScreenshotEditorWindow { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { self.sync_appearance(window, cx); @@ -4426,11 +5136,7 @@ impl Render for ScreenshotEditorWindow { .flex_col() .font_family("Geist") .font_weight(FontWeight::MEDIUM) - .bg(if theme.is_dark() { - theme.gray_1 - } else { - theme.gray_2 - }) + .bg(Hsla::from(theme.editor.window)) .text_color(theme.gray_12); if !self.ready { @@ -4446,12 +5152,12 @@ impl Render for ScreenshotEditorWindow { .min_h_0() .w_full() .overflow_hidden() - .when(self.layers_panel_open, |this| { - this.child(self.render_layers_panel(cx)) - }) - .child(self.render_preview(cx)), + .gap(px(8.)) + .px(px(2.)) + .pb(px(2.)) + .child(self.render_preview(cx)) + .child(self.render_screenshot_sidebar(cx)), ) - .children(self.render_annotation_config_bar(cx)) .children(self.render_annotation_overlays(window, cx)) .children(self.render_popover(window, cx)) .children(self.render_menu(cx)) @@ -4536,19 +5242,27 @@ fn tool_button( div() .id(id.into()) .flex() + .flex_col() .items_center() .justify_center() - .size(px(32.)) + .gap(px(4.)) + .w(px(84.)) + .h(px(68.)) .flex_shrink_0() - .rounded(px(8.)) + .rounded(px(12.)) .cursor_pointer() - .when(active, |this| this.bg(theme.blue_3)) - .when(!active, |this| this.hover(|style| style.bg(theme.gray_3))) - .child(svg().path(icon).size(px(16.)).text_color(if active { - theme.blue_11 + .bg(if active { + Hsla::from(theme.editor.accent_2) + } else { + Hsla::from(theme.editor.ctl) + }) + .hover(move |style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .child(svg().path(icon).size(px(20.)).text_color(if active { + Hsla::from(theme.editor.accent) } else { - theme.gray_11 + Hsla::from(theme.editor.text_2) })) + .child(div().text_size(px(11.)).truncate().child(label.clone())) .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) .tooltip(move |_window, cx| { ui::Tooltip::new(&theme, label.clone()) @@ -4558,6 +5272,56 @@ fn tool_button( .on_click(on_click) } +fn appearance_tile( + theme: &Theme, + id: &'static str, + icon: &'static str, + label: &'static str, + subtitle: Option<&'static str>, + active: bool, + enabled: bool, + on_click: impl Fn(&gpui::ClickEvent, &mut Window, &mut App) + 'static, +) -> impl IntoElement { + let theme = *theme; + let foreground = if active { + Hsla::from(theme.editor.accent) + } else { + Hsla::from(theme.editor.text_2) + }; + div() + .id(id) + .tab_index(0) + .flex() + .flex_col() + .items_center() + .justify_center() + .gap(px(4.)) + .w(px(84.)) + .h(px(68.)) + .rounded(px(12.)) + .bg(if active { + Hsla::from(theme.editor.accent_2) + } else { + Hsla::from(theme.editor.ctl) + }) + .text_color(foreground) + .when(enabled, |tile| { + tile.cursor_pointer() + .hover(move |style| style.bg(Hsla::from(theme.editor.ctl_hover))) + .on_click(on_click) + }) + .when(!enabled, |tile| tile.opacity(0.5)) + .child(svg().path(icon).size(px(20.)).text_color(foreground)) + .child(div().text_size(px(11.)).truncate().child(label)) + .children(subtitle.map(|subtitle| { + div() + .text_size(px(9.)) + .text_color(Hsla::from(theme.editor.text_3)) + .truncate() + .child(subtitle) + })) +} + /// The transparent swatch's mini checkerboard -- four 16px quarters, which is /// what the CSS pattern reduces to at `size-8`. fn mini_checker() -> Vec { @@ -4713,24 +5477,61 @@ fn frame_buffers(frame: &RenderedFrame) -> Option<(Arc>, Arc, + handle: WindowHandle, + cx: &mut App, +) { + load_embedded_project(bundle, handle, None, cx); +} + +pub fn load_image_drawing_project_embedded( + bundle: PathBuf, + image_index: usize, + handle: WindowHandle, + cx: &mut App, +) { + load_embedded_project(bundle, handle, Some(image_index), cx); +} + +fn embedded_workspace( + editor: &EditorWindow, + image_index: Option, +) -> Option<&Entity> { + if image_index.is_some() { + editor.image_drawing_workspace.as_ref() + } else { + editor.screenshot_workspace.as_ref() + } +} + +fn load_embedded_project( + bundle: PathBuf, + handle: WindowHandle, + image_index: Option, cx: &mut App, ) { cx.spawn(async move |cx| { let load_bundle = bundle.clone(); let source = cx .background_executor() - .spawn(async move { load_source(&load_bundle) }) + .spawn(async move { + if let Some(index) = image_index { + load_image_drawing_source(&load_bundle, index) + } else { + load_source(&load_bundle) + } + }) .await; let source = match source { Ok(source) => source, Err(message) => { handle - .update(cx, |view, _window, cx| view.set_error(message, cx)) + .update(cx, |editor, _window, cx| { + if let Some(workspace) = embedded_workspace(editor, image_index) { + workspace.update(cx, |view, cx| view.set_error(message, cx)); + } + }) .ok(); return; } @@ -4741,14 +5542,14 @@ pub fn load_screenshot_project( config: source.config.clone(), }); let (export_tx, export_rx) = tokio::sync::mpsc::channel(1); - // Bounded and latest-wins like the video pump; stills only re-render - // on edits, so it never actually fills. let (frame_tx, frame_rx) = flume::bounded(2); let (setup_tx, setup_rx) = flume::bounded(1); - let image_size = (source.width, source.height); - if handle - .update(cx, |view, window, cx| { + let loaded = handle.update(cx, |editor, window, cx| { + let Some(workspace) = embedded_workspace(editor, image_index) else { + return false; + }; + workspace.update(cx, |view, cx| { view.set_loaded( source.pretty_name.clone(), LoadedScreenshot { @@ -4760,9 +5561,10 @@ pub fn load_screenshot_project( window, cx, ) - }) - .is_err() - { + }); + true + }); + if !loaded.unwrap_or(false) { return; } @@ -4776,7 +5578,11 @@ pub fn load_screenshot_project( if let Ok(Err(message)) = setup_rx.recv_async().await { handle - .update(cx, |view, _window, cx| view.set_error(message, cx)) + .update(cx, |editor, _window, cx| { + if let Some(workspace) = embedded_workspace(editor, image_index) { + workspace.update(cx, |view, cx| view.set_error(message, cx)); + } + }) .ok(); return; } @@ -4797,12 +5603,16 @@ pub fn load_screenshot_project( height = size.1, "screenshot frame" ); - if handle - .update(cx, |view, window, cx| { + let alive = handle.update(cx, |editor, window, cx| { + let Some(workspace) = embedded_workspace(editor, image_index) else { + return false; + }; + workspace.update(cx, |view, cx| { view.frame_arrived(image, rgba, size, window, cx) - }) - .is_err() - { + }); + true + }); + if !alive.unwrap_or(false) { return; } } @@ -4823,6 +5633,8 @@ fn aspect_eq(a: &Option, b: &Option, + wake: tokio::sync::Notify, +} + +impl WaveformCancellation { + fn cancel(&self) { + self.flag.store(true, Ordering::Release); + self.wake.notify_one(); + } +} + +type WaveformRequests = HashMap>; + +static ACTIVE_WAVEFORMS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +static NEXT_WAVEFORM_REQUEST: AtomicU64 = AtomicU64::new(0); + +struct WaveformRequest { + id: u64, + window_label: String, + cancellation: Arc, +} + +impl WaveformRequest { + fn new(window_label: &str) -> Result { + let id = NEXT_WAVEFORM_REQUEST.fetch_add(1, Ordering::Relaxed); + let cancellation = Arc::new(WaveformCancellation { + flag: Arc::new(AtomicBool::new(false)), + wake: tokio::sync::Notify::new(), + }); + ACTIVE_WAVEFORMS + .lock() + .map_err(|error| format!("Waveform request registry unavailable: {error}"))? + .entry(window_label.to_string()) + .or_default() + .insert(id, cancellation.clone()); + Ok(Self { + id, + window_label: window_label.to_string(), + cancellation, + }) + } +} + +impl Drop for WaveformRequest { + fn drop(&mut self) { + self.cancellation.cancel(); + if let Ok(mut active) = ACTIVE_WAVEFORMS.lock() + && let Some(requests) = active.get_mut(&self.window_label) + { + requests.remove(&self.id); + if requests.is_empty() { + active.remove(&self.window_label); + } + } + } +} + +pub fn cancel_imported_waveforms_for_window(window_label: &str) { + if let Ok(active) = ACTIVE_WAVEFORMS.lock() + && let Some(requests) = active.get(window_label) + { + for cancellation in requests.values() { + cancellation.cancel(); + } + } +} + +#[tauri::command] +#[specta::specta] +pub fn cancel_imported_waveforms(window: Window) { + cancel_imported_waveforms_for_window(window.label()); +} + #[derive(Serialize, Deserialize, Type, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct AudioLibraryTrack { @@ -163,3 +244,57 @@ pub async fn import_audio_track_file( .await .map_err(|e| format!("Audio import task failed: {e}"))? } + +#[tauri::command] +#[specta::specta] +#[tracing::instrument(skip(editor_instance, window))] +pub async fn get_imported_waveform( + editor_instance: WindowEditorInstance, + window: Window, + path: String, +) -> Result { + let request = WaveformRequest::new(window.label())?; + let _slot = tokio::select! { + permit = cap_audio::imported_waveform_slots().acquire() => { + permit.map_err(|error| format!("Waveform worker unavailable: {error}"))? + } + _ = request.cancellation.wake.notified() => { + return Err("Waveform request cancelled".into()); + } + }; + if request.cancellation.flag.load(Ordering::Acquire) { + return Err("Waveform request cancelled".into()); + } + let project_path = editor_instance.project_path.clone(); + let cancellation = request.cancellation.flag.clone(); + let peaks = tokio::task::spawn_blocking(move || { + cap_audio::imported_waveform(&project_path, &path, cancellation) + }) + .await + .map_err(|error| format!("Waveform task failed: {error}"))??; + if request.cancellation.flag.load(Ordering::Acquire) { + return Err("Waveform request cancelled".into()); + } + Ok(base64::engine::general_purpose::STANDARD.encode(peaks.as_ref())) +} + +#[cfg(test)] +mod waveform_tests { + use super::*; + + #[tokio::test] + async fn closing_editor_cancels_queued_waveforms_and_clears_registry() { + let request = WaveformRequest::new("waveform-test-editor").unwrap(); + cancel_imported_waveforms_for_window("waveform-test-editor"); + request.cancellation.wake.notified().await; + assert!(request.cancellation.flag.load(Ordering::Acquire)); + drop(request); + assert!( + ACTIVE_WAVEFORMS + .lock() + .unwrap() + .get("waveform-test-editor") + .is_none() + ); + } +} diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index f5b87f89cb5..7dcc3a83fd0 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -1876,6 +1876,28 @@ pub struct ExportPreviewResult { pub total_frames: u32, } +fn shared_preview_settings( + settings: ExportPreviewSettings, +) -> cap_export::preview::ExportPreviewSettings { + cap_export::preview::ExportPreviewSettings { + fps: settings.fps, + resolution_base: settings.resolution_base, + compression_bpp: settings.compression_bpp, + cursor_only: settings.cursor_only, + } +} + +fn from_shared_preview(result: cap_export::preview::ExportPreviewResult) -> ExportPreviewResult { + ExportPreviewResult { + jpeg_base64: result.jpeg_base64, + estimated_size_mb: result.estimated_size_mb, + actual_width: result.actual_width, + actual_height: result.actual_height, + frame_render_time_ms: result.frame_render_time_ms, + total_frames: result.total_frames, + } +} + fn estimate_cursor_only_size_mb(total_pixels: f64, total_frames: f64) -> f64 { let bytes_per_frame = total_pixels * 0.4; (bytes_per_frame * total_frames) / (1024.0 * 1024.0) @@ -1964,6 +1986,19 @@ async fn generate_export_preview_inner( .collect::>(), ); + if recordings.segments.is_empty() || project_config.get_segment_time(frame_time).is_none() { + return cap_export::preview::render_preview_with_config( + project_path, + project_config, + frame_time, + shared_preview_settings(settings), + should_force_ffmpeg_preview(), + ) + .await + .map(from_shared_preview) + .map_err(|error| error.to_string()); + } + let render_constants = Arc::new( RenderVideoConstants::new( &recordings.segments, @@ -2195,6 +2230,7 @@ async fn generate_export_preview_inner( #[cfg(test)] mod tests { use super::*; + use base64::{Engine, engine::general_purpose::STANDARD}; use tempfile::tempdir; #[test] @@ -2279,6 +2315,48 @@ mod tests { assert!(malformed.contains("Failed to read saved project config")); } + #[tokio::test] + async fn image_only_project_has_tauri_export_preview() { + let directory = tempdir().unwrap(); + let bundle = + cap_project::create_media_project(directory.path(), "Tauri image preview").unwrap(); + let images = bundle.join("content/images"); + std::fs::create_dir_all(&images).unwrap(); + image::RgbaImage::from_pixel(640, 360, image::Rgba([40, 210, 50, 255])) + .save(images.join("source.png")) + .unwrap(); + let mut config = cap_project::ProjectConfiguration::load(&bundle).unwrap(); + config + .timeline + .as_mut() + .unwrap() + .image_segments + .push(cap_project::ImageSegment { + end: 3.0, + path: "content/images/source.png".to_string(), + size: XY::new(1.0, 1.0), + ..Default::default() + }); + config.write(&bundle).unwrap(); + let preview = generate_export_preview_inner( + bundle, + 0.5, + ExportPreviewSettings { + fps: 30, + resolution_base: XY::new(640, 360), + compression_bpp: 0.15, + cursor_only: false, + }, + ) + .await + .unwrap(); + assert_eq!(preview.total_frames, 90); + let jpeg = STANDARD.decode(preview.jpeg_base64).unwrap(); + let frame = image::load_from_memory(&jpeg).unwrap().to_rgb8(); + let pixel = frame.get_pixel(frame.width() / 2, frame.height() / 2); + assert!(pixel[1] > pixel[0] + 80); + } + #[tokio::test] async fn saved_export_preview_projects_captions_after_a_cut() { let directory = tempdir().unwrap(); @@ -2560,6 +2638,19 @@ async fn generate_export_preview_fast_inner( }) .await .map_err(|error| format!("Failed to synchronize export preview timing: {error}"))?; + if editor.recordings.segments.is_empty() + || project_config.get_segment_time(frame_time).is_none() + { + return cap_export::preview::render_preview_with_editor( + &editor, + project_config, + frame_time, + shared_preview_settings(settings), + ) + .await + .map(from_shared_preview) + .map_err(|error| error.to_string()); + } let transition_mapping = project_config.timeline.as_ref().and_then(|timeline| { if timeline.transitions.is_empty() { return None; diff --git a/apps/desktop/src-tauri/src/gpui_app.rs b/apps/desktop/src-tauri/src/gpui_app.rs index 3a82e010c52..d9d4daa7d85 100644 --- a/apps/desktop/src-tauri/src/gpui_app.rs +++ b/apps/desktop/src-tauri/src/gpui_app.rs @@ -133,10 +133,15 @@ fn binary_path(app: &AppHandle) -> Option { None } -/// Mirror of `store::app_data_dir` in `apps/desktop-gpui`: the pidfile and the -/// handoff marker live under the shared production identifier -/// (`so.cap.desktop`), not this app's possibly-`.dev` one. +/// Mirror of `store::app_data_dir` in `apps/desktop-gpui`: both handoff sides +/// must use the same explicit sandbox directory when one is supplied. fn shared_data_dir() -> PathBuf { + if let Ok(dir) = std::env::var("CAP_GPUI_APP_DATA_DIR") + && !dir.trim().is_empty() + { + return PathBuf::from(dir); + } + #[cfg(target_os = "macos")] let base = PathBuf::from(std::env::var("HOME").unwrap_or_else(|_| ".".into())) .join("Library/Application Support/so.cap.desktop"); diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index cebe7d9c065..b8273ed3b14 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -6,9 +6,9 @@ use cap_enc_ffmpeg::{ }; use cap_media_info::{AudioInfo, FFRational, Pixel, VideoInfo, ensure_even}; use cap_project::{ - AudioMeta, ClipConfiguration, CursorEvents, CursorMeta, Cursors, InstantRecordingMeta, - MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, RecordingMeta, - RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, + AudioMeta, ClipConfiguration, CursorEvents, CursorMeta, Cursors, ImageSegment, + InstantRecordingMeta, MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, + RecordingMeta, RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, TimelineConfiguration, TimelineSegment, VideoMeta, XY, }; use ffmpeg::{ @@ -16,12 +16,13 @@ use ffmpeg::{ codec::{self as avcodec}, format::{self as avformat}, }; -use image::ImageEncoder; +use image::{ImageDecoder, ImageEncoder}; use relative_path::{Component as RelativeComponent, RelativePathBuf}; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ collections::HashMap, + io::{BufReader, BufWriter, Read, Write}, path::{Path, PathBuf}, str::FromStr, }; @@ -42,6 +43,30 @@ const AUDIO_IMPORT_EXTENSIONS: &[&str] = &["ogg", "m4a", "mp3", "wav", "aac", "f const KEYBOARD_IMPORT_EXTENSIONS: &[&str] = &["bin", "json"]; const CURSOR_EVENTS_IMPORT_EXTENSIONS: &[&str] = &["json"]; const MAX_IMAGE_DIMENSION: u32 = 16_384; +const MAX_EDITOR_IMAGE_BYTES: u64 = 64 * 1024 * 1024; +const MAX_EDITOR_IMAGE_PIXELS: u64 = 16_777_216; +const MAX_EDITOR_IMAGE_DECODED_BYTES: u64 = 128 * 1024 * 1024; + +#[derive(Serialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct ImportedEditorImage { + pub path: String, + pub name: String, + pub width: u32, + pub height: u32, +} + +#[derive(Serialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct ImportedEditorVideo { + pub path: String, + pub name: String, + pub duration: f64, + pub fps: u32, + pub width: u32, + pub height: u32, + pub has_audio: bool, +} #[derive(Serialize, Deserialize, Type, Clone, Debug)] pub enum ImportStage { @@ -380,6 +405,7 @@ fn ensure_project_timeline<'a>( scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -1917,6 +1943,289 @@ pub async fn add_existing_recording_to_editor( Ok(imported_count) } +fn import_editor_image_file( + project_path: &Path, + source_path: &Path, +) -> Result { + if !is_supported_image_import_path(source_path) { + return Err("Choose a PNG, JPEG, WebP, GIF, BMP or TIFF image".to_string()); + } + let source = std::fs::File::open(source_path) + .map_err(|error| format!("Failed to open image: {error}"))?; + let size = source + .metadata() + .map_err(|error| format!("Failed to inspect image: {error}"))? + .len(); + if size == 0 || size > MAX_EDITOR_IMAGE_BYTES { + return Err("Choose a non-empty image no larger than 64 MiB".to_string()); + } + + let image_dir = project_path.join("content/images"); + std::fs::create_dir_all(&image_dir) + .map_err(|error| format!("Failed to create image directory: {error}"))?; + let project_root = project_path + .canonicalize() + .map_err(|error| format!("Failed to resolve project path: {error}"))?; + if !image_dir + .canonicalize() + .map_err(|error| format!("Failed to resolve image directory: {error}"))? + .starts_with(project_root) + { + return Err("Image directory is outside the project".to_string()); + } + + let id = uuid::Uuid::new_v4(); + let temporary = image_dir.join(format!(".{id}.import")); + let result = (|| -> Result { + let target = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| format!("Failed to prepare image import: {error}"))?; + let mut source = + BufReader::with_capacity(1024 * 1024, source).take(MAX_EDITOR_IMAGE_BYTES + 1); + let mut target = BufWriter::with_capacity(1024 * 1024, target); + let copied = std::io::copy(&mut source, &mut target) + .map_err(|error| format!("Failed to copy image: {error}"))?; + if copied != size { + return Err("The image changed while importing".to_string()); + } + target + .flush() + .map_err(|error| format!("Failed to finish image copy: {error}"))?; + target + .get_ref() + .sync_all() + .map_err(|error| format!("Failed to save image: {error}"))?; + drop(target); + + let mut reader = image::ImageReader::open(&temporary) + .map_err(|error| format!("Failed to inspect copied image: {error}"))? + .with_guessed_format() + .map_err(|error| format!("Failed to identify image format: {error}"))?; + let extension = match reader.format() { + Some(image::ImageFormat::Png) => "png", + Some(image::ImageFormat::Jpeg) => "jpg", + Some(image::ImageFormat::WebP) => "webp", + Some(image::ImageFormat::Gif) => "gif", + Some(image::ImageFormat::Bmp) => "bmp", + Some(image::ImageFormat::Tiff) => "tiff", + _ => return Err("Unsupported or damaged image format".to_string()), + }; + let mut limits = image::Limits::default(); + limits.max_alloc = Some(MAX_EDITOR_IMAGE_DECODED_BYTES); + limits.max_image_width = Some(32_768); + limits.max_image_height = Some(32_768); + reader.limits(limits); + let mut decoder = reader + .into_decoder() + .map_err(|error| format!("Failed to decode image: {error}"))?; + let (source_width, source_height) = decoder.dimensions(); + if source_width == 0 + || source_height == 0 + || u64::from(source_width) * u64::from(source_height) > MAX_EDITOR_IMAGE_PIXELS + || decoder.total_bytes() > MAX_EDITOR_IMAGE_DECODED_BYTES + { + return Err("Image dimensions are too large".to_string()); + } + let orientation = decoder + .orientation() + .map_err(|error| format!("Failed to read image orientation: {error}"))?; + let mut decoded = image::DynamicImage::from_decoder(decoder) + .map_err(|error| format!("Failed to decode image: {error}"))?; + decoded.apply_orientation(orientation); + let (width, height) = (decoded.width(), decoded.height()); + drop(decoded); + + let relative = format!("content/images/{id}.{extension}"); + std::fs::rename(&temporary, project_path.join(&relative)) + .map_err(|error| format!("Failed to finish image import: {error}"))?; + Ok(ImportedEditorImage { + path: relative, + name: source_path + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("Image") + .to_string(), + width, + height, + }) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result +} + +#[tauri::command] +#[specta::specta] +pub async fn import_editor_image( + window: Window, + source_path: PathBuf, +) -> Result { + let project_path = editor_project_path_from_window(&window)?; + tokio::task::spawn_blocking(move || import_editor_image_file(&project_path, &source_path)) + .await + .map_err(|error| format!("Image import worker failed: {error}"))? +} + +#[tauri::command] +#[specta::specta] +pub async fn import_editor_video( + window: Window, + source_path: PathBuf, +) -> Result { + let project_path = editor_project_path_from_window(&window)?; + let imported = tokio::task::spawn_blocking(move || { + cap_media_info::video_import::import_video(&project_path, &source_path) + }) + .await + .map_err(|error| format!("Video import worker failed: {error}"))??; + Ok(ImportedEditorVideo { + path: imported.path, + name: imported.name, + duration: imported.duration, + fps: imported.fps, + width: imported.width, + height: imported.height, + has_audio: imported.has_audio, + }) +} + +#[tauri::command] +#[specta::specta] +pub async fn create_media_project_from_video( + app: AppHandle, + source_path: PathBuf, +) -> Result { + let base = crate::general_settings::GeneralSettingsStore::recordings_dir(&app); + let (project_path, asset_path) = tokio::task::spawn_blocking(move || { + let name = source_path + .file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| !stem.is_empty()) + .unwrap_or("Imported video"); + let project_path = cap_project::create_media_project(&base, name)?; + let result: Result<(PathBuf, PathBuf), String> = (|| { + let imported = cap_media_info::video_import::import_video(&project_path, &source_path)?; + let asset_path = project_path.join(&imported.path); + let mut config = ProjectConfiguration::load(&project_path) + .map_err(|error| format!("Cannot load media timeline: {error}"))?; + config + .timeline + .get_or_insert_with(TimelineConfiguration::default) + .video_segments + .push(cap_project::VideoSegment { + end: imported.duration, + path: imported.path, + name: imported.name, + source_duration: imported.duration, + muted: !imported.has_audio, + ..Default::default() + }); + config + .write(&project_path) + .map_err(|error| format!("Cannot save imported video timeline: {error}"))?; + Ok((project_path.clone(), asset_path)) + })(); + if result.is_err() { + let _ = std::fs::remove_dir_all(&project_path); + } + result + }) + .await + .map_err(|error| format!("Video import worker failed: {error}"))??; + let thumbnail = project_path.join("screenshots/display.jpg"); + if let Err(error) = std::fs::create_dir_all(thumbnail.parent().unwrap()) { + tracing::warn!(%error, "could not create imported video thumbnail directory"); + } else if let Err(error) = create_screenshot(asset_path, thumbnail, None).await { + tracing::warn!(%error, "could not create imported video thumbnail"); + } + Ok(project_path) +} + +fn create_image_thumbnail(path: &Path, thumbnail: &Path) -> Result<(), String> { + let mut reader = image::ImageReader::open(path) + .map_err(|error| format!("Cannot open imported image thumbnail: {error}"))? + .with_guessed_format() + .map_err(|error| format!("Cannot identify imported image thumbnail: {error}"))?; + let mut limits = image::Limits::default(); + limits.max_alloc = Some(MAX_EDITOR_IMAGE_DECODED_BYTES); + limits.max_image_width = Some(32_768); + limits.max_image_height = Some(32_768); + reader.limits(limits); + let mut decoder = reader + .into_decoder() + .map_err(|error| format!("Cannot decode imported image thumbnail: {error}"))?; + let orientation = decoder + .orientation() + .map_err(|error| format!("Cannot orient imported image thumbnail: {error}"))?; + let mut image = image::DynamicImage::from_decoder(decoder) + .map_err(|error| format!("Cannot read imported image thumbnail: {error}"))?; + image.apply_orientation(orientation); + std::fs::create_dir_all( + thumbnail + .parent() + .ok_or("Imported image thumbnail has no directory")?, + ) + .map_err(|error| format!("Cannot create imported image thumbnail directory: {error}"))?; + image + .thumbnail(400, 225) + .save_with_format(thumbnail, image::ImageFormat::Jpeg) + .map_err(|error| format!("Cannot save imported image thumbnail: {error}")) +} + +fn create_media_project_from_image_file( + base: &Path, + source_path: &Path, +) -> Result { + let name = source_path + .file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| !stem.is_empty()) + .unwrap_or("Imported image"); + let project_path = cap_project::create_media_project(base, name)?; + let result: Result = (|| { + let imported = import_editor_image_file(&project_path, source_path)?; + let mut config = ProjectConfiguration::load(&project_path) + .map_err(|error| format!("Cannot load media timeline: {error}"))?; + config + .timeline + .get_or_insert_with(TimelineConfiguration::default) + .image_segments + .push(ImageSegment { + end: 5.0, + path: imported.path.clone(), + size: XY::new(1.0, 1.0), + ..Default::default() + }); + config + .write(&project_path) + .map_err(|error| format!("Cannot save imported image timeline: {error}"))?; + create_image_thumbnail( + &project_path.join(imported.path), + &project_path.join("screenshots/display.jpg"), + )?; + Ok(project_path.clone()) + })(); + if result.is_err() { + let _ = std::fs::remove_dir_all(&project_path); + } + result +} + +#[tauri::command] +#[specta::specta] +pub async fn create_media_project_from_image( + app: AppHandle, + source_path: PathBuf, +) -> Result { + let base = crate::general_settings::GeneralSettingsStore::recordings_dir(&app); + tokio::task::spawn_blocking(move || create_media_project_from_image_file(&base, &source_path)) + .await + .map_err(|error| format!("Image import worker failed: {error}"))? +} + #[tauri::command] #[specta::specta] pub async fn start_image_import(app: AppHandle, source_path: PathBuf) -> Result { @@ -2106,6 +2415,74 @@ pub async fn check_import_ready(project_path: PathBuf) -> Result { mod tests { use super::*; + #[test] + fn editor_image_import_copies_external_media_into_unique_project_assets() { + let project = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + let source = external.path().join("photo.png"); + image::RgbaImage::new(8, 6).save(&source).unwrap(); + + let first = import_editor_image_file(project.path(), &source).unwrap(); + let second = import_editor_image_file(project.path(), &source).unwrap(); + + assert_eq!((first.width, first.height), (8, 6)); + assert_eq!(first.name, "photo"); + assert!(first.path.starts_with("content/images/")); + assert_ne!(first.path, second.path); + assert_eq!( + std::fs::read(project.path().join(&first.path)).unwrap(), + std::fs::read(&source).unwrap() + ); + assert!(project.path().join(&second.path).is_file()); + } + + #[test] + fn editor_image_import_removes_damaged_partial_assets() { + let project = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + let source = external.path().join("broken.png"); + std::fs::write(&source, b"broken image").unwrap(); + + assert!(import_editor_image_file(project.path(), &source).is_err()); + assert_eq!( + std::fs::read_dir(project.path().join("content/images")) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn direct_image_import_creates_media_timeline_and_thumbnail() { + let library = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + let source = external.path().join("photo.png"); + image::RgbaImage::from_pixel(24, 16, image::Rgba([25, 80, 210, 255])) + .save(&source) + .unwrap(); + let original = std::fs::read(&source).unwrap(); + let project_path = create_media_project_from_image_file(library.path(), &source).unwrap(); + let meta = RecordingMeta::load_for_project(&project_path).unwrap(); + assert!(matches!( + meta.studio_meta().unwrap(), + StudioRecordingMeta::MultipleSegments { inner } if inner.segments.is_empty() + )); + let config = ProjectConfiguration::load(&project_path).unwrap(); + let image = &config.timeline.unwrap().image_segments[0]; + assert_eq!(image.end, 5.0); + assert_eq!( + std::fs::read(project_path.join(&image.path)).unwrap(), + original + ); + assert_eq!(std::fs::read(&source).unwrap(), original); + assert!(project_path.join("screenshots/display.jpg").is_file()); + + let damaged = external.path().join("damaged.png"); + std::fs::write(&damaged, b"invalid").unwrap(); + assert!(create_media_project_from_image_file(library.path(), &damaged).is_err()); + assert_eq!(std::fs::read_dir(library.path()).unwrap().count(), 1); + } + #[test] fn imported_video_frames_share_reference_counted_pixel_storage() { let mut source = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUV420P, 16, 12); diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index a78c370641e..4609ba3dbf6 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -99,9 +99,10 @@ use recording::{InProgressRecording, RecordingEvent, RecordingInputKind}; use scap_targets::{Display, DisplayId, WindowId, bounds::LogicalBounds}; use screenshot_editor::{ PendingScreenshotEditorInstances, ScreenshotEditorInstances, WindowScreenshotEditorInstance, - create_screenshot_editor_instance, prewarm_screenshot_background, recognize_screenshot_text, - render_screenshot_for_export, render_screenshot_png, render_screenshot_project_for_export, - update_screenshot_config, + close_image_drawing_instance, commit_image_drawing, create_image_drawing_instance, + create_screenshot_editor_instance, image_drawing_temp_path, prewarm_screenshot_background, + recognize_screenshot_text, render_screenshot_for_export, render_screenshot_png, + render_screenshot_project_for_export, update_screenshot_config, }; mod gpu_context; @@ -6801,7 +6802,11 @@ fn specta_builder() -> tauri_specta::Builder { export::generate_export_preview, export::generate_export_preview_fast, import::start_video_import, + import::create_media_project_from_video, + import::create_media_project_from_image, import::add_existing_recording_to_editor, + import::import_editor_image, + import::import_editor_video, import::start_image_import, import::check_import_ready, copy_file_to_path, @@ -6820,6 +6825,8 @@ fn specta_builder() -> tauri_specta::Builder { get_editor_project_path, get_mic_waveforms, get_system_audio_waveforms, + audio_library::get_imported_waveform, + audio_library::cancel_imported_waveforms, audio_library::list_audio_library, audio_library::add_audio_library_track, audio_library::import_audio_track_file, @@ -6845,6 +6852,10 @@ fn specta_builder() -> tauri_specta::Builder { upload_screenshot, upload_rendered_screenshot, create_screenshot_editor_instance, + create_image_drawing_instance, + close_image_drawing_instance, + commit_image_drawing, + image_drawing_temp_path, update_screenshot_config, prewarm_screenshot_background, recognize_screenshot_text, @@ -7579,6 +7590,9 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { match event { WindowEvent::CloseRequested { api, .. } => { let window_id = CapWindowId::from_str(label).ok(); + if matches!(&window_id, Some(CapWindowId::Editor { .. })) { + audio_library::cancel_imported_waveforms_for_window(label); + } if !matches!( window_id, Some(CapWindowId::Editor { .. }) @@ -7624,6 +7638,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { } } WindowEvent::Destroyed => { + audio_library::cancel_imported_waveforms_for_window(label); fake_window::cancel_fake_window_listener(app, label); let window_id = CapWindowId::from_str(label).ok(); if let Some(window_id) = &window_id { @@ -7785,7 +7800,27 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { WindowEvent::DragDrop(tauri::DragDropEvent::Drop { paths, .. }) => { let window_id = CapWindowId::from_str(label).ok(); for path in paths { - let result = if matches!(window_id, Some(CapWindowId::Main)) { + let result = if matches!(window_id, Some(CapWindowId::Editor { .. })) + && import::is_supported_image_import_path(path) + { + path.to_str() + .ok_or_else(|| "Image path has invalid text encoding".to_string()) + .and_then(|source_path| { + window + .emit_to(label, "editor-image-dropped", source_path) + .map_err(|error| error.to_string()) + }) + } else if matches!(window_id, Some(CapWindowId::Editor { .. })) + && import::is_supported_video_import_path(path) + { + path.to_str() + .ok_or_else(|| "Video path has invalid text encoding".to_string()) + .and_then(|source_path| { + window + .emit_to(label, "editor-video-dropped", source_path) + .map_err(|error| error.to_string()) + }) + } else if matches!(window_id, Some(CapWindowId::Main)) { open_importable_from_path(path, app.clone()) } else { open_project_from_path(path, app.clone()) @@ -8448,10 +8483,13 @@ fn retire_project_window(window: &Window, window_id: &CapWindowId) { export::cancel_exports_for_window(window.label()); let label = window.label().to_string(); let pending = PendingEditorInstances::get(app); + let screenshot_pending = PendingScreenshotEditorInstances::get(app); spawn_on_runtime(async move { pending.cancel_prewarm(&label).await; + screenshot_pending.cancel_prewarm(&label).await; }); spawn_on_runtime(EditorInstances::remove(window.clone())); + spawn_on_runtime(ScreenshotEditorInstances::remove(window.clone())); } CapWindowId::ScreenshotEditor { id } => { let window_ids = ScreenshotEditorWindowIds::get(app); @@ -9174,7 +9212,7 @@ fn open_importable_from_path(path: &Path, app: AppHandle) -> Result<(), String> if import::is_supported_video_import_path(path) { let source_path = path.to_path_buf(); tokio::spawn(async move { - match import::start_video_import(app.clone(), source_path).await { + match import::create_media_project_from_video(app.clone(), source_path).await { Ok(project_path) => { if let Err(err) = (ShowCapWindow::Editor { project_path }).show(&app).await { error!("Failed to show imported video editor: {err}"); @@ -9199,9 +9237,9 @@ fn open_importable_from_path(path: &Path, app: AppHandle) -> Result<(), String> if import::is_supported_image_import_path(path) { let source_path = path.to_path_buf(); tokio::spawn(async move { - match import::start_image_import(app.clone(), source_path).await { - Ok(path) => { - if let Err(err) = (ShowCapWindow::ScreenshotEditor { path }).show(&app).await { + match import::create_media_project_from_image(app.clone(), source_path).await { + Ok(project_path) => { + if let Err(err) = (ShowCapWindow::Editor { project_path }).show(&app).await { error!("Failed to show imported image editor: {err}"); show_import_error_dialog( &app, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 87cae9cedb8..19c7b490d05 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6633,6 +6633,7 @@ pub(crate) fn recording_timeline( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), } } diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 1138336b1e7..726ba0f0710 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -1,8 +1,10 @@ use crate::PendingScreenshots; +use crate::editor_window::WindowEditorInstance; use crate::frame_ws::{WSFrame, create_watch_frame_ws}; use crate::gpu_context; -use crate::windows::{CapWindowId, ScreenshotEditorWindowIds}; +use crate::windows::{CapWindowId, EditorWindowIds, ScreenshotEditorWindowIds}; use cap_project::{ + Annotation, AspectRatio, BackgroundConfiguration, BackgroundSource, ImageSegment, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SingleSegment, StudioRecordingMeta, VideoMeta, }; @@ -16,11 +18,16 @@ use image::{ use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use specta::Type; -use std::io::Cursor; +use std::io::{Cursor, Read}; use std::str::FromStr; use std::sync::atomic::{AtomicU8, Ordering}; use std::time::Instant; -use std::{collections::HashMap, ops::Deref, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + ops::Deref, + path::{Path, PathBuf}, + sync::Arc, +}; use tauri::{ AppHandle, Manager, Runtime, Window, ipc::{CommandArg, InvokeError}, @@ -115,6 +122,7 @@ pub struct ScreenshotEditorInstance { pub pretty_name: String, pub image_width: u32, pub image_height: u32, + image_drawing: bool, source_rgba: Arc>, } @@ -175,6 +183,8 @@ impl ScreenshotEditorInstances { app_handle: &AppHandle, path: PathBuf, start_preview: bool, + initial_config: Option, + image_drawing: bool, ) -> Result, String> { let create_started = Instant::now(); @@ -285,6 +295,7 @@ impl ScreenshotEditorInstances { } else { (None, None) }; + let loaded_config = initial_config.or(loaded_config); if !start_preview { let pretty_name = recording_meta @@ -304,6 +315,7 @@ impl ScreenshotEditorInstances { pretty_name, image_width: width, image_height: height, + image_drawing, source_rgba: Arc::new(data), })); } @@ -448,6 +460,7 @@ impl ScreenshotEditorInstances { pretty_name: recording_meta.pretty_name.clone(), image_width: width, image_height: height, + image_drawing, source_rgba: source_rgba.clone(), }); ws_guard.disarm(); @@ -581,13 +594,7 @@ impl ScreenshotEditorInstances { window: &Window, path: PathBuf, ) -> Result, String> { - let CapWindowId::ScreenshotEditor { id } = - CapWindowId::from_str(window.label()).map_err(|error| error.to_string())? - else { - return Err("Invalid screenshot editor window".to_string()); - }; - let window_ids = ScreenshotEditorWindowIds::get(window.app_handle()); - with_registered_screenshot_editor(&window_ids, id, || ())?; + with_registered_screenshot_workspace(window, || ())?; let instances = match window.try_state::() { Some(s) => (*s).clone(), None => { @@ -596,7 +603,7 @@ impl ScreenshotEditorInstances { } }; let mut instances = instances.0.write().await; - if let Some(instance) = with_registered_screenshot_editor(&window_ids, id, || { + if let Some(instance) = with_registered_screenshot_workspace(window, || { instances.get(window.label()).map(|instance| { let instance = instance.clone(); let config = instance.config_tx.borrow().clone(); @@ -624,15 +631,20 @@ impl ScreenshotEditorInstances { let instance = match prewarmed { Some(instance) => instance, None => { - with_registered_screenshot_editor(&window_ids, id, || ())?; + with_registered_screenshot_workspace(window, || ())?; let cleanup_runtime = tokio::runtime::Handle::current(); - let instance = - Self::create_standalone_instance(window.app_handle(), path.clone(), true) - .await?; + let instance = Self::create_standalone_instance( + window.app_handle(), + path.clone(), + true, + None, + false, + ) + .await?; ScreenshotEditorInstanceDelivery::new(instance, cleanup_runtime) } }; - let published = with_registered_screenshot_editor(&window_ids, id, || { + let published = with_registered_screenshot_workspace(window, || { instance.adopt_into(&mut instances, window.label()) }); drop(instances); @@ -646,6 +658,44 @@ impl ScreenshotEditorInstances { Ok(instance) } + pub async fn create_for_image( + window: &Window, + source_path: PathBuf, + config: ProjectConfiguration, + ) -> Result, String> { + with_registered_screenshot_workspace(window, || ())?; + let instances = match window.try_state::() { + Some(state) => (*state).clone(), + None => { + window.manage(Self(Arc::new(RwLock::new(HashMap::new())))); + (*window.state::()).clone() + } + }; + let mut instances = instances.0.write().await; + if let Some(existing) = instances.get(window.label()) { + if existing.path == source_path { + return Ok(existing.clone()); + } + return Err("Another image drawing workspace is already active".to_string()); + } + let instance = Self::create_standalone_instance( + window.app_handle(), + source_path, + true, + Some(config), + true, + ) + .await?; + if let Err(error) = with_registered_screenshot_workspace(window, || { + instances.insert(window.label().to_string(), instance.clone()); + }) { + drop(instances); + instance.dispose().await; + return Err(error); + } + Ok(instance) + } + pub async fn remove(window: Window) { let instances = match window.try_state::() { Some(s) => (*s).clone(), @@ -682,14 +732,20 @@ impl ScreenshotEditorInstances { } } -fn with_registered_screenshot_editor( - window_ids: &ScreenshotEditorWindowIds, - id: u32, +fn with_registered_screenshot_workspace( + window: &Window, action: impl FnOnce() -> T, ) -> Result { - let ids = window_ids.ids.lock().map_err(|error| error.to_string())?; + let (ids, id) = match CapWindowId::from_str(window.label())? { + CapWindowId::Editor { id } => (EditorWindowIds::get(window.app_handle()).ids, id), + CapWindowId::ScreenshotEditor { id } => { + (ScreenshotEditorWindowIds::get(window.app_handle()).ids, id) + } + _ => return Err("Invalid editor window for screenshot workspace".to_string()), + }; + let ids = ids.lock().map_err(|error| error.to_string())?; if !ids.iter().any(|(_, registered_id)| *registered_id == id) { - return Err("Screenshot editor window is no longer registered".to_string()); + return Err("Screenshot workspace window is no longer registered".to_string()); } Ok(action()) } @@ -706,45 +762,6 @@ impl PendingScreenshotEditorInstances { } } - pub async fn start_prewarm(app: &AppHandle, window_label: String, path: PathBuf) { - let Ok(CapWindowId::ScreenshotEditor { id }) = CapWindowId::from_str(&window_label) else { - return; - }; - let window_ids = ScreenshotEditorWindowIds::get(app); - let pending = Self::get(app); - let app = app.clone(); - let tx = { - let mut instances = pending.0.write().await; - let admitted = with_registered_screenshot_editor(&window_ids, id, || { - use std::collections::hash_map::Entry; - match instances.entry(window_label) { - Entry::Vacant(entry) => { - let (tx, rx) = watch::channel(None); - entry.insert(rx); - Some(tx) - } - Entry::Occupied(_) => None, - } - }); - match admitted { - Ok(Some(tx)) => tx, - Ok(None) => return, - Err(error) => { - tracing::debug!(%error, "Skipping prewarm for a retired screenshot editor"); - return; - } - } - }; - - let cleanup_runtime = tokio::runtime::Handle::current(); - tokio::spawn(async move { - let result = ScreenshotEditorInstances::create_standalone_instance(&app, path, true) - .await - .map(|instance| ScreenshotEditorInstanceDelivery::new(instance, cleanup_runtime)); - tx.send(Some(result)).ok(); - }); - } - pub async fn take_prewarmed(&self, window_label: &str) -> Option { let mut instances = self.0.write().await; instances.remove(window_label) @@ -891,20 +908,20 @@ struct ScreenshotOcrImage { pub async fn create_screenshot_editor_instance( window: Window, ) -> Result { - let CapWindowId::ScreenshotEditor { id } = - CapWindowId::from_str(window.label()).map_err(|e| e.to_string())? - else { - return Err("Invalid window".to_string()); - }; - - let path = { - let window_ids = ScreenshotEditorWindowIds::get(window.app_handle()); - let window_ids = window_ids.ids.lock().unwrap(); - let Some((path, _)) = window_ids.iter().find(|(_, _id)| *_id == id) else { - return Err("Screenshot editor instance not found".to_string()); - }; - path.clone() + let (ids, id) = match CapWindowId::from_str(window.label())? { + CapWindowId::Editor { id } => (EditorWindowIds::get(window.app_handle()).ids, id), + CapWindowId::ScreenshotEditor { id } => { + (ScreenshotEditorWindowIds::get(window.app_handle()).ids, id) + } + _ => return Err("Invalid window for screenshot workspace".to_string()), }; + let path = ids + .lock() + .map_err(|error| error.to_string())? + .iter() + .find(|(_, registered_id)| *registered_id == id) + .map(|(path, _)| path.clone()) + .ok_or("Screenshot project window not found")?; let instance = ScreenshotEditorInstances::get_or_create(&window, path).await?; let config = instance.config_tx.borrow().config.clone(); @@ -919,6 +936,501 @@ pub async fn create_screenshot_editor_instance( }) } +fn image_segment_source( + project_path: &Path, + segment: &ImageSegment, +) -> Result<(String, PathBuf), String> { + let relative = segment.source_path.as_deref().unwrap_or(&segment.path); + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path.components().any(|component| { + matches!( + component, + std::path::Component::ParentDir | std::path::Component::Prefix(_) + ) + }) + { + return Err("Image source path escapes the project".to_string()); + } + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let source_path = project_path + .join(relative_path) + .canonicalize() + .map_err(|error| error.to_string())?; + if !source_path.starts_with(&root) + || !crate::import::is_supported_image_import_path(&source_path) + { + return Err("Image source is unavailable inside the project".to_string()); + } + Ok((relative.to_string(), source_path)) +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImageAppearance { + version: u8, + background: BackgroundConfiguration, + aspect_ratio: Option, +} + +fn image_appearance_file(project_path: &Path, image_path: &str) -> Option { + let relative = Path::new(image_path); + if relative.parent()? != Path::new("content/images") || relative.extension()?.to_str()? != "png" + { + return None; + } + let stem = relative.file_stem()?.to_str()?; + if stem.len() != 40 + || !stem.starts_with("drawing-") + || !stem[8..] + .chars() + .all(|character| character.is_ascii_hexdigit()) + { + return None; + } + Some( + project_path + .join("content/images") + .join(format!("{stem}.style.json")), + ) +} + +fn portable_project_path(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(root) + .map_err(|_| "Screenshot background is outside the project".to_string())?; + let mut parts = Vec::new(); + for component in relative.components() { + let std::path::Component::Normal(part) = component else { + return Err("Screenshot background path is invalid".to_string()); + }; + parts.push(part.to_string_lossy().into_owned()); + } + Ok(parts.join("/")) +} + +fn load_image_appearance( + project_path: &Path, + image_path: &str, +) -> Result, String> { + let Some(file) = image_appearance_file(project_path, image_path) else { + return Ok(None); + }; + if !file.exists() { + return Ok(None); + } + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let canonical = file.canonicalize().map_err(|error| error.to_string())?; + if !canonical.starts_with(&root) { + return Err("Screenshot appearance escapes the project".to_string()); + } + let mut input = std::fs::File::open(&canonical).map_err(|error| error.to_string())?; + let mut bytes = Vec::new(); + input + .by_ref() + .take(65_537) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 65_536 { + return Err("Screenshot appearance exceeds the size limit".to_string()); + } + let mut appearance: ImageAppearance = + serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if appearance.version != 1 { + return Err("Screenshot appearance version is unsupported".to_string()); + } + match &mut appearance.background.source { + BackgroundSource::Wallpaper { path } | BackgroundSource::Image { path } => { + if let Some(relative) = path { + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err("Screenshot background path is invalid".to_string()); + } + let source = project_path + .join(relative_path) + .canonicalize() + .map_err(|error| error.to_string())?; + if !source.starts_with(&root) || !source.is_file() { + return Err("Screenshot background escapes the project".to_string()); + } + *relative = source.to_string_lossy().into_owned(); + } + } + _ => {} + } + Ok(Some(appearance)) +} + +fn save_image_appearance( + project_path: &Path, + image_path: &str, + mut appearance: ImageAppearance, +) -> Result, String> { + let file = image_appearance_file(project_path, image_path) + .ok_or("Screenshot appearance output path is invalid")?; + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let images_dir = project_path.join("content/images"); + let canonical_images = images_dir + .canonicalize() + .map_err(|error| error.to_string())?; + if !canonical_images.starts_with(&root) { + return Err("Screenshot appearance directory escapes the project".to_string()); + } + let mut created = Vec::new(); + let result = (|| -> Result<(), String> { + match &mut appearance.background.source { + BackgroundSource::Wallpaper { path } | BackgroundSource::Image { path } => { + if let Some(source_path) = path { + let source = Path::new(source_path) + .canonicalize() + .map_err(|error| error.to_string())?; + let extension = source + .extension() + .and_then(|part| part.to_str()) + .ok_or("Screenshot background type is unsupported")? + .to_ascii_lowercase(); + if !matches!( + extension.as_str(), + "png" | "jpg" | "jpeg" | "webp" | "gif" | "bmp" | "tif" | "tiff" + ) { + return Err("Screenshot background type is unsupported".to_string()); + } + let portable = if source.starts_with(&root) { + portable_project_path(&root, &source)? + } else { + let mut input = + std::fs::File::open(&source).map_err(|error| error.to_string())?; + let size = input.metadata().map_err(|error| error.to_string())?.len(); + if size == 0 || size > 64 * 1024 * 1024 { + return Err("Screenshot background exceeds the size limit".to_string()); + } + let name = format!( + "screenshot-background-{}.{}", + uuid::Uuid::new_v4().simple(), + extension + ); + let output = canonical_images.join(&name); + let mut target = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .map_err(|error| error.to_string())?; + created.push(output); + std::io::copy(&mut input, &mut target) + .map_err(|error| error.to_string())?; + target.sync_all().map_err(|error| error.to_string())?; + format!("content/images/{name}") + }; + *source_path = portable; + } + } + _ => {} + } + let bytes = serde_json::to_vec(&appearance).map_err(|error| error.to_string())?; + if bytes.len() > 65_536 { + return Err("Screenshot appearance exceeds the size limit".to_string()); + } + let temp = canonical_images.join(format!( + ".screenshot-style-{}.tmp", + uuid::Uuid::new_v4().simple() + )); + let mut target = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(|error| error.to_string())?; + created.push(temp.clone()); + std::io::Write::write_all(&mut target, &bytes).map_err(|error| error.to_string())?; + target.sync_all().map_err(|error| error.to_string())?; + std::fs::rename(&temp, &file).map_err(|error| error.to_string())?; + let _ = created.pop(); + created.push(file); + Ok(()) + })(); + if let Err(error) = result { + for path in &created { + let _ = std::fs::remove_file(path); + } + return Err(error); + } + Ok(created) +} + +#[tauri::command] +#[specta::specta] +pub async fn create_image_drawing_instance( + window: Window, + image_index: u32, +) -> Result { + let CapWindowId::Editor { id } = CapWindowId::from_str(window.label())? else { + return Err("Image drawing requires an editor window".to_string()); + }; + let ids = EditorWindowIds::get(window.app_handle()).ids; + let project_path = ids + .lock() + .map_err(|error| error.to_string())? + .iter() + .find(|(_, registered_id)| *registered_id == id) + .map(|(path, _)| path.clone()) + .ok_or("Editor project window not found")?; + let project = ProjectConfiguration::load(&project_path).map_err(|error| error.to_string())?; + let segment = project + .timeline + .as_ref() + .and_then(|timeline| timeline.image_segments.get(image_index as usize)) + .ok_or("Image track item not found")?; + let (_, source_path) = image_segment_source(&project_path, segment)?; + let mut drawing_config = ProjectConfiguration::default(); + drawing_config.background.source = BackgroundSource::Color { + value: [255, 255, 255], + alpha: 0, + }; + drawing_config.background.padding = 0.0; + drawing_config.background.shadow = 0.0; + if let Some(appearance) = load_image_appearance(&project_path, &segment.path)? { + drawing_config.background = appearance.background; + drawing_config.aspect_ratio = appearance.aspect_ratio; + } else if segment.path == "original.png" { + drawing_config.background = project.background.clone(); + drawing_config.aspect_ratio = project.aspect_ratio.clone(); + } + drawing_config.annotations = segment.annotations.clone(); + let instance = + ScreenshotEditorInstances::create_for_image(&window, source_path, drawing_config).await?; + let config = instance.config_tx.borrow().config.clone(); + Ok(SerializedScreenshotEditorInstance { + frames_socket_url: format!("ws://localhost:{}", instance.ws_port), + path: instance.path.clone(), + config: Some(config), + pretty_name: segment.name.clone(), + image_width: instance.image_width, + image_height: instance.image_height, + }) +} + +#[tauri::command] +#[specta::specta] +pub async fn close_image_drawing_instance(window: Window) -> Result<(), String> { + if !matches!( + CapWindowId::from_str(window.label())?, + CapWindowId::Editor { .. } + ) { + return Err("Image drawing requires an editor window".to_string()); + } + ScreenshotEditorInstances::remove(window).await; + Ok(()) +} + +#[derive(Serialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct ImageDrawingCommit { + pub image_index: u32, + pub path: String, + pub source_path: String, + pub annotations: Vec, +} + +fn commit_image_drawing_file( + project_path: &Path, + image_index: u32, + expected_source: &Path, + annotations: Vec, + appearance: ImageAppearance, + png_bytes: &[u8], +) -> Result { + if png_bytes.is_empty() || png_bytes.len() > 64 * 1024 * 1024 { + return Err("Drawing image exceeds the import size limit".to_string()); + } + let (width, height) = + image::ImageReader::with_format(Cursor::new(png_bytes), image::ImageFormat::Png) + .into_dimensions() + .map_err(|error| format!("Drawing PNG is invalid: {error}"))?; + if width == 0 + || height == 0 + || width > MAX_DIMENSION + || height > MAX_DIMENSION + || u64::from(width) * u64::from(height) > 64_000_000 + { + return Err("Drawing image dimensions exceed the editor limit".to_string()); + } + image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png) + .map_err(|error| format!("Drawing PNG is invalid: {error}"))?; + let mut project = + ProjectConfiguration::load(project_path).map_err(|error| error.to_string())?; + let segment = project + .timeline + .as_mut() + .and_then(|timeline| timeline.image_segments.get_mut(image_index as usize)) + .ok_or("Image track item not found")?; + let (source_path, canonical_source) = image_segment_source(project_path, segment)?; + if canonical_source != expected_source { + return Err("Image source changed while drawing".to_string()); + } + let images_dir = project_path.join("content/images"); + std::fs::create_dir_all(&images_dir).map_err(|error| error.to_string())?; + let root = project_path + .canonicalize() + .map_err(|error| error.to_string())?; + let canonical_images = images_dir + .canonicalize() + .map_err(|error| error.to_string())?; + if !canonical_images.starts_with(root) { + return Err("Drawing asset directory escapes the project".to_string()); + } + let filename = format!("drawing-{}.png", uuid::Uuid::new_v4().simple()); + let relative_output = format!("content/images/{filename}"); + let output = images_dir.join(&filename); + let temp = images_dir.join(format!(".{filename}.tmp")); + let write_result = (|| -> Result<(), std::io::Error> { + let mut file = std::fs::File::create(&temp)?; + std::io::Write::write_all(&mut file, png_bytes)?; + file.sync_all()?; + std::fs::rename(&temp, &output) + })(); + if let Err(error) = write_result { + let _ = std::fs::remove_file(&temp); + return Err(format!("Cannot save drawing asset: {error}")); + } + let style_files = match save_image_appearance(project_path, &relative_output, appearance) { + Ok(files) => files, + Err(error) => { + let _ = std::fs::remove_file(&output); + return Err(error); + } + }; + segment.source_path = Some(source_path.clone()); + segment.annotations = annotations.clone(); + segment.path = relative_output.clone(); + if let Err(error) = project.write(project_path) { + let _ = std::fs::remove_file(&output); + for file in style_files { + let _ = std::fs::remove_file(file); + } + return Err(format!("Cannot save image drawing: {error}")); + } + Ok(ImageDrawingCommit { + image_index, + path: relative_output, + source_path, + annotations, + }) +} + +fn image_drawing_cache_dir(app: &AppHandle) -> Result { + let app_data = app + .path() + .app_data_dir() + .map_err(|error| error.to_string())?; + std::fs::create_dir_all(&app_data).map_err(|error| error.to_string())?; + let canonical_app_data = app_data.canonicalize().map_err(|error| error.to_string())?; + let cache_dir = app_data.join("image-drawing-temp"); + std::fs::create_dir_all(&cache_dir).map_err(|error| error.to_string())?; + let canonical_cache = cache_dir + .canonicalize() + .map_err(|error| error.to_string())?; + if !canonical_cache.starts_with(canonical_app_data) { + return Err("Drawing cache directory escapes app data".to_string()); + } + Ok(canonical_cache) +} + +#[tauri::command] +#[specta::specta] +pub fn image_drawing_temp_path(app: AppHandle) -> Result { + Ok(image_drawing_cache_dir(&app)?.join(format!( + "cap-image-drawing-{}.png", + uuid::Uuid::new_v4().simple() + ))) +} + +#[tauri::command] +#[specta::specta] +pub async fn commit_image_drawing( + window: Window, + instance: WindowScreenshotEditorInstance, + editor: WindowEditorInstance, + image_index: u32, + png_path: PathBuf, +) -> Result { + let CapWindowId::Editor { id } = CapWindowId::from_str(window.label())? else { + return Err("Image drawing requires an editor window".to_string()); + }; + let ids = EditorWindowIds::get(window.app_handle()).ids; + let project_path = ids + .lock() + .map_err(|error| error.to_string())? + .iter() + .find(|(_, registered_id)| *registered_id == id) + .map(|(path, _)| path.clone()) + .ok_or("Editor project window not found")?; + if editor.project_path != project_path { + return Err("Editor project changed while drawing".to_string()); + } + if instance + .path + .extension() + .and_then(|extension| extension.to_str()) + == Some("cap") + { + return Err("Image drawing instance is unavailable".to_string()); + } + let source_path = instance + .path + .canonicalize() + .map_err(|error| error.to_string())?; + let drawing_config = instance.config_tx.borrow().config.clone(); + let annotations = drawing_config.annotations.clone(); + let appearance = ImageAppearance { + version: 1, + background: drawing_config.background, + aspect_ratio: drawing_config.aspect_ratio, + }; + let project_for_write = project_path.clone(); + let cache_dir = image_drawing_cache_dir(window.app_handle())?; + let committed = tokio::task::spawn_blocking(move || { + let canonical_png = png_path.canonicalize().map_err(|error| error.to_string())?; + let valid_name = png_path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("cap-image-drawing-") && name.ends_with(".png")); + if !valid_name || canonical_png.parent() != Some(cache_dir.as_path()) { + return Err("Drawing temporary file is outside the editor cache".to_string()); + } + let size = std::fs::metadata(&canonical_png) + .map_err(|error| error.to_string())? + .len(); + if size == 0 || size > 64 * 1024 * 1024 { + return Err("Drawing image exceeds the import size limit".to_string()); + } + let png_bytes = std::fs::read(&canonical_png).map_err(|error| error.to_string())?; + let committed = commit_image_drawing_file( + &project_for_write, + image_index, + &source_path, + annotations, + appearance, + &png_bytes, + )?; + let _ = std::fs::remove_file(&canonical_png); + Ok(committed) + }) + .await + .map_err(|error| format!("Drawing save worker failed: {error}"))??; + let config = ProjectConfiguration::load(&project_path).map_err(|error| error.to_string())?; + editor.project_config.0.send(config).ok(); + Ok(committed) +} + /// Renders one tiny throwaway frame on the shared GPU at startup so the Metal /// render pipelines are compiled before the user opens the editor. On Apple GPUs /// pipeline *creation* is cheap but the driver defers shader compilation to the @@ -1089,23 +1601,25 @@ pub async fn update_screenshot_config( config: config.clone(), }); - if !save { + if !save || instance.image_drawing { return Ok(()); } - let Some(parent) = instance.path.parent() else { - return Ok(()); - }; - - if parent.extension().and_then(|s| s.to_str()) == Some("cap") { - let path = parent.to_path_buf(); - if let Err(e) = config.write(&path) { - eprintln!("Failed to save screenshot config: {e}"); - } else { - println!("Saved screenshot config to {path:?}"); - } + let bundle = if instance.path.is_dir() + && instance + .path + .extension() + .and_then(|extension| extension.to_str()) + == Some("cap") + { + Some(instance.path.as_path()) } else { - println!("Not saving config: parent {parent:?} is not a .cap directory"); + instance.path.parent().filter(|parent| { + parent.extension().and_then(|extension| extension.to_str()) == Some("cap") + }) + }; + if let Some(bundle) = bundle { + config.write(bundle).map_err(|error| error.to_string())?; } Ok(()) } @@ -1617,7 +2131,9 @@ pub async fn render_screenshot_project_for_export( app: AppHandle, path: PathBuf, ) -> Result { - let instance = ScreenshotEditorInstances::create_standalone_instance(&app, path, false).await?; + let instance = + ScreenshotEditorInstances::create_standalone_instance(&app, path, false, None, false) + .await?; let config = instance.config_tx.borrow().config.clone(); let image_width = instance.image_width; let image_height = instance.image_height; @@ -1895,3 +2411,185 @@ pub async fn render_screenshot_png(instance: &ScreenshotEditorInstance) -> Resul Ok(png_data.into_inner()) } + +#[cfg(test)] +mod image_drawing_tests { + use super::*; + + fn png(rgb: [u8; 3]) -> Vec { + let pixels = image::RgbaImage::from_pixel(8, 6, image::Rgba([rgb[0], rgb[1], rgb[2], 255])); + let mut bytes = Vec::new(); + PngEncoder::new(&mut bytes) + .write_image(pixels.as_raw(), 8, 6, image::ExtendedColorType::Rgba8) + .unwrap(); + bytes + } + + fn appearance(padding: f64) -> ImageAppearance { + let mut background = BackgroundConfiguration::default(); + background.padding = padding; + ImageAppearance { + version: 1, + background, + aspect_ratio: Some(AspectRatio::Wide), + } + } + + #[tokio::test] + async fn image_drawing_updates_keep_media_project_config() { + let temp = tempfile::tempdir().unwrap(); + let bundle = cap_project::create_media_project(temp.path(), "Drawing config test").unwrap(); + let mut project = ProjectConfiguration::load(&bundle).unwrap(); + project + .timeline + .as_mut() + .unwrap() + .image_segments + .push(ImageSegment { + end: 5.0, + path: "source.png".to_string(), + ..Default::default() + }); + project.write(&bundle).unwrap(); + let original = std::fs::read(bundle.join("project-config.json")).unwrap(); + let (config_tx, _config_rx) = watch::channel(ScreenshotConfigUpdate { + revision: 0, + config: ProjectConfiguration::default(), + }); + let instance = Arc::new(ScreenshotEditorInstance { + ws_port: 0, + ws_shutdown_token: CancellationToken::new(), + config_tx, + path: bundle.join("source.png"), + pretty_name: "Drawing config test".to_string(), + image_width: 8, + image_height: 6, + image_drawing: true, + source_rgba: Arc::new(Vec::new()), + }); + let mut drawing = ProjectConfiguration::default(); + drawing.background.padding = 42.0; + update_screenshot_config( + WindowScreenshotEditorInstance(instance.clone()), + drawing, + true, + 1, + ) + .await + .unwrap(); + assert_eq!(instance.config_tx.borrow().config.background.padding, 42.0); + assert_eq!( + std::fs::read(bundle.join("project-config.json")).unwrap(), + original + ); + } + + #[test] + fn image_drawing_preserves_source_and_previous_edits() { + let temp = tempfile::tempdir().unwrap(); + let bundle = cap_project::create_media_project(temp.path(), "Drawing test").unwrap(); + let images = bundle.join("content/images"); + std::fs::create_dir_all(&images).unwrap(); + let source = images.join("source.png"); + let source_bytes = png([255, 255, 255]); + std::fs::write(&source, &source_bytes).unwrap(); + let mut project = ProjectConfiguration::load(&bundle).unwrap(); + project + .timeline + .as_mut() + .unwrap() + .image_segments + .push(ImageSegment { + end: 5.0, + path: "content/images/source.png".to_string(), + ..Default::default() + }); + project.write(&bundle).unwrap(); + let annotation: Annotation = serde_json::from_value(serde_json::json!({ + "id":"rectangle-one", "type":"rectangle", "x":1.0, "y":1.0, + "width":4.0, "height":3.0, "strokeColor":"#ff0000", + "strokeWidth":2.0, "fillColor":"transparent", "opacity":1.0, + "rotation":0.0, "text":null + })) + .unwrap(); + let first = commit_image_drawing_file( + &bundle, + 0, + &source.canonicalize().unwrap(), + vec![annotation.clone()], + appearance(12.0), + &png([255, 0, 0]), + ) + .unwrap(); + assert_ne!(first.path, first.source_path); + assert_eq!(std::fs::read(&source).unwrap(), source_bytes); + assert_eq!(first.annotations.len(), 1); + assert_eq!( + load_image_appearance(&bundle, &first.path) + .unwrap() + .unwrap() + .background + .padding, + 12.0 + ); + let second = commit_image_drawing_file( + &bundle, + 0, + &source.canonicalize().unwrap(), + vec![annotation], + appearance(28.0), + &png([0, 0, 255]), + ) + .unwrap(); + assert_ne!(first.path, second.path); + assert_eq!( + std::fs::read(bundle.join(&first.path)).unwrap(), + png([255, 0, 0]) + ); + assert_eq!( + std::fs::read(bundle.join(&second.path)).unwrap(), + png([0, 0, 255]) + ); + let persisted = ProjectConfiguration::load(&bundle).unwrap(); + let segment = &persisted.timeline.unwrap().image_segments[0]; + assert_eq!( + segment.source_path.as_deref(), + Some("content/images/source.png") + ); + assert_eq!(segment.path, second.path); + assert_eq!(segment.annotations.len(), 1); + assert_eq!(std::fs::read(&source).unwrap(), source_bytes); + assert_eq!( + load_image_appearance(&bundle, &second.path) + .unwrap() + .unwrap() + .background + .padding, + 28.0 + ); + } + + #[test] + fn screenshot_background_stays_with_moved_project() { + let temp = tempfile::tempdir().unwrap(); + let bundle = cap_project::create_media_project(temp.path(), "Appearance test").unwrap(); + std::fs::create_dir_all(bundle.join("content/images")).unwrap(); + let background = temp.path().join("chosen-background.png"); + let bytes = png([18, 30, 42]); + std::fs::write(&background, &bytes).unwrap(); + let image_path = "content/images/drawing-00000000000000000000000000000001.png"; + let mut style = appearance(16.0); + style.background.source = BackgroundSource::Image { + path: Some(background.to_string_lossy().into_owned()), + }; + save_image_appearance(&bundle, image_path, style).unwrap(); + let moved = temp.path().join("moved.cap"); + std::fs::rename(&bundle, &moved).unwrap(); + let loaded = load_image_appearance(&moved, image_path).unwrap().unwrap(); + let BackgroundSource::Image { path: Some(path) } = loaded.background.source else { + panic!("Background image was not restored"); + }; + assert!(Path::new(&path).starts_with(moved.canonicalize().unwrap())); + assert_eq!(std::fs::read(path).unwrap(), bytes); + } +} diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 8e25dc422a0..f929b4fc245 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -1421,6 +1421,25 @@ impl ShowCapWindow { &'a self, app: &'a AppHandle, ) -> futures::future::BoxFuture<'a, tauri::Result> { + if let Self::ScreenshotEditor { path } = self { + let path = path.clone(); + return Box::pin(async move { + let project_path = if path.is_dir() + && path.extension().and_then(|extension| extension.to_str()) == Some("cap") + { + path + } else if let Some(parent) = path.parent().filter(|parent| { + parent.extension().and_then(|extension| extension.to_str()) == Some("cap") + }) { + parent.to_path_buf() + } else { + crate::import::create_media_project_from_image(app.clone(), path) + .await + .map_err(|error| tauri::Error::Io(std::io::Error::other(error)))? + }; + Self::Editor { project_path }.show(app).await + }); + } let picker_session = matches!(self, Self::TargetSelectOverlay { .. }) .then(|| app.state::().picker_session()) .flatten(); @@ -1432,6 +1451,9 @@ impl ShowCapWindow { app: &'a AppHandle, session: u32, ) -> futures::future::BoxFuture<'a, tauri::Result> { + if matches!(self, Self::ScreenshotEditor { .. }) { + return self.show(app); + } Box::pin(self.show_inner(app, Some(session))) } @@ -1474,20 +1496,6 @@ impl ShowCapWindow { .await?, ) } - Self::ScreenshotEditor { path } => { - let state = app.state::(); - Some( - ProjectWindowOpening::acquire( - app, - path, - state.ids.clone(), - &state.counter, - &state.open_gates, - |id| CapWindowId::ScreenshotEditor { id }, - ) - .await?, - ) - } _ => None, }; let window_id = project_opening @@ -2357,30 +2365,7 @@ impl ShowCapWindow { window } - Self::ScreenshotEditor { path } => { - hide_recording_windows(app, false); - release_camera_preview_if_idle(app); - - PendingScreenshotEditorInstances::start_prewarm(app, _id.label(), path.clone()) - .await; - - let window = self - .window_builder_with_id(app, "/screenshot-editor", &_id, _id.label()) - .maximizable(true) - .focused(true) - .build()?; - if let Some(opening) = project_opening.as_mut() { - opening.own_window(&window); - } - lock_window_text_scale(&window); - - fit_content_window_bounds(&window, &_id, true).await; - - window.show().ok(); - window.set_focus().ok(); - - window - } + Self::ScreenshotEditor { .. } => unreachable!(), Self::Upgrade => { crate::hide_main_window(app); @@ -4194,7 +4179,8 @@ impl Drop for ProjectWindowOpening { let native_window = window.as_ref().window().clone(); match self.id { CapWindowId::Editor { .. } => { - tauri::async_runtime::spawn(EditorInstances::remove(native_window)); + tauri::async_runtime::spawn(EditorInstances::remove(native_window.clone())); + tauri::async_runtime::spawn(ScreenshotEditorInstances::remove(native_window)); } CapWindowId::ScreenshotEditor { .. } => { tauri::async_runtime::spawn(ScreenshotEditorInstances::remove(native_window)); @@ -4211,6 +4197,9 @@ impl Drop for ProjectWindowOpening { PendingEditorInstances::get(&app) .cancel_prewarm(&label) .await; + PendingScreenshotEditorInstances::get(&app) + .cancel_prewarm(&label) + .await; } CapWindowId::ScreenshotEditor { .. } => { PendingScreenshotEditorInstances::get(&app) @@ -4239,8 +4228,6 @@ impl EditorWindowIds { #[derive(Default, Clone)] pub struct ScreenshotEditorWindowIds { pub ids: Arc>>, - pub counter: Arc, - open_gates: ProjectWindowOpenGates, } impl ScreenshotEditorWindowIds { diff --git a/apps/desktop/src/app.tsx b/apps/desktop/src/app.tsx index 64f8ed73a7f..5f4a7faaac8 100644 --- a/apps/desktop/src/app.tsx +++ b/apps/desktop/src/app.tsx @@ -95,7 +95,6 @@ const InProgressRecordingPage = lazy( const ModeSelectPage = lazy(() => import("./routes/mode-select")); const NotificationsPage = lazy(() => import("./routes/notifications")); const RecordingsOverlayPage = lazy(() => import("./routes/recordings-overlay")); -const ScreenshotEditorPage = lazy(() => import("./routes/screenshot-editor")); const TargetSelectOverlayPage = lazy( () => import("./routes/target-select-overlay"), ); @@ -252,11 +251,6 @@ function Inner() { - { - if ( - typeof exclude === "object" && - exclude !== null && - "image" in exclude && - exclude.image === index - ) - return; - if (!segment.enabled || t < segment.start || t >= segment.end) return; - const angle = (segment.rotation * Math.PI) / 180; - const outputWidth = layout?.output_width ?? 1920; - const outputHeight = layout?.output_height ?? 1080; - const w = - (Math.abs(segment.size.x * outputWidth * Math.cos(angle)) + - Math.abs(segment.size.y * outputHeight * Math.sin(angle))) / - outputWidth; - const h = - (Math.abs(segment.size.x * outputWidth * Math.sin(angle)) + - Math.abs(segment.size.y * outputHeight * Math.cos(angle))) / - outputHeight; - rects.push({ - x: segment.center.x - w / 2, - y: segment.center.y - h / 2, - w, - h, + for (const [kind, segments] of [ + ["image", project.timeline?.imageSegments ?? []], + ["video", project.timeline?.videoSegments ?? []], + ] as const) { + segments.forEach((segment, index) => { + if ( + typeof exclude === "object" && + exclude !== null && + ((kind === "image" && + "image" in exclude && + exclude.image === index) || + (kind === "video" && "video" in exclude && exclude.video === index)) + ) + return; + if (!segment.enabled || t < segment.start || t >= segment.end) return; + const angle = (segment.rotation * Math.PI) / 180; + const outputWidth = layout?.output_width ?? 1920; + const outputHeight = layout?.output_height ?? 1080; + const w = + (Math.abs(segment.size.x * outputWidth * Math.cos(angle)) + + Math.abs(segment.size.y * outputHeight * Math.sin(angle))) / + outputWidth; + const h = + (Math.abs(segment.size.x * outputWidth * Math.sin(angle)) + + Math.abs(segment.size.y * outputHeight * Math.cos(angle))) / + outputHeight; + rects.push({ + x: segment.center.x - w / 2, + y: segment.center.y - h / 2, + w, + h, + }); }); - }); + } // The classic camera-inset margin lines only make sense for the camera // and display; for text/mask boxes they just cause spurious re-snaps // right next to the frame-edge lines. diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 8d1e8615992..e1aaf86f45d 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -151,6 +151,7 @@ import { topSlideAnimateClasses, } from "./ui"; import { formatTime } from "./utils"; +import { VideoSegmentConfig } from "./video-segment-config"; import { ZoomModeHelper } from "./ZoomModeHelper"; // Split out of the sidebar chunk: the captions tab is not visible at first @@ -787,7 +788,7 @@ function ConfigSidebarContent() { onChange={(v) => setProject("audio", "mute", v)} /> - {editorInstance.recordings.segments[0].mic?.channels === 2 && ( + {editorInstance.recordings.segments[0]?.mic?.channels === 2 && ( options={STEREO_MODES} @@ -1202,6 +1203,18 @@ function ConfigSidebarContent() { {(index) => } + + + {(index) => } + + { const captionSelection = selection(); diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index aac65a6c360..03e65b667b7 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -8,12 +8,13 @@ import { makePersisted } from "@solid-primitives/storage"; import { createMutation, createQuery, skipToken } from "@tanstack/solid-query"; import { convertFileSrc } from "@tauri-apps/api/core"; import { LogicalPosition } from "@tauri-apps/api/dpi"; -import { emitTo } from "@tauri-apps/api/event"; +import { emitTo, listen } from "@tauri-apps/api/event"; import { Menu } from "@tauri-apps/api/menu"; import { getCurrentWindow } from "@tauri-apps/api/window"; import { ask } from "@tauri-apps/plugin-dialog"; import { cx } from "cva"; import { + type Accessor, createEffect, createMemo, createResource, @@ -43,7 +44,7 @@ import { import { Toggle } from "~/components/Toggle"; import { composeEventHandlers } from "~/utils/composeEventHandlers"; import { createTauriEventListener } from "~/utils/createEventListener"; -import { commands, events } from "~/utils/tauri"; +import { commands, events, type ImageDrawingCommit } from "~/utils/tauri"; import { ConfigSidebar } from "./ConfigSidebar"; import { EditorContextProvider, @@ -59,9 +60,11 @@ import { DEFAULT_TIMELINE_HEIGHT, editorVerticalLayout } from "./editor-layout"; import { EditorSkeleton } from "./editor-skeleton"; import { Header, type TitleSaveRegistration } from "./Header"; import { ImportProgress } from "./ImportProgress"; +import { ImageEditorSidebar } from "./image-editor-sidebar"; import { PlayerContent } from "./Player"; import { usePreparingEditor } from "./preparing-editor-context"; import { Timeline } from "./Timeline"; +import { getUsedTrackCount } from "./timelineTracks"; import { Dialog, DialogContent, EditorButton, Input, Subfield } from "./ui"; // Deferred surfaces: these are not visible at first paint (export mode, @@ -150,11 +153,44 @@ function getPreviewProjectConfig( return config; } -export function Editor() { +type EditorMediaDrop = { + kind: "image" | "video"; + sourcePath: string; +}; + +export function Editor(props: { + drawingCommit?: Accessor; +}) { const currentWindow = getCurrentWindow(); let flushTitleSave: (() => Promise) | undefined; let setTitleReadOnly: ((readOnly: boolean) => void) | undefined; let activeTitleSave: { generation: number; requestId: string } | undefined; + let mediaDropHandler: ((drop: EditorMediaDrop) => void) | undefined; + const pendingMediaDrops: EditorMediaDrop[] = []; + const registerMediaDropHandler = ( + handler: ((drop: EditorMediaDrop) => void) | undefined, + ) => { + mediaDropHandler = handler; + if (handler) { + for (const drop of pendingMediaDrops.splice(0)) handler(drop); + } + }; + createTauriEventListener( + { listen: (callback) => listen("editor-image-dropped", callback) }, + (sourcePath) => { + const drop: EditorMediaDrop = { kind: "image", sourcePath }; + if (mediaDropHandler) mediaDropHandler(drop); + else pendingMediaDrops.push(drop); + }, + ); + createTauriEventListener( + { listen: (callback) => listen("editor-video-dropped", callback) }, + (sourcePath) => { + const drop: EditorMediaDrop = { kind: "video", sourcePath }; + if (mediaDropHandler) mediaDropHandler(drop); + else pendingMediaDrops.push(drop); + }, + ); const registerTitleSave = ( registration: TitleSaveRegistration | undefined, ) => { @@ -322,8 +358,10 @@ export function Editor() { flushTitleSave} registerTitleSave={registerTitleSave} + registerMediaDropHandler={registerMediaDropHandler} /> @@ -335,8 +373,12 @@ export function Editor() { function EditorContent(props: { projectPath: string; + drawingCommit?: Accessor; getTitleSave: () => (() => Promise) | undefined; registerTitleSave: (registration: TitleSaveRegistration | undefined) => void; + registerMediaDropHandler: ( + handler: ((drop: EditorMediaDrop) => void) | undefined, + ) => void; }) { const ctx = useEditorInstanceContext(); @@ -377,10 +419,14 @@ function EditorContent(props: { {(values) => ( - + )} @@ -392,6 +438,9 @@ function EditorContent(props: { function Inner(props: { getTitleSave: () => (() => Promise) | undefined; registerTitleSave: (registration: TitleSaveRegistration | undefined) => void; + registerMediaDropHandler: ( + handler: ((drop: EditorMediaDrop) => void) | undefined, + ) => void; }) { const { project, @@ -399,6 +448,7 @@ function Inner(props: { flushProjectConfig, editorInstance, editorState, + projectActions, setEditorState, previewResolutionBase, dialog, @@ -408,10 +458,39 @@ function Inner(props: { } = useEditorContext(); const preparingSession = usePreparingEditor(); + onMount(() => { + const restoreImageSelection = (event: Event) => { + const index = (event as CustomEvent<{ index: number }>).detail?.index; + if ( + Number.isInteger(index) && + index >= 0 && + project.timeline?.imageSegments[index] + ) { + setEditorState("timeline", "selection", { + type: "image", + indices: [index], + }); + } + }; + window.addEventListener("cap-image-edit-return", restoreImageSelection); + onCleanup(() => + window.removeEventListener( + "cap-image-edit-return", + restoreImageSelection, + ), + ); + }); const editorReady = () => preparingSession?.ordinaryReady() ?? canvasControls()?.hasRenderedFrame() ?? false; + const imageSelection = () => { + const selection = editorState.timeline.selection; + if (selection?.type !== "image" || selection.indices.length !== 1) + return null; + const index = selection.indices[0]; + return project.timeline?.imageSegments[index] ? index : null; + }; onMount(() => { const blockPreparingKeys = (event: KeyboardEvent) => { if (editorReady()) return; @@ -463,6 +542,52 @@ function Inner(props: { void appendRecordedClip(payload.recording_path); }); + let mediaDropQueue = Promise.resolve(); + let mediaDropAborted = false; + onCleanup(() => { + mediaDropAborted = true; + props.registerMediaDropHandler(undefined); + }); + props.registerMediaDropHandler((drop) => { + const dropTime = editorState.playbackTime; + mediaDropQueue = mediaDropQueue + .then(async () => { + while ( + (editorState.importingImage || editorState.importingVideo) && + !mediaDropAborted + ) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (mediaDropAborted) return; + const lane = + drop.kind === "video" + ? getUsedTrackCount(project.timeline?.videoSegments ?? []) + : getUsedTrackCount(project.timeline?.imageSegments ?? []); + const toastId = toast.loading(`Importing ${drop.kind}…`); + const added = + drop.kind === "video" + ? await projectActions.importVideoSegment( + lane, + dropTime, + drop.sourcePath, + ) + : await projectActions.importImageSegment( + lane, + dropTime, + undefined, + drop.sourcePath, + ); + if (added) + toast.success(`${drop.kind === "video" ? "Video" : "Image"} added`, { + id: toastId, + }); + else toast.dismiss(toastId); + }) + .catch((error) => { + toast.error(getEditorErrorMessage(error)); + }); + }); + const appendRecordedClip = async (recordingPath: string) => { const toastId = toast.loading("Adding clip…"); try { @@ -967,14 +1092,22 @@ function Inner(props: {
- +
+ +
+ + +
diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 3ef4105cf0f..f1b55f9d456 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -43,6 +43,7 @@ import { TextOverlay } from "./TextOverlay"; import { EditorButton, Slider } from "./ui"; import { useEditorShortcuts } from "./useEditorShortcuts"; import { formatTime } from "./utils"; +import { VideoOverlay } from "./video-overlay"; export function PlayerContent(props: { compactness?: number }) { const { @@ -143,6 +144,7 @@ export function PlayerContent(props: { compactness?: number }) { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], camera3dSegments: [], transitions: [], }), @@ -782,6 +784,7 @@ function PreviewCanvas(props: {
+
diff --git a/apps/desktop/src/routes/editor/Timeline/AudioTrack.tsx b/apps/desktop/src/routes/editor/Timeline/AudioTrack.tsx index c39e0418752..efdef6697c7 100644 --- a/apps/desktop/src/routes/editor/Timeline/AudioTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/AudioTrack.tsx @@ -7,6 +7,7 @@ import { type AudioTrackSegment, MIN_AUDIO_SEGMENT_DURATION } from "../audio"; import { useEditorContext } from "../context"; import { getSegmentTrack, sortTrackSegments } from "../timelineTracks"; import { useTimelineContext } from "./context"; +import { ImportedWaveformCanvas } from "./imported-waveform"; import { SegmentContent, SegmentHandle, @@ -217,6 +218,7 @@ export function AudioTrack(props: { totalDuration, projectHistory, projectActions, + importedWaveform, } = useEditorContext(); const { secsPerPixel } = useTimelineContext(); const setPreviewTime = useSetPreviewTime(); @@ -517,6 +519,15 @@ export function AudioTrack(props: { }, )} > + ( diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx index 26663d440f8..b30f2a621a4 100644 --- a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -29,6 +29,10 @@ const TRACK_META: Record = { description: "Place images and logos on your video.", unavailableHint: "", }, + video: { + description: "Add imported videos on their own editable tracks.", + unavailableHint: "", + }, clip: { description: "Your recorded screen footage.", unavailableHint: "", diff --git a/apps/desktop/src/routes/editor/Timeline/imported-waveform-data.ts b/apps/desktop/src/routes/editor/Timeline/imported-waveform-data.ts new file mode 100644 index 00000000000..25ee7934206 --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/imported-waveform-data.ts @@ -0,0 +1,53 @@ +export type ImportedWaveform = { + levels: Uint8Array[]; +}; + +export function decodeImportedWaveform(encoded: string): ImportedWaveform { + const binary = atob(encoded); + const first = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) { + first[index] = binary.charCodeAt(index); + } + const levels = [first]; + let previous = first; + while (previous.length > 1) { + const next = new Uint8Array(Math.ceil(previous.length / 2)); + for (let index = 0; index < next.length; index++) { + next[index] = Math.max( + previous[index * 2] ?? 0, + previous[index * 2 + 1] ?? 0, + ); + } + levels.push(next); + previous = next; + } + return { levels }; +} + +export function waveformRangeMax( + waveform: ImportedWaveform, + start: number, + end: number, +): number { + const length = waveform.levels[0]?.length ?? 0; + const first = Math.max(0, Math.min(length, Math.floor(start))); + const last = Math.max(first + 1, Math.min(length, Math.ceil(end))); + if (first >= length) return 0; + let maximum = 0; + let left = first; + let right = last; + for (let level = 0; left < right; level++) { + const peaks = waveform.levels[level]; + if (left % 2 !== 0) { + maximum = Math.max(maximum, peaks[left] ?? 0); + left++; + } + if (right % 2 !== 0) { + right--; + maximum = Math.max(maximum, peaks[right] ?? 0); + } + left = Math.floor(left / 2); + right = Math.floor(right / 2); + } + return maximum; +} diff --git a/apps/desktop/src/routes/editor/Timeline/imported-waveform.tsx b/apps/desktop/src/routes/editor/Timeline/imported-waveform.tsx new file mode 100644 index 00000000000..d52cfe40c86 --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/imported-waveform.tsx @@ -0,0 +1,130 @@ +import { createEffect, onCleanup } from "solid-js"; +import { useEditorContext } from "../context"; +import { useSegmentContext, useTimelineContext } from "./context"; +import type { ImportedWaveform } from "./imported-waveform-data"; +import { waveformRangeMax } from "./imported-waveform-data"; + +const MAX_CANVAS_WIDTH = 2000; +const MAX_DRAW_SAMPLES = 6000; + +export function ImportedWaveformCanvas(props: { + waveform?: ImportedWaveform; + start: number; + end: number; + sourceStart: number; + volumeDb: number; + enabled: boolean; + color: string; +}) { + const { editorState } = useEditorContext(); + const { timelineBounds } = useTimelineContext(); + const { width } = useSegmentContext(); + let canvas: HTMLCanvasElement | undefined; + let frame: number | undefined; + + function render() { + frame = undefined; + if (!canvas) return; + const context = canvas.getContext("2d"); + if (!context) return; + const duration = props.end - props.start; + const fullWidth = width(); + const waveform = props.waveform; + const peaks = waveform?.levels[0]; + if ( + !props.enabled || + props.volumeDb <= -30 || + !waveform || + !peaks?.length || + duration <= 0 || + fullWidth <= 0 + ) { + canvas.width = 1; + return; + } + const transform = editorState.timeline.transform; + const visibleStart = Math.max(props.start, transform.position); + const visibleEnd = Math.min(props.end, transform.position + transform.zoom); + if (visibleEnd <= visibleStart) { + canvas.width = 1; + return; + } + const pixelsPerSecond = fullWidth / duration; + const virtualized = fullWidth > MAX_CANVAS_WIDTH; + const rangeStart = virtualized ? visibleStart - props.start : 0; + const rangeEnd = virtualized ? visibleEnd - props.start : duration; + const renderedWidth = virtualized + ? Math.min( + (rangeEnd - rangeStart) * pixelsPerSecond, + (timelineBounds.width ?? 800) + 200, + ) + : fullWidth; + const canvasWidth = Math.max( + 1, + Math.min(MAX_CANVAS_WIDTH, Math.ceil(renderedWidth)), + ); + if (canvas.width !== canvasWidth) canvas.width = canvasWidth; + canvas.style.left = `${rangeStart * pixelsPerSecond}px`; + canvas.style.width = `${renderedWidth}px`; + context.clearRect(0, 0, canvasWidth, canvas.height); + const samples = Math.max( + 1, + Math.min( + MAX_DRAW_SAMPLES, + Math.ceil(canvasWidth * 2), + Math.ceil((rangeEnd - rangeStart) * 10), + ), + ); + const step = (rangeEnd - rangeStart) / samples; + const scale = Math.max(0, (props.volumeDb + 30) / 30); + context.beginPath(); + context.moveTo(0, canvas.height); + for (let index = 0; index <= samples; index++) { + const time = rangeStart + index * step; + const source = props.sourceStart + time; + const peak = waveformRangeMax( + waveform, + source * 10, + (source + step) * 10, + ); + context.lineTo( + (index / samples) * canvasWidth, + canvas.height * (1 - (peak / 255) * scale), + ); + } + context.lineTo(canvasWidth, canvas.height); + context.closePath(); + context.fillStyle = getComputedStyle(canvas).color; + context.globalAlpha = 0.55; + context.fill(); + context.globalAlpha = 1; + } + + createEffect(() => { + width(); + timelineBounds.width; + editorState.timeline.transform.position; + editorState.timeline.transform.zoom; + props.waveform; + props.start; + props.end; + props.sourceStart; + props.volumeDb; + props.enabled; + if (frame !== undefined) cancelAnimationFrame(frame); + frame = requestAnimationFrame(render); + }); + onCleanup(() => { + if (frame !== undefined) cancelAnimationFrame(frame); + }); + return ( + { + canvas = element; + }} + class="absolute bottom-0 h-[18px] pointer-events-none" + style={{ left: "0px", color: props.color }} + height={52} + /> + ); +} diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index d990e2af5fd..1602ee5e84e 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -25,10 +25,12 @@ import { } from "solid-js"; import { produce } from "solid-js/store"; import toast from "solid-toast"; +import IconLucideFilm from "~icons/lucide/film"; import IconLucidePalette from "~icons/lucide/palette"; import { stylesRevealCamera } from "../style"; import { ImageTrack } from "./image-track"; import { type OverlayDragState, StyleTrack } from "./style-track"; +import { VideoTrack } from "./video-track"; import "./styles.css"; @@ -96,6 +98,7 @@ const RULER_SCRUB_OVERHANG_PX = 4; const trackIcons: Record JSX.Element> = { style: () => , image: () => , + video: () => , clip: () => , caption: () => , keyboard: () => , @@ -117,6 +120,7 @@ type TrackDefinition = { const trackDefinitions: TrackDefinition[] = [ { type: "style", label: "Style", icon: trackIcons.style, locked: false }, { type: "image", label: "Image", icon: trackIcons.image, locked: false }, + { type: "video", label: "Video", icon: trackIcons.video, locked: false }, { type: "clip", label: "Clip", @@ -209,7 +213,8 @@ export function Timeline(props: { requestHandoffPlayback, } = useEditorContext(); - const duration = () => editorInstance.recordingDuration; + const duration = () => + Math.max(editorInstance.recordingDuration, totalDuration()); const transform = () => editorState.timeline.transform; const [timelineContainerRef, setTimelineContainerRef] = @@ -300,7 +305,9 @@ export function Timeline(props: { trackDefinitions.map((definition) => ({ ...definition, active: - definition.type === "style" || definition.type === "image" + definition.type === "style" || + definition.type === "image" || + definition.type === "video" ? trackState()[definition.type] > 0 : definition.type === "caption" ? trackState().caption @@ -321,11 +328,14 @@ export function Timeline(props: { supportsMultiple: definition.type === "style" || definition.type === "image" || + definition.type === "video" || definition.type === "mask" || definition.type === "text" || definition.type === "audio", count: - definition.type === "style" || definition.type === "image" + definition.type === "style" || + definition.type === "image" || + definition.type === "video" ? trackState()[definition.type] : definition.type === "mask" ? trackState().mask @@ -491,6 +501,14 @@ export function Timeline(props: { // existing lane with room at the playhead is reused, otherwise a new lane // is stacked on. Same 1s / 80px sizing as the tracks' click-to-add. function handleAddTrack(type: TimelineTrackType) { + if (type === "video") { + const lane = Math.max( + getUsedTrackCount(project.timeline?.videoSegments ?? []), + trackState().video, + ); + void projectActions.importVideoSegment(lane); + return; + } if (type === "style" || type === "image") { const segments = (type === "style" @@ -628,15 +646,17 @@ export function Timeline(props: { } function handleDeleteTrackLane( - type: "text" | "mask" | "audio" | "style" | "image", + type: "text" | "mask" | "audio" | "style" | "image" | "video", laneIndex: number, ) { - if (type === "style" || type === "image") { + if (type === "style" || type === "image" || type === "video") { const resumeHistory = projectHistory.pause(); const segments = (type === "style" ? project.timeline?.styleSegments - : project.timeline?.imageSegments) ?? []; + : type === "video" + ? project.timeline?.videoSegments + : project.timeline?.imageSegments) ?? []; projectActions.deleteOverlaySegments( type, segments.flatMap((segment, index) => @@ -648,10 +668,12 @@ export function Timeline(props: { const remaining = type === "style" ? project.timeline?.styleSegments - : project.timeline?.imageSegments; + : type === "video" + ? project.timeline?.videoSegments + : project.timeline?.imageSegments; for (const segment of remaining ?? []) if (segment.track > laneIndex) segment.track -= 1; - if (type === "image") + if (type === "image" || type === "video") project.overlayOrder = removeOverlayTrack( project.overlayOrder, type, @@ -757,6 +779,7 @@ export function Timeline(props: { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], captionSegments: [], keyboardSegments: [], camera3dSegments: [], @@ -784,6 +807,7 @@ export function Timeline(props: { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], captionSegments: [], keyboardSegments: [], camera3dSegments: [], @@ -825,7 +849,7 @@ export function Timeline(props: { async function handleOpenTrackMenu( e: MouseEvent, - type: "text" | "mask" | "audio" | "style" | "image", + type: "text" | "mask" | "audio" | "style" | "image" | "video", laneIndex: number, ) { e.preventDefault(); @@ -860,6 +884,7 @@ export function Timeline(props: { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], captionSegments: [], keyboardSegments: [], camera3dSegments: [], @@ -908,6 +933,7 @@ export function Timeline(props: { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], captionSegments: [], keyboardSegments: [], camera3dSegments: [], @@ -922,12 +948,14 @@ export function Timeline(props: { project.timeline.camera3dSegments ??= []; project.timeline.styleSegments ??= []; project.timeline.imageSegments ??= []; + project.timeline.videoSegments ??= []; }), ); } let styleSegmentDragState: OverlayDragState = { type: "idle" }; let imageSegmentDragState: OverlayDragState = { type: "idle" }; + let videoSegmentDragState: OverlayDragState = { type: "idle" }; let zoomSegmentDragState = { type: "idle" } as ZoomSegmentDragState; let sceneSegmentDragState = { type: "idle" } as SceneSegmentDragState; let maskSegmentDragState = { type: "idle" } as MaskSegmentDragState; @@ -1046,6 +1074,7 @@ export function Timeline(props: { if ( styleSegmentDragState.type !== "moving" && imageSegmentDragState.type !== "moving" && + videoSegmentDragState.type !== "moving" && zoomSegmentDragState.type !== "moving" && sceneSegmentDragState.type !== "moving" && maskSegmentDragState.type !== "moving" && @@ -1243,6 +1272,7 @@ export function Timeline(props: { const segmentCount = { style: timeline?.styleSegments?.length ?? 0, image: timeline?.imageSegments?.length ?? 0, + video: timeline?.videoSegments?.length ?? 0, clip: timeline?.segments.length ?? 0, zoom: timeline?.zoomSegments?.length ?? 0, scene: timeline?.sceneSegments?.length ?? 0, @@ -1478,12 +1508,21 @@ export function Timeline(props: { }} >
- - - + 0} + fallback={ +
+
+
+ } + > + + + + + + { + videoSegmentDragState = value; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + )} - 0 - ? () => handleClearTrackSegments("zoom") - : undefined - } - deleteLabel="Clear all" - deleteTitle="Delete all zoom segments" - > - { - zoomSegmentDragState = v; - }} - handleUpdatePlayhead={handleUpdatePlayhead} - /> - + 0}> + 0 + ? () => handleClearTrackSegments("zoom") + : undefined + } + deleteLabel="Clear all" + deleteTitle="Delete all zoom segments" + > + { + zoomSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + + (props.type === "style" ? project.timeline?.styleSegments - : project.timeline?.imageSegments) ?? []; + : props.type === "video" + ? project.timeline?.videoSegments + : project.timeline?.imageSegments) ?? []; const segments = () => allSegments() .map((segment, index) => ({ segment, index })) @@ -60,6 +64,8 @@ export function OverlayTrack( const add = (time: number) => { if (props.type === "style") projectActions.addStyleSegment(props.laneIndex, time); + else if (props.type === "video") + void projectActions.importVideoSegment(props.laneIndex, time); else void projectActions.importImageSegment(props.laneIndex, time); }; function select(index: number, event: MouseEvent) { @@ -116,11 +122,31 @@ export function OverlayTrack( const segment = allSegments()[index]; if (!segment) return; const initial = { start: segment.start, end: segment.end }; + const originalVideo = + props.type === "video" + ? project.timeline?.videoSegments[index] + : undefined; + const initialSourceStart = originalVideo?.sourceStart ?? 0; const initialPlaybackTime = editorState.playbackTime; const lane = segments(); const position = lane.findIndex((item) => item.index === index); - const previousEnd = lane[position - 1]?.segment.end ?? 0; - const nextStart = lane[position + 1]?.segment.start ?? totalDuration(); + const previousEnd = Math.max( + lane[position - 1]?.segment.end ?? 0, + props.type === "video" && edge === "start" + ? segment.start - initialSourceStart + : 0, + ); + const nextStart = + props.type === "video" + ? Math.min( + lane[position + 1]?.segment.start ?? Number.POSITIVE_INFINITY, + edge === "end" && originalVideo + ? segment.start + + originalVideo.sourceDuration - + initialSourceStart + : Number.POSITIVE_INFINITY, + ) + : (lane[position + 1]?.segment.start ?? totalDuration()); const resume = projectHistory.pause(); let moved = false; props.onDragStateChanged({ type: "movePending" }); @@ -138,6 +164,13 @@ export function OverlayTrack( ); if (props.type === "style") setProject("timeline", "styleSegments", index, interval); + else if (props.type === "video") + setProject("timeline", "videoSegments", index, { + ...interval, + sourceStart: + initialSourceStart + + (edge === "start" ? interval.start - initial.start : 0), + }); else setProject("timeline", "imageSegments", index, interval); setEditorState("previewTime", null); setEditorState( @@ -156,6 +189,11 @@ export function OverlayTrack( if (cancelled && moved && allSegments()[index] === segment) { if (props.type === "style") setProject("timeline", "styleSegments", index, initial); + else if (props.type === "video") + setProject("timeline", "videoSegments", index, { + ...initial, + sourceStart: initialSourceStart, + }); else setProject("timeline", "imageSegments", index, initial); setEditorState("playbackTime", initialPlaybackTime); } @@ -188,7 +226,10 @@ export function OverlayTrack( @@ -236,6 +285,26 @@ export function OverlayTrack( class="cursor-grab overflow-hidden" onMouseDown={(event) => drag(event, index, "move")} > + + + (
diff --git a/apps/desktop/src/routes/editor/Timeline/video-track.tsx b/apps/desktop/src/routes/editor/Timeline/video-track.tsx new file mode 100644 index 00000000000..9503c172d46 --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/video-track.tsx @@ -0,0 +1,5 @@ +import { OverlayTrack, type OverlayTrackProps } from "./style-track"; + +export function VideoTrack(props: OverlayTrackProps) { + return ; +} diff --git a/apps/desktop/src/routes/editor/TranscriptPage.tsx b/apps/desktop/src/routes/editor/TranscriptPage.tsx index 738f640781a..cd3f73347de 100644 --- a/apps/desktop/src/routes/editor/TranscriptPage.tsx +++ b/apps/desktop/src/routes/editor/TranscriptPage.tsx @@ -218,6 +218,7 @@ export function TranscriptPanel() { textSegments: [], styleSegments: [], imageSegments: [], + videoSegments: [], captionSegments: [], keyboardSegments: [], camera3dSegments: [], diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index c8b3c478184..998ec761766 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -47,6 +47,7 @@ import { events, type FrameLayoutEvent, type FramesRendered, + type ImageDrawingCommit, type ImportedAudioTrack, type MultipleSegments, type ProjectConfiguration, @@ -56,6 +57,7 @@ import { type SingleSegment, type TimelineConfiguration, type TimelineSegment, + type VideoSegment, type XY, } from "~/utils/tauri"; import { @@ -105,6 +107,10 @@ import { type StyleSegment, splitOverlaySegment, } from "./style"; +import { + decodeImportedWaveform, + type ImportedWaveform, +} from "./Timeline/imported-waveform-data"; import type { TextSegment } from "./text"; import { applyMotionTemplate, @@ -140,6 +146,7 @@ import { sortTrackSegments, } from "./timelineTracks"; import { createProgressBar } from "./utils"; +import { defaultVideoSegment, pickVideo } from "./video"; export type ModalDialog = | { type: "createPreset" } @@ -211,6 +218,7 @@ export const getPreviewResolution = ( export type TimelineTrackType = | "style" | "image" + | "video" | "clip" | "caption" | "keyboard" @@ -256,6 +264,7 @@ type EditorTimelineConfiguration = Omit< textSegments: TextSegment[]; styleSegments: StyleSegment[]; imageSegments: ImageSegment[]; + videoSegments: VideoSegment[]; audioSegments?: AudioTrackSegment[]; camera3dSegments: Camera3DSegment[]; }; @@ -312,6 +321,9 @@ export function normalizeProject( imageSegments: (config.overlayOrder?.length ? sortTrackSegments : normalizeTrackSegments)(config.timeline.imageSegments ?? []), + videoSegments: (config.overlayOrder?.length + ? sortTrackSegments + : normalizeTrackSegments)(config.timeline.videoSegments ?? []), transitions: ( config.timeline as TimelineConfiguration & { @@ -383,6 +395,7 @@ export function serializeProjectConfiguration( transitions: project.timeline.transitions ?? [], styleSegments: project.timeline.styleSegments ?? [], imageSegments: project.timeline.imageSegments ?? [], + videoSegments: project.timeline.videoSegments ?? [], captionSegments: project.timeline.captionSegments ?? [], keyboardSegments: project.timeline.keyboardSegments ?? [], maskSegments: project.timeline.maskSegments ?? [], @@ -412,11 +425,31 @@ export const [EditorContextProvider, useBaseEditorContext] = meta: () => TransformedMeta; editorInstance: SerializedEditorInstance; refetchMeta(): Promise; + imageDrawingCommit?: Accessor; }) => { const editorInstanceContext = useEditorInstanceContext(); const [project, setProject] = createStore( normalizeProject(props.editorInstance.savedProjectConfig), ); + createEffect( + on( + () => props.imageDrawingCommit?.(), + (commit) => { + if (!commit) return; + setProject( + "timeline", + "imageSegments", + commit.imageIndex, + (segment) => ({ + ...segment, + path: commit.path, + sourcePath: commit.sourcePath, + annotations: commit.annotations, + }), + ); + }, + ), + ); const setClipTransition = ( segmentIndex: number, @@ -467,6 +500,7 @@ export const [EditorContextProvider, useBaseEditorContext] = const tracks = [ timeline.styleSegments, timeline.imageSegments, + timeline.videoSegments, timeline.zoomSegments, timeline.sceneSegments ?? [], timeline.maskSegments, @@ -611,14 +645,16 @@ export const [EditorContextProvider, useBaseEditorContext] = : null; }; const selectAddedOverlay = ( - type: "style" | "image", + type: "style" | "image" | "video", lane: number, start: number, ) => { const segments = (type === "style" ? project.timeline?.styleSegments - : project.timeline?.imageSegments) ?? []; + : type === "video" + ? project.timeline?.videoSegments + : project.timeline?.imageSegments) ?? []; const index = segments.findIndex( (segment) => segment.track === lane && segment.start === start, ); @@ -1003,11 +1039,16 @@ export const [EditorContextProvider, useBaseEditorContext] = } }, splitOverlaySegment: ( - type: "style" | "image", + type: "style" | "image" | "video", index: number, time: number, ) => { - const key = type === "style" ? "styleSegments" : "imageSegments"; + const key = + type === "style" + ? "styleSegments" + : type === "video" + ? "videoSegments" + : "imageSegments"; setProject( produce((value) => { const timeline = value.timeline; @@ -1018,6 +1059,15 @@ export const [EditorContextProvider, useBaseEditorContext] = time, ); if (parts) timeline.styleSegments.splice(index, 1, ...parts); + } else if (type === "video") { + const segment = structuredClone( + unwrap(timeline.videoSegments[index]), + ); + const parts = splitOverlaySegment(segment, time); + if (parts) { + parts[1].sourceStart += time - segment.start; + timeline.videoSegments.splice(index, 1, ...parts); + } } else { const parts = splitOverlaySegment( structuredClone(unwrap(timeline.imageSegments[index])), @@ -1030,7 +1080,10 @@ export const [EditorContextProvider, useBaseEditorContext] = setEditorState("timeline", "selection", { type, indices: [index] }); if (type === "style") enterStyleScope(index); }, - deleteOverlaySegments: (type: "style" | "image", indices: number[]) => { + deleteOverlaySegments: ( + type: "style" | "image" | "video", + indices: number[], + ) => { const remove = new Set(indices); batch(() => { setProject( @@ -1041,6 +1094,11 @@ export const [EditorContextProvider, useBaseEditorContext] = value.timeline.styleSegments.filter( (_, index) => !remove.has(index), ); + else if (type === "video") + value.timeline.videoSegments = + value.timeline.videoSegments.filter( + (_, index) => !remove.has(index), + ); else value.timeline.imageSegments = value.timeline.imageSegments.filter( @@ -1075,22 +1133,23 @@ export const [EditorContextProvider, useBaseEditorContext] = lane: number, time = editorState.playbackTime, replaceIndex?: number, + sourcePath?: string, ) => { - if (editorState.importingImage) return; + if (editorState.importingImage) return false; const original = replaceIndex === undefined ? null : project.timeline?.imageSegments[replaceIndex]; setEditorState("importingImage", true); try { - const asset = await pickImage(props.editorInstance.path); - if (!asset || !project.timeline) return; + const asset = await pickImage(sourcePath); + if (!asset || !project.timeline) return false; if (replaceIndex !== undefined) { if ( !original || project.timeline.imageSegments[replaceIndex] !== original ) - return; + return false; const output = editorInstanceContext.latestFrameLayout(); const width = output?.output_width ?? 1920; const height = output?.output_height ?? 1080; @@ -1101,6 +1160,8 @@ export const [EditorContextProvider, useBaseEditorContext] = setProject("timeline", "imageSegments", replaceIndex, { path: asset.path, name: asset.name, + sourcePath: null, + annotations: [], ...(original.lockAspect ? { size: { @@ -1110,10 +1171,10 @@ export const [EditorContextProvider, useBaseEditorContext] = } : {}), }); - return; + return true; } const placement = overlayPlacement("image", lane, time); - if (!placement) return; + if (!placement) return false; const layout = editorInstanceContext.latestFrameLayout(); const output = { width: layout?.output_width ?? 1920, @@ -1136,14 +1197,66 @@ export const [EditorContextProvider, useBaseEditorContext] = }), ); selectAddedOverlay("image", placement.lane, placement.start); + return true; } catch (error) { toast.error( error instanceof Error ? error.message : "Unable to import image", ); + return false; } finally { setEditorState("importingImage", false); } }, + importVideoSegment: async ( + lane: number, + time = editorState.playbackTime, + sourcePath?: string, + ) => { + if (editorState.importingVideo) return false; + setEditorState("importingVideo", true); + try { + const asset = await pickVideo(sourcePath); + if (!asset || !project.timeline) return false; + const segments = project.timeline.videoSegments; + const start = Math.max(0, time); + const end = start + asset.duration; + let placementLane = lane; + while ( + segments.some( + (segment) => + segment.track === placementLane && + segment.start < end && + segment.end > start, + ) + ) { + placementLane += 1; + } + const layout = editorInstanceContext.latestFrameLayout(); + const output = { + width: layout?.output_width ?? 1920, + height: layout?.output_height ?? 1080, + }; + setProject( + "timeline", + "videoSegments", + produce((videoSegments) => { + videoSegments.push( + defaultVideoSegment(asset, start, placementLane, output), + ); + sortTrackSegments(videoSegments); + }), + ); + selectAddedOverlay("video", placementLane, start); + return true; + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Unable to import video", + ); + return false; + } finally { + setEditorState("importingVideo", false); + } + }, splitMaskSegment: (index: number, time: number) => { setProject( "timeline", @@ -1801,10 +1914,18 @@ export const [EditorContextProvider, useBaseEditorContext] = const totalDuration = () => project.timeline - ? clipTimelineDuration( - project.timeline.segments, - project.timeline.transitions ?? [], - ) + totalHeldDuration(holdWindows(project.timeline.textSegments)) + ? Math.max( + clipTimelineDuration( + project.timeline.segments, + project.timeline.transitions ?? [], + ) + totalHeldDuration(holdWindows(project.timeline.textSegments)), + ...project.timeline.imageSegments + .filter((segment) => segment.enabled) + .map((segment) => segment.end), + ...project.timeline.videoSegments + .filter((segment) => segment.enabled) + .map((segment) => segment.end), + ) : props.editorInstance.recordingDuration; type State = { @@ -1855,6 +1976,7 @@ export const [EditorContextProvider, useBaseEditorContext] = const [editorState, setEditorState] = createStore({ styleEditIndex: null as number | null, importingImage: false, + importingVideo: false, previewTime: null as number | null, playbackTime: preparing?.handoffTarget()?.playback.playheadSeconds ?? 0, playing: false, @@ -1880,6 +2002,7 @@ export const [EditorContextProvider, useBaseEditorContext] = | null | { type: "style"; indices: number[] } | { type: "image"; indices: number[] } + | { type: "video"; indices: number[] } | { type: "zoom"; indices: number[] } | { type: "clip"; indices: number[] } | { type: "transition"; index: number } @@ -1929,6 +2052,7 @@ export const [EditorContextProvider, useBaseEditorContext] = tracks: { style: getUsedTrackCount(project.timeline?.styleSegments ?? []), image: getUsedTrackCount(project.timeline?.imageSegments ?? []), + video: getUsedTrackCount(project.timeline?.videoSegments ?? []), clip: true, caption: initialCaptionTrackVisible, keyboard: initialKeyboardTrackVisible, @@ -2122,6 +2246,39 @@ export const [EditorContextProvider, useBaseEditorContext] = const [micWaveforms, setMicWaveforms] = createSignal(); const [systemAudioWaveforms, setSystemAudioWaveforms] = createSignal(); + const [importedWaveforms, setImportedWaveforms] = createSignal< + Map + >(new Map()); + const waveformRequested = new Set(); + let waveformAlive = true; + onCleanup(() => { + waveformAlive = false; + void commands.cancelImportedWaveforms().catch(() => undefined); + }); + createEffect(() => { + const timeline = project.timeline; + const paths = [ + ...(timeline?.audioSegments ?? []), + ...(timeline?.videoSegments ?? []), + ].map((segment) => segment.path); + for (const path of new Set(paths)) { + if (!path || waveformRequested.has(path)) continue; + waveformRequested.add(path); + commands + .getImportedWaveform(path) + .then((encoded) => { + if (!waveformAlive) return; + const waveform = decodeImportedWaveform(encoded); + setImportedWaveforms((current) => + new Map(current).set(path, waveform), + ); + }) + .catch((error) => { + if (waveformAlive) + console.error(`Failed to load waveform for ${path}:`, error); + }); + } + }); onMount(() => { commands .getMicWaveforms() @@ -2482,6 +2639,7 @@ export const [EditorContextProvider, useBaseEditorContext] = customDomain, refetchMeta: () => props.refetchMeta(), editorInstance: props.editorInstance, + importedWaveform: (path: string) => importedWaveforms().get(path), dialog, setDialog, project, @@ -2559,15 +2717,15 @@ function transformMeta({ pretty_name, ...rawMeta }: RecordingMeta) { prettyName: pretty_name, hasCamera: (() => { if (meta.type === "single") return !!meta.camera; - return !!meta.segments[0].camera; + return !!meta.segments[0]?.camera; })(), hasSystemAudio: (() => { if (meta.type === "single") return false; - return !!meta.segments[0].system_audio; + return !!meta.segments[0]?.system_audio; })(), hasMicrophone: (() => { if (meta.type === "single") return !!meta.audio; - return !!meta.segments[0].mic; + return !!meta.segments[0]?.mic; })(), hasRecordedCursorData: (() => { if (meta.type === "single") return !!meta.cursor; diff --git a/apps/desktop/src/routes/editor/image-editor-sidebar.tsx b/apps/desktop/src/routes/editor/image-editor-sidebar.tsx new file mode 100644 index 00000000000..6529ea88ffd --- /dev/null +++ b/apps/desktop/src/routes/editor/image-editor-sidebar.tsx @@ -0,0 +1,27 @@ +import IconLucideArrowLeft from "~icons/lucide/arrow-left"; +import { useEditorContext } from "./context"; +import { ImageSegmentConfig } from "./image-segment-config"; + +export function ImageEditorSidebar(props: { index: number }) { + const { setEditorState } = useEditorContext(); + return ( + + ); +} diff --git a/apps/desktop/src/routes/editor/image-segment-config.tsx b/apps/desktop/src/routes/editor/image-segment-config.tsx index 98b98b9bfd0..527b9084df0 100644 --- a/apps/desktop/src/routes/editor/image-segment-config.tsx +++ b/apps/desktop/src/routes/editor/image-segment-config.tsx @@ -1,17 +1,148 @@ import { convertFileSrc } from "@tauri-apps/api/core"; -import { createEffect, createSignal, Show } from "solid-js"; +import { createEffect, createSignal, For, type JSX, Show } from "solid-js"; +import toast from "solid-toast"; import { Toggle } from "~/components/Toggle"; +import { commands } from "~/utils/tauri"; +import IconCapCorners from "~icons/cap/corners"; +import IconCapCrop from "~icons/cap/crop"; +import IconCapImage from "~icons/cap/image"; +import IconCapLayout from "~icons/cap/layout"; +import IconCapPadding from "~icons/cap/padding"; +import IconCapShadow from "~icons/cap/shadow"; +import IconCapSquare from "~icons/cap/square"; +import IconLucideArrowUpRight from "~icons/lucide/arrow-up-right"; +import IconLucideChevronDown from "~icons/lucide/chevron-down"; +import IconLucideCircle from "~icons/lucide/circle"; import IconLucideCrosshair from "~icons/lucide/crosshair"; +import IconLucideEyeOff from "~icons/lucide/eye-off"; +import IconLucideMousePointer2 from "~icons/lucide/mouse-pointer-2"; +import IconLucidePencil from "~icons/lucide/pencil"; +import IconLucideSquare from "~icons/lucide/square"; +import IconLucideType from "~icons/lucide/type"; +import type { ScreenshotSidebarAction } from "../screenshot-editor/screenshot-sidebar"; import { useEditorContext } from "./context"; import { imageAssetPath } from "./images"; import { EditorButton, Field, SectionLabel, Slider } from "./ui"; -export function ImageSegmentConfig(props: { index: number }) { - const { project, setProject, editorInstance, projectActions, editorState } = - useEditorContext(); +export function ImageSegmentConfig(props: { + index: number; + dedicated?: boolean; +}) { + const { + project, + setProject, + editorInstance, + projectActions, + editorState, + setEditorState, + flushProjectConfig, + requestHandoffPlayback, + } = useEditorContext(); const segment = () => project.timeline?.imageSegments[props.index]; const path = () => imageAssetPath(editorInstance.path, segment()?.path ?? ""); const [failed, setFailed] = createSignal(false); + const [layoutOpen, setLayoutOpen] = createSignal(false); + const openScreenshot = async (action: ScreenshotSidebarAction) => { + try { + const pending = requestHandoffPlayback(false); + if (pending && !(await pending)) return; + if (editorState.playing) { + await commands.stopPlayback(); + setEditorState("playing", false); + } + await flushProjectConfig(); + setEditorState("timeline", "selection", null); + window.dispatchEvent( + new CustomEvent("cap-edit-image", { + detail: { index: props.index, action }, + }), + ); + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + } + }; + const tools: Array<{ + label: string; + icon: JSX.Element; + action: ScreenshotSidebarAction; + }> = [ + { + label: "Select", + icon: , + action: { type: "tool", tool: "select" }, + }, + { + label: "Draw", + icon: , + action: { type: "tool", tool: "draw" }, + }, + { + label: "Arrow", + icon: , + action: { type: "tool", tool: "arrow" }, + }, + { + label: "Rectangle", + icon: , + action: { type: "tool", tool: "rectangle" }, + }, + { + label: "Mask", + icon: , + action: { type: "tool", tool: "mask" }, + }, + { + label: "Circle", + icon: , + action: { type: "tool", tool: "circle" }, + }, + { + label: "Text", + icon: , + action: { type: "tool", tool: "text" }, + }, + ]; + const appearance: Array<{ + label: string; + icon: JSX.Element; + action: ScreenshotSidebarAction; + }> = [ + { + label: "Aspect", + icon: , + action: { type: "appearance", panel: "aspect" }, + }, + { + label: "Crop", + icon: , + action: { type: "appearance", panel: "crop" }, + }, + { + label: "Background", + icon: , + action: { type: "appearance", panel: "background" }, + }, + { + label: "Padding", + icon: , + action: { type: "appearance", panel: "padding" }, + }, + { + label: "Corners", + icon: , + action: { type: "appearance", panel: "rounding" }, + }, + { + label: "Shadow", + icon: , + action: { type: "appearance", panel: "shadow" }, + }, + { + label: "Border", + icon: , + action: { type: "appearance", panel: "border" }, + }, + ]; createEffect(() => { path(); setFailed(false); @@ -19,218 +150,307 @@ export function ImageSegmentConfig(props: { index: number }) { return ( {(image) => ( -
-
- - setProject( - "timeline", - "imageSegments", - props.index, - "name", - event.currentTarget.value.trim() || "Image", - ) - } - /> - - setProject( - "timeline", - "imageSegments", - props.index, - "enabled", - value, - ) - } - /> -
- - Image unavailable. Replace it to restore this layer. -

- } - > - {image().name} setFailed(true)} - /> - -
- - void projectActions.importImageSegment( - image().track, - image().start, - props.index, - ) - } - > - {editorState.importingImage ? "Importing…" : "Replace image"} - - - projectActions.deleteOverlaySegments("image", [props.index]) +
+
+ + Image unavailable. Replace it to restore this layer. +

} > - Delete - + {image().name} setFailed(true)} + /> +
-
- -

- Drag the image to move it. Pull a corner to resize, or use the - rotation handle to turn it. Arrow keys nudge it into place. -

+
+
+

+ {image().name} +

+

Image track

+
} + class="shrink-0" + leftIcon={} onClick={() => - setProject("timeline", "imageSegments", props.index, "center", { - x: 0.5, - y: 0.5, - }) + void openScreenshot({ type: "tool", tool: "select" }) } > - Center on canvas + Open canvas
-
- - - setProject( - "timeline", - "imageSegments", - props.index, - "opacity", - value[0] / 100, - ) - } - /> - - - `${value}°`} - onChange={(value) => - setProject( - "timeline", - "imageSegments", - props.index, - "rotation", - value[0], - ) - } - /> - - - - setProject( - "timeline", - "imageSegments", - props.index, - "rounding", - value[0], - ) - } - /> - - - - setProject( - "timeline", - "imageSegments", - props.index, - "lockAspect", - value, - ) - } - /> - - - - setProject( - "timeline", - "imageSegments", - props.index, - "flipX", - value, - ) - } - /> - - - - setProject( - "timeline", - "imageSegments", - props.index, - "flipY", - value, - ) - } - /> - -
- - setProject("timeline", "imageSegments", props.index, { - center: { x: 0.5, y: 0.5 }, - rotation: 0, - flipX: false, - flipY: false, - }) - } +
+ +
+ + {(option) => ( + void openScreenshot(option.action)} + /> + )} + +
+
+
+ +
+ + {(option) => ( + void openScreenshot(option.action)} + /> + )} + +
+
+ + +
+
+ + setProject( + "timeline", + "imageSegments", + props.index, + "name", + event.currentTarget.value.trim() || "Image", + ) + } + /> + + setProject( + "timeline", + "imageSegments", + props.index, + "enabled", + value, + ) + } + /> +
+
+ + void projectActions.importImageSegment( + image().track, + image().start, + props.index, + ) + } + > + {editorState.importingImage ? "Importing…" : "Replace image"} + + + projectActions.deleteOverlaySegments("image", [props.index]) + } + > + Delete + +
+
+

+ Drag the image to move it. Pull a corner to resize, or use the + rotation handle to turn it. Arrow keys nudge it into place. +

+ } + onClick={() => + setProject( + "timeline", + "imageSegments", + props.index, + "center", + { + x: 0.5, + y: 0.5, + }, + ) + } + > + Center on canvas + +
+
+ + + setProject( + "timeline", + "imageSegments", + props.index, + "opacity", + value[0] / 100, + ) + } + /> + + + `${value}°`} + onChange={(value) => + setProject( + "timeline", + "imageSegments", + props.index, + "rotation", + value[0], + ) + } + /> + + + + setProject( + "timeline", + "imageSegments", + props.index, + "rounding", + value[0], + ) + } + /> + + + + setProject( + "timeline", + "imageSegments", + props.index, + "lockAspect", + value, + ) + } + /> + + + + setProject( + "timeline", + "imageSegments", + props.index, + "flipX", + value, + ) + } + /> + + + + setProject( + "timeline", + "imageSegments", + props.index, + "flipY", + value, + ) + } + /> + +
+ + setProject("timeline", "imageSegments", props.index, { + center: { x: 0.5, y: 0.5 }, + rotation: 0, + flipX: false, + flipY: false, + }) + } + > + Center and reset rotation + +

+ Drag the image to move it. Drag a corner to resize. Shift + temporarily disables snapping. +

+
+
)} ); } + +function ScreenshotActionButton(props: { + label: string; + icon: JSX.Element; + onClick: () => void; +}) { + return ( + + ); +} diff --git a/apps/desktop/src/routes/editor/images.ts b/apps/desktop/src/routes/editor/images.ts index 1757311d8f6..1b55b2ad275 100644 --- a/apps/desktop/src/routes/editor/images.ts +++ b/apps/desktop/src/routes/editor/images.ts @@ -1,4 +1,4 @@ -import type { ImageSegment, XY } from "~/utils/tauri"; +import { commands, type ImageSegment, type XY } from "~/utils/tauri"; export type { ImageSegment } from "~/utils/tauri"; export type ImageAsset = { @@ -43,6 +43,7 @@ export function defaultImageSegment( track, enabled: true, path: asset.path, + annotations: [], name: asset.name, center: { x: 0.5, y: 0.5 }, size: fitImageSize(asset.width, asset.height, output.width, output.height), @@ -56,7 +57,10 @@ export function defaultImageSegment( } export function imageAssetPath(projectPath: string, relative: string) { - if (!/^content\/images\/[^/\\]+$/.test(relative) || relative.includes("..")) + if ( + !/^(?:original\.png|content\/images\/[^/\\]+)$/.test(relative) || + relative.includes("..") + ) return null; return `${projectPath.replace(/[\\/]$/, "")}/${relative}`; } @@ -291,56 +295,22 @@ export async function importImagePath( } export async function pickImage( - projectPath: string, + sourcePath?: string, ): Promise { - const [{ open }, fs] = await Promise.all([ - import("@tauri-apps/plugin-dialog"), - import("@tauri-apps/plugin-fs"), - ]); - const source = await open({ - multiple: false, - directory: false, - filters: [ - { - name: "Images", - extensions: ["png", "jpg", "jpeg", "webp", "gif", "bmp"], - }, - ], - }); + const source = + sourcePath ?? + (await import("@tauri-apps/plugin-dialog").then(({ open }) => + open({ + multiple: false, + directory: false, + filters: [ + { + name: "Images", + extensions: ["png", "jpg", "jpeg", "webp", "gif", "bmp"], + }, + ], + }), + )); if (typeof source !== "string") return null; - return importImagePath(projectPath, source, { - read: async (path) => readBoundedImage(await fs.open(path, { read: true })), - mkdir: (path) => fs.mkdir(path, { recursive: true }), - write: async (path, bytes) => { - const file = await fs.open(path, { write: true, createNew: true }); - try { - let offset = 0; - while (offset < bytes.length) { - const written = await file.write( - bytes.subarray( - offset, - Math.min(bytes.length, offset + 1024 * 1024), - ), - ); - if (!written) - throw new Error("Unable to save the image in this project."); - offset += written; - } - } finally { - await file.close(); - } - }, - id: () => crypto.randomUUID(), - decode: async (bytes) => { - const bitmap = await createImageBitmap( - new Blob([new Uint8Array(bytes)]), - { imageOrientation: "from-image" }, - ); - try { - return { width: bitmap.width, height: bitmap.height }; - } finally { - bitmap.close(); - } - }, - }); + return commands.importEditorImage(source); } diff --git a/apps/desktop/src/routes/editor/index.tsx b/apps/desktop/src/routes/editor/index.tsx index 2d7ea67e4f5..6cab95f288c 100644 --- a/apps/desktop/src/routes/editor/index.tsx +++ b/apps/desktop/src/routes/editor/index.tsx @@ -1,16 +1,46 @@ import { Effect, getCurrentWindow } from "@tauri-apps/api/window"; import { type as ostype } from "@tauri-apps/plugin-os"; import { cx } from "cva"; -import { createEffect, onMount, Suspense } from "solid-js"; +import { + createEffect, + createSignal, + lazy, + onCleanup, + onMount, + Show, + Suspense, +} from "solid-js"; import { generalSettingsStore } from "~/store"; -import { commands } from "~/utils/tauri"; +import { commands, type ImageDrawingCommit } from "~/utils/tauri"; +import type { ScreenshotSidebarAction } from "../screenshot-editor/screenshot-sidebar"; import { Editor } from "./Editor"; import { EditorSkeleton } from "./editor-skeleton"; import { PreparingEditorProvider } from "./preparing-editor-context"; +const ScreenshotWorkspace = lazy(() => import("./screenshot-workspace")); + export default function () { const generalSettings = generalSettingsStore.createQuery(); - + const [drawingIndex, setDrawingIndex] = createSignal(null); + const [drawingAction, setDrawingAction] = + createSignal(); + const [drawingCommit, setDrawingCommit] = createSignal(); + onMount(() => { + const openDrawing = (event: Event) => { + const detail = ( + event as CustomEvent<{ + index: number; + action?: ScreenshotSidebarAction; + }> + ).detail; + if (Number.isInteger(detail?.index) && detail.index >= 0) { + setDrawingAction(detail.action); + setDrawingIndex(detail.index); + } + }; + window.addEventListener("cap-edit-image", openDrawing); + onCleanup(() => window.removeEventListener("cap-edit-image", openDrawing)); + }); // The window is normally revealed by Rust as soon as it's built and // positioned (the native background color is themed, so there's no flash). // This reveal path only matters when window transparency is enabled — Rust @@ -46,17 +76,38 @@ export default function () { return (
- - }> - - - + }> + + + + +
+ { + const index = drawingIndex(); + setDrawingIndex(null); + setDrawingAction(undefined); + if (index !== null) { + window.dispatchEvent( + new CustomEvent("cap-image-edit-return", { + detail: { index }, + }), + ); + } + }} + onCommit={setDrawingCommit} + /> +
+ +
); } diff --git a/apps/desktop/src/routes/editor/screenshot-workspace.tsx b/apps/desktop/src/routes/editor/screenshot-workspace.tsx new file mode 100644 index 00000000000..af148a3da68 --- /dev/null +++ b/apps/desktop/src/routes/editor/screenshot-workspace.tsx @@ -0,0 +1,113 @@ +import { remove, writeFile } from "@tauri-apps/plugin-fs"; +import { createSignal } from "solid-js"; +import toast from "solid-toast"; +import { commands, type ImageDrawingCommit } from "~/utils/tauri"; +import { + ScreenshotEditorProvider, + useScreenshotEditorContext, +} from "../screenshot-editor/context"; +import { Editor as ScreenshotDrawingEditor } from "../screenshot-editor/Editor"; +import type { ScreenshotSidebarAction } from "../screenshot-editor/screenshot-sidebar"; +import { canvasToBlob } from "../screenshot-editor/screenshotExport"; +import { useScreenshotExport } from "../screenshot-editor/useScreenshotExport"; + +function ImageDrawingActions(props: { + imageIndex: number; + onExit: () => Promise; + onCommit?: (commit: ImageDrawingCommit) => void; + saving: () => boolean; + setSaving: (value: boolean) => void; +}) { + const { editorInstance } = useScreenshotEditorContext(); + const { renderExportCanvas } = useScreenshotExport(); + const apply = async () => { + if (props.saving() || !editorInstance()) return; + props.setSaving(true); + let tempPath: string | undefined; + try { + const canvas = await renderExportCanvas(); + const blob = await canvasToBlob(canvas, "image/png"); + const bytes = new Uint8Array(await blob.arrayBuffer()); + tempPath = await commands.imageDrawingTempPath(); + await writeFile(tempPath, bytes); + const commit = await commands.commitImageDrawing( + props.imageIndex, + tempPath, + ); + props.onCommit?.(commit); + if (await props.onExit()) toast.success("Drawing added to image track"); + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + } finally { + if (tempPath) void remove(tempPath).catch(() => {}); + props.setSaving(false); + } + }; + return ( +
+ + +
+ ); +} + +export default function ScreenshotWorkspace(props: { + imageDrawingIndex?: number; + onExit?: () => void; + onCommit?: (commit: ImageDrawingCommit) => void; + initialAction?: ScreenshotSidebarAction; +}) { + const [saving, setSaving] = createSignal(false); + const exit = async () => { + try { + await commands.closeImageDrawingInstance(); + props.onExit?.(); + return true; + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + return false; + } + }; + return ( + +
+ void exit() + : undefined + } + backDisabled={saving()} + sidebarFooter={ + props.imageDrawingIndex !== undefined && props.onExit ? ( + + ) : undefined + } + /> +
+ + ); +} diff --git a/apps/desktop/src/routes/editor/timelineTracks.ts b/apps/desktop/src/routes/editor/timelineTracks.ts index b98eb401ad2..e57f0c5533e 100644 --- a/apps/desktop/src/routes/editor/timelineTracks.ts +++ b/apps/desktop/src/routes/editor/timelineTracks.ts @@ -125,12 +125,15 @@ type OverlayProject = { timeline?: { textSegments?: TrackSegment[]; imageSegments?: TrackSegment[]; + videoSegments?: TrackSegment[]; maskSegments?: TrackSegment[]; } | null; }; export function isOverlayTrackKind(kind: string): kind is OverlayTrackKind { - return kind === "text" || kind === "image" || kind === "mask"; + return ( + kind === "text" || kind === "image" || kind === "video" || kind === "mask" + ); } export function sameOverlayTrack(a: OverlayTrack, b: OverlayTrack) { @@ -165,7 +168,7 @@ export function getOverlayTrackRows( ) { const timeline = project.timeline; const available: OverlayTrack[] = []; - for (const kind of ["text", "image", "mask"] as const) { + for (const kind of ["text", "image", "video", "mask"] as const) { const segments = timeline?.[`${kind}Segments`] ?? []; for (const track of getTrackRowsWithCount( segments, diff --git a/apps/desktop/src/routes/editor/video-overlay.tsx b/apps/desktop/src/routes/editor/video-overlay.tsx new file mode 100644 index 00000000000..6dcccd17ca6 --- /dev/null +++ b/apps/desktop/src/routes/editor/video-overlay.tsx @@ -0,0 +1,290 @@ +import { createEventListener } from "@solid-primitives/event-listener"; +import { For, onCleanup, Show } from "solid-js"; +import { unwrap } from "solid-js/store"; +import { useCanvasSnapTargets } from "./CanvasElementsOverlay"; +import { useEditorContext } from "./context"; +import { resizeImage } from "./images"; +import { createOverlaySegments } from "./overlay-segments"; +import { SNAP_PX, snapMovingRect } from "./snapping"; +import { getOverlayZIndex } from "./timelineTracks"; + +export function VideoOverlay(props: { + size: { width: number; height: number }; +}) { + const { + project, + setProject, + editorState, + setEditorState, + projectHistory, + setSnapGuides, + } = useEditorContext(); + const snapTargets = useCanvasSnapTargets(); + const time = () => editorState.previewTime ?? editorState.playbackTime; + const { visible } = createOverlaySegments( + () => project.timeline?.videoSegments ?? [], + time, + ); + const selected = (index: number) => + editorState.timeline.selection?.type === "video" && + editorState.timeline.selection.indices.includes(index); + let endDrag: (() => void) | undefined; + onCleanup(() => endDrag?.()); + + function drag( + event: MouseEvent, + index: number, + mode: "move" | "rotate" | { x: number; y: number }, + ) { + if (event.button !== 0 || editorState.playing) return; + event.preventDefault(); + event.stopPropagation(); + endDrag?.(); + const source = project.timeline?.videoSegments[index]; + if (!source) return; + setEditorState("timeline", "selection", { + type: "video", + indices: [index], + }); + const initial = structuredClone(unwrap(source)); + const canvas = { ...props.size }; + const rect = (event.currentTarget as HTMLElement) + .closest("[data-video-overlay]") + ?.getBoundingClientRect(); + const center = rect + ? { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 } + : { x: event.clientX, y: event.clientY }; + const startAngle = Math.atan2( + event.clientY - center.y, + event.clientX - center.x, + ); + const targets = snapTargets({ video: index }); + const resume = projectHistory.pause(); + let moved = false; + let frame: number | undefined; + let pending: MouseEvent | undefined; + const move = (next: MouseEvent) => { + if (project.timeline?.videoSegments[index] !== source) return; + const delta = { + x: next.clientX - event.clientX, + y: next.clientY - event.clientY, + }; + if (!moved && Math.hypot(delta.x, delta.y) < 2) return; + moved = true; + if (mode === "rotate") { + let rotation = + initial.rotation + + ((Math.atan2(next.clientY - center.y, next.clientX - center.x) - + startAngle) * + 180) / + Math.PI; + if (next.shiftKey) rotation = Math.round(rotation / 15) * 15; + setProject( + "timeline", + "videoSegments", + index, + "rotation", + ((((rotation + 180) % 360) + 360) % 360) - 180, + ); + } else if (mode === "move") { + const radians = (initial.rotation * Math.PI) / 180; + const width = initial.size.x * canvas.width; + const height = initial.size.y * canvas.height; + const w = + (Math.abs(width * Math.cos(radians)) + + Math.abs(height * Math.sin(radians))) / + canvas.width; + const h = + (Math.abs(width * Math.sin(radians)) + + Math.abs(height * Math.cos(radians))) / + canvas.height; + const raw = { + x: initial.center.x + delta.x / canvas.width - w / 2, + y: initial.center.y + delta.y / canvas.height - h / 2, + w, + h, + }; + const snap = next.shiftKey + ? { dx: 0, dy: 0, guides: [] } + : snapMovingRect( + raw, + targets, + SNAP_PX / canvas.width, + SNAP_PX / canvas.height, + ); + setSnapGuides(snap.guides); + setProject("timeline", "videoSegments", index, "center", { + x: Math.max(0, Math.min(1, raw.x + w / 2 + snap.dx)), + y: Math.max(0, Math.min(1, raw.y + h / 2 + snap.dy)), + }); + } else + setProject( + "timeline", + "videoSegments", + index, + resizeImage(initial, delta, mode, canvas), + ); + }; + const scheduleMove = (next: MouseEvent) => { + pending = next; + if (frame !== undefined) return; + frame = requestAnimationFrame(() => { + frame = undefined; + const latest = pending; + pending = undefined; + if (latest) move(latest); + }); + }; + const finish = (next?: MouseEvent, cancelled = false) => { + if (!endDrag) return; + if (frame !== undefined) cancelAnimationFrame(frame); + frame = undefined; + pending = undefined; + if (next) move(next); + window.removeEventListener("mousemove", scheduleMove); + window.removeEventListener("mouseup", finish); + window.removeEventListener("blur", cancel); + window.removeEventListener("keydown", keydown, true); + endDrag = undefined; + if ( + cancelled && + moved && + project.timeline?.videoSegments[index] === source + ) + setProject("timeline", "videoSegments", index, { + center: initial.center, + size: initial.size, + rotation: initial.rotation, + }); + setSnapGuides([]); + resume(); + }; + const cancel = () => finish(undefined, true); + const keydown = (next: KeyboardEvent) => { + if (next.key !== "Escape") return; + next.preventDefault(); + next.stopImmediatePropagation(); + cancel(); + }; + endDrag = cancel; + window.addEventListener("mousemove", scheduleMove); + window.addEventListener("mouseup", finish); + window.addEventListener("blur", cancel); + window.addEventListener("keydown", keydown, true); + } + + createEventListener(window, "keydown", (event) => { + if ( + editorState.playing || + (event.target instanceof HTMLElement && + (event.target.isContentEditable || + ["INPUT", "TEXTAREA", "SELECT"].includes(event.target.tagName))) + ) + return; + if ( + event.key === "Escape" && + editorState.timeline.selection?.type === "video" + ) { + event.preventDefault(); + setEditorState("timeline", "selection", null); + return; + } + const direction = { + ArrowLeft: [-1, 0], + ArrowRight: [1, 0], + ArrowUp: [0, -1], + ArrowDown: [0, 1], + }[event.key]; + if (!direction) return; + const items = visible().filter(({ index }) => selected(index)); + if (!items.length) return; + event.preventDefault(); + event.stopPropagation(); + const resume = projectHistory.pause(); + for (const { segment, index } of items) { + const step = event.shiftKey ? 10 : 1; + setProject("timeline", "videoSegments", index, "center", { + x: Math.max( + 0, + Math.min( + 1, + segment.center.x + (direction[0] * step) / props.size.width, + ), + ), + y: Math.max( + 0, + Math.min( + 1, + segment.center.y + (direction[1] * step) / props.size.height, + ), + ), + }); + } + resume(); + }); + + return ( +
+ + + {({ segment, index }) => ( +
drag(event, index, "move")} + > +
+ + + {segment.name} + +
+ )} + + +
+ ); +} diff --git a/apps/desktop/src/routes/editor/video-segment-config.tsx b/apps/desktop/src/routes/editor/video-segment-config.tsx new file mode 100644 index 00000000000..acb6aef5569 --- /dev/null +++ b/apps/desktop/src/routes/editor/video-segment-config.tsx @@ -0,0 +1,224 @@ +import { Show } from "solid-js"; +import { Toggle } from "~/components/Toggle"; +import IconLucideCrosshair from "~icons/lucide/crosshair"; +import { useEditorContext } from "./context"; +import { EditorButton, Field, SectionLabel, Slider } from "./ui"; + +export function VideoSegmentConfig(props: { index: number }) { + const { project, setProject, projectActions } = useEditorContext(); + const segment = () => project.timeline?.videoSegments[props.index]; + return ( + + {(video) => ( +
+
+ + setProject( + "timeline", + "videoSegments", + props.index, + "name", + event.currentTarget.value.trim() || "Video", + ) + } + /> + + setProject( + "timeline", + "videoSegments", + props.index, + "enabled", + value, + ) + } + /> +
+
+
+ {video().name} +
+
+ {(video().end - video().start).toFixed(2)}s on timeline · source + starts at {video().sourceStart.toFixed(2)}s +
+
+
+ +

+ Drag the video to move it. Pull a corner to resize, or use the + rotation handle to turn it. Arrow keys nudge it into place. +

+ } + onClick={() => + setProject("timeline", "videoSegments", props.index, "center", { + x: 0.5, + y: 0.5, + }) + } + > + Center on canvas + +
+
+ + + setProject( + "timeline", + "videoSegments", + props.index, + "opacity", + value[0] / 100, + ) + } + /> + + + `${value}°`} + onChange={(value) => + setProject( + "timeline", + "videoSegments", + props.index, + "rotation", + value[0], + ) + } + /> + + + + setProject( + "timeline", + "videoSegments", + props.index, + "rounding", + value[0], + ) + } + /> + + + + setProject( + "timeline", + "videoSegments", + props.index, + "lockAspect", + value, + ) + } + /> + + + + setProject( + "timeline", + "videoSegments", + props.index, + "flipX", + value, + ) + } + /> + + + + setProject( + "timeline", + "videoSegments", + props.index, + "flipY", + value, + ) + } + /> + + + + setProject( + "timeline", + "videoSegments", + props.index, + "muted", + value, + ) + } + /> + + + `${value} dB`} + onChange={(value) => + setProject( + "timeline", + "videoSegments", + props.index, + "volumeDb", + value[0], + ) + } + /> + +
+ + projectActions.deleteOverlaySegments("video", [props.index]) + } + > + Delete video + +
+ )} + + ); +} diff --git a/apps/desktop/src/routes/editor/video.ts b/apps/desktop/src/routes/editor/video.ts new file mode 100644 index 00000000000..dd0e7390619 --- /dev/null +++ b/apps/desktop/src/routes/editor/video.ts @@ -0,0 +1,72 @@ +import { + commands, + type ImportedEditorVideo, + type VideoSegment, +} from "~/utils/tauri"; + +export type { VideoSegment } from "~/utils/tauri"; + +export async function pickVideo( + sourcePath?: string, +): Promise { + const source = + sourcePath ?? + (await import("@tauri-apps/plugin-dialog").then(({ open }) => + open({ + multiple: false, + directory: false, + filters: [ + { + name: "Videos", + extensions: [ + "mp4", + "mov", + "avi", + "mkv", + "webm", + "wmv", + "m4v", + "flv", + ], + }, + ], + }), + )); + if (typeof source !== "string") return null; + return commands.importEditorVideo(source); +} + +export function defaultVideoSegment( + asset: ImportedEditorVideo, + start: number, + track: number, + output: { width: number; height: number }, +): VideoSegment { + const scale = Math.min( + output.width / Math.max(1, asset.width), + output.height / Math.max(1, asset.height), + ); + return { + start, + end: start + asset.duration, + track, + enabled: true, + path: asset.path, + name: asset.name, + sourceStart: 0, + sourceDuration: asset.duration, + muted: !asset.hasAudio, + volumeDb: 0, + center: { x: 0.5, y: 0.5 }, + size: { + x: (asset.width * scale) / output.width, + y: (asset.height * scale) / output.height, + }, + opacity: 1, + rotation: 0, + rounding: 0, + flipX: false, + flipY: false, + lockAspect: true, + }; +} diff --git a/apps/desktop/src/routes/screenshot-editor/AnnotationConfig.tsx b/apps/desktop/src/routes/screenshot-editor/AnnotationConfig.tsx index cfa13212ccc..0bb4cab9599 100644 --- a/apps/desktop/src/routes/screenshot-editor/AnnotationConfig.tsx +++ b/apps/desktop/src/routes/screenshot-editor/AnnotationConfig.tsx @@ -6,7 +6,7 @@ import { BACKGROUND_COLORS, hexToRgb, RgbInput, rgbToHex } from "./ColorPicker"; import { type Annotation, useScreenshotEditorContext } from "./context"; import { Slider } from "./ui"; -export function AnnotationConfigBar() { +export function AnnotationConfigBar(props: { sidebar?: boolean }) { const { annotations, selectedAnnotationId, @@ -38,11 +38,20 @@ export function AnnotationConfigBar() { return (
-
+
-
+
-
+
+ -
); } @@ -80,6 +103,7 @@ function ToolButton(props: { icon: Component<{ class?: string }>; label: string; shortcut?: string; + sidebar?: boolean; }) { const { activeTool, setActiveTool, setSelectedAnnotationId } = useScreenshotEditorContext(); @@ -97,13 +121,20 @@ function ToolButton(props: { } }} class={cx( - "flex items-center justify-center rounded-lg transition-all size-8", + props.sidebar + ? "flex h-[68px] w-full flex-col items-center justify-center gap-2 rounded-xl border px-1 text-[11px] font-medium transition-colors" + : "flex size-8 items-center justify-center rounded-lg transition-colors", activeTool() === props.tool - ? "bg-blue-3 text-blue-11" - : "bg-transparent hover:bg-gray-3 text-gray-11", + ? props.sidebar + ? "border-ed-accent bg-ed-ctl-hover text-ed-text-1" + : "bg-blue-3 text-blue-11" + : props.sidebar + ? "border-transparent bg-ed-ctl text-ed-text-2 hover:border-ed-line hover:bg-ed-ctl-hover" + : "bg-transparent hover:bg-gray-3 text-gray-11", )} > - + + {props.sidebar && {props.label}} ); diff --git a/apps/desktop/src/routes/screenshot-editor/Editor.tsx b/apps/desktop/src/routes/screenshot-editor/Editor.tsx index 86bffee703b..d6fc93eb64d 100644 --- a/apps/desktop/src/routes/screenshot-editor/Editor.tsx +++ b/apps/desktop/src/routes/screenshot-editor/Editor.tsx @@ -4,9 +4,11 @@ import { makePersisted } from "@solid-primitives/storage"; import { convertFileSrc } from "@tauri-apps/api/core"; import { LogicalPosition } from "@tauri-apps/api/dpi"; import { Menu } from "@tauri-apps/api/menu"; +import { type as ostype } from "@tauri-apps/plugin-os"; import { createEffect, createSignal, + type JSX, Match, onCleanup, onMount, @@ -23,6 +25,8 @@ import { createCropOptionsMenuItems, type Ratio, } from "~/components/Cropper"; +import CaptionControlsMacOS from "~/components/titlebar/controls/CaptionControlsMacOS"; +import CaptionControlsWindows11 from "~/components/titlebar/controls/CaptionControlsWindows11"; import { composeEventHandlers } from "~/utils/composeEventHandlers"; import IconCapCircleX from "~icons/cap/circle-x"; import IconLucideMaximize from "~icons/lucide/maximize"; @@ -33,9 +37,20 @@ import { Header } from "./Header"; import { LayersPanel } from "./LayersPanel"; import { Preview } from "./Preview"; import { ScreenshotEditorSkeleton } from "./screenshot-editor-skeleton"; +import { + ScreenshotSidebar, + type ScreenshotSidebarAction, +} from "./screenshot-sidebar"; import { Dialog, EditorButton } from "./ui"; -export function Editor() { +export function Editor(props: { + imageDrawingMode?: boolean; + sidebarLayout?: boolean; + sidebarFooter?: JSX.Element; + initialAction?: ScreenshotSidebarAction; + onBack?: () => void; + backDisabled?: boolean; +}) { const [zoom, setZoom] = createSignal(1); const { projectHistory, @@ -49,6 +64,7 @@ export function Editor() { activePopover, setActivePopover, isRenderReady, + prettyName, } = useScreenshotEditorContext(); const [copiedAnnotation, setCopiedAnnotation] = createSignal(null); @@ -176,22 +192,84 @@ export function Editor() { }); return ( - }> -
-
- -
-
- - - -
-
- + +
+ {prettyName} +
+
+
+ Preparing image… +
+
+
+
+ ) : ( + + ) + } + > + +
+
+ +
+
+ + + +
+
+ +
+
+ +
+ + } + > +
+
+
+ +
+ + + + + {prettyName} + .cap +
+ + Image editing + + + + +
+
+
+ + +
+
- -
+ ); } diff --git a/apps/desktop/src/routes/screenshot-editor/Header.tsx b/apps/desktop/src/routes/screenshot-editor/Header.tsx index a41171e16a5..fabb78077c6 100644 --- a/apps/desktop/src/routes/screenshot-editor/Header.tsx +++ b/apps/desktop/src/routes/screenshot-editor/Header.tsx @@ -5,7 +5,7 @@ import { remove } from "@tauri-apps/plugin-fs"; import { revealItemInDir } from "@tauri-apps/plugin-opener"; import { type as ostype } from "@tauri-apps/plugin-os"; import { cx } from "cva"; -import { createEffect, onCleanup, Suspense } from "solid-js"; +import { createEffect, onCleanup, Show, Suspense } from "solid-js"; import CaptionControlsMacOS from "~/components/titlebar/controls/CaptionControlsMacOS"; import CaptionControlsWindows11 from "~/components/titlebar/controls/CaptionControlsWindows11"; import IconCapCrop from "~icons/cap/crop"; @@ -33,7 +33,7 @@ import { } from "./ui"; import { useScreenshotExport } from "./useScreenshotExport"; -export function Header() { +export function Header(props: { imageDrawingMode?: boolean }) { const ctx = useScreenshotEditorContext(); const { setDialog, @@ -47,6 +47,7 @@ export function Header() { const { exportImage, exportStatus, isExporting } = useScreenshotExport(); createEffect(() => { + if (props.imageDrawingMode) return; const handleKeyDown = (e: KeyboardEvent) => { if (e.defaultPrevented) return; const target = e.target as HTMLElement | null; @@ -117,21 +118,25 @@ export function Header() {
- - } - /> -
+ + + } + /> +
+ -
- - - - - + +
+ + + + + +
-
- - { - exportImage("clipboard"); - }} - tooltipText="Copy to Clipboard" - disabled={isExporting()} - leftIcon={} - /> - - exportImage("file")} - disabled={isExporting()} - leftIcon={} - /> - - exportImage("share")} - disabled={isExporting()} - leftIcon={} - /> - - - - as={DropdownMenu.Trigger} - tooltipText="More Actions" - leftIcon={} + +
+ + { + exportImage("clipboard"); + }} + tooltipText="Copy to Clipboard" disabled={isExporting()} + leftIcon={} /> - - - - as={DropdownMenu.Content} - class={cx("min-w-[200px]", topSlideAnimateClasses)} - > - - as={DropdownMenu.Group} - class="p-1" + + exportImage("file")} + disabled={isExporting()} + leftIcon={} + /> + + exportImage("share")} + disabled={isExporting()} + leftIcon={} + /> + + + + as={DropdownMenu.Trigger} + tooltipText="More Actions" + leftIcon={} + disabled={isExporting()} + /> + + + + as={DropdownMenu.Content} + class={cx("min-w-[200px]", topSlideAnimateClasses)} > - { - revealItemInDir(path()); - }} - > - - Open Folder - - { - if ( - await ask( - "Are you sure you want to delete this screenshot?", - ) - ) { - await remove(path()); - await getCurrentWindow().close(); - } - }} + + as={DropdownMenu.Group} + class="p-1" > - - Delete - - - - - - + { + revealItemInDir(path()); + }} + > + + Open Folder + + { + if ( + await ask( + "Are you sure you want to delete this screenshot?", + ) + ) { + await remove(path()); + await getCurrentWindow().close(); + } + }} + > + + Delete + + + + + + + {ostype() === "windows" && }
diff --git a/apps/desktop/src/routes/screenshot-editor/LayersPanel.tsx b/apps/desktop/src/routes/screenshot-editor/LayersPanel.tsx index 0e186c093b4..d377498b23a 100644 --- a/apps/desktop/src/routes/screenshot-editor/LayersPanel.tsx +++ b/apps/desktop/src/routes/screenshot-editor/LayersPanel.tsx @@ -29,7 +29,7 @@ const ANNOTATION_TYPE_LABELS = { draw: "Draw", }; -export function LayersPanel() { +export function LayersPanel(props: { sidebar?: boolean }) { const { annotations, setAnnotations, @@ -200,7 +200,14 @@ export function LayersPanel() { ); return ( -
+
diff --git a/apps/desktop/src/routes/screenshot-editor/Preview.tsx b/apps/desktop/src/routes/screenshot-editor/Preview.tsx index 5bfdccb6fec..fda0cecc0c2 100644 --- a/apps/desktop/src/routes/screenshot-editor/Preview.tsx +++ b/apps/desktop/src/routes/screenshot-editor/Preview.tsx @@ -30,7 +30,11 @@ const gridStyle = { "background-position": "0 0, 0 10px, 10px -10px, -10px 0px", }; -export function Preview(props: { zoom: number; setZoom: (z: number) => void }) { +export function Preview(props: { + zoom: number; + setZoom: (z: number) => void; + editorLayout?: boolean; +}) { const { latestFrame, annotations, @@ -465,7 +469,13 @@ export function Preview(props: { zoom: number; setZoom: (z: number) => void }) { }); return ( -
+
{/* Preview Area */}
void }) { style={gridStyle} onMouseDown={handleMiddleMouseDown} > -
+
(DEFAULT_PROJECT); const [annotations, setAnnotations] = createStore([]); const [selectedAnnotationId, setSelectedAnnotationId] = createSignal< @@ -221,7 +221,10 @@ function createScreenshotEditorContext() { const [editorInstance] = createResource(async () => { const perfStart = performance.now(); const sincePerfStart = () => Math.round(performance.now() - perfStart); - const instance = await commands.createScreenshotEditorInstance(); + const instance = + props.imageDrawingIndex === undefined + ? await commands.createScreenshotEditorInstance() + : await commands.createImageDrawingInstance(props.imageDrawingIndex); console.info( `[screenshot-editor] createScreenshotEditorInstance resolved in ${sincePerfStart()}ms`, ); diff --git a/apps/desktop/src/routes/screenshot-editor/index.tsx b/apps/desktop/src/routes/screenshot-editor/index.tsx deleted file mode 100644 index f1d993c6033..00000000000 --- a/apps/desktop/src/routes/screenshot-editor/index.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { Effect, getCurrentWindow } from "@tauri-apps/api/window"; -import { type as ostype } from "@tauri-apps/plugin-os"; -import { cx } from "cva"; -import { createEffect } from "solid-js"; -import { generalSettingsStore } from "~/store"; -import { commands } from "~/utils/tauri"; -import { ScreenshotEditorProvider } from "./context"; -import { Editor } from "./Editor"; - -export default function ScreenshotEditorRoute() { - const generalSettings = generalSettingsStore.createQuery(); - - createEffect(() => { - const transparent = generalSettings.data?.windowTransparency ?? false; - commands.setWindowTransparent(transparent); - getCurrentWindow().setEffects({ - effects: transparent ? [Effect.HudWindow] : [], - }); - }); - - return ( -
- - - -
- ); -} diff --git a/apps/desktop/src/routes/screenshot-editor/popovers/AspectRatioSelect.tsx b/apps/desktop/src/routes/screenshot-editor/popovers/AspectRatioSelect.tsx index d4d298fb6b8..13b581a7461 100644 --- a/apps/desktop/src/routes/screenshot-editor/popovers/AspectRatioSelect.tsx +++ b/apps/desktop/src/routes/screenshot-editor/popovers/AspectRatioSelect.tsx @@ -14,9 +14,9 @@ import { topLeftAnimateClasses, } from "../ui"; -export function AspectRatioSelect() { +export function AspectRatioSelect(props: { initialOpen?: boolean }) { const { project, setProject } = useScreenshotEditorContext(); - const [open, setOpen] = createSignal(false); + const [open, setOpen] = createSignal(props.initialOpen ?? false); let triggerSelect: HTMLDivElement | undefined; return ( diff --git a/apps/desktop/src/routes/screenshot-editor/screenshot-sidebar.tsx b/apps/desktop/src/routes/screenshot-editor/screenshot-sidebar.tsx new file mode 100644 index 00000000000..622fecbb945 --- /dev/null +++ b/apps/desktop/src/routes/screenshot-editor/screenshot-sidebar.tsx @@ -0,0 +1,285 @@ +import { DropdownMenu } from "@kobalte/core/dropdown-menu"; +import { getCurrentWindow } from "@tauri-apps/api/window"; +import { ask } from "@tauri-apps/plugin-dialog"; +import { remove } from "@tauri-apps/plugin-fs"; +import { revealItemInDir } from "@tauri-apps/plugin-opener"; +import { createEffect, type JSX, onCleanup, Show } from "solid-js"; +import IconCapCrop from "~icons/cap/crop"; +import IconCapTrash from "~icons/cap/trash"; +import IconLucideArrowLeft from "~icons/lucide/arrow-left"; +import IconLucideCopy from "~icons/lucide/copy"; +import IconLucideFolder from "~icons/lucide/folder"; +import IconLucideLink from "~icons/lucide/link"; +import IconLucideMoreHorizontal from "~icons/lucide/more-horizontal"; +import IconLucideSave from "~icons/lucide/save"; +import { AnnotationConfigBar } from "./AnnotationConfig"; +import { AnnotationTools } from "./AnnotationTools"; +import { + type ScreenshotEditorTool, + useScreenshotEditorContext, +} from "./context"; +import { LayersPanel } from "./LayersPanel"; +import { AspectRatioSelect } from "./popovers/AspectRatioSelect"; +import { BackgroundSettingsPopover } from "./popovers/BackgroundSettingsPopover"; +import { BorderPopover } from "./popovers/BorderPopover"; +import { PaddingPopover } from "./popovers/PaddingPopover"; +import { RoundingPopover } from "./popovers/RoundingPopover"; +import { ShadowPopover } from "./popovers/ShadowPopover"; +import { useScreenshotExport } from "./useScreenshotExport"; + +export type ScreenshotSidebarAction = + | { type: "tool"; tool: ScreenshotEditorTool } + | { + type: "appearance"; + panel: + | "aspect" + | "crop" + | "background" + | "padding" + | "rounding" + | "shadow" + | "border"; + }; + +export function ScreenshotSidebar(props: { + initialAction?: ScreenshotSidebarAction; + footer?: JSX.Element; + onBack?: () => void; + backDisabled?: boolean; +}) { + const ctx = useScreenshotEditorContext(); + const { + originalImageSize, + isImageFileReady, + isRenderReady, + layersPanelOpen, + setDialog, + setActiveTool, + setSelectedAnnotationId, + setActivePopover, + } = ctx; + let initialActionApplied = false; + const openCrop = () => { + const size = originalImageSize(); + if (!size || !isImageFileReady()) return; + setDialog({ + open: true, + type: "crop", + originalSize: { x: size.width, y: size.height }, + currentCrop: ctx.project.background.crop, + }); + }; + createEffect(() => { + if (!isRenderReady() || initialActionApplied || !props.initialAction) + return; + const action = props.initialAction; + if ( + action.type === "appearance" && + action.panel === "crop" && + !isImageFileReady() + ) + return; + initialActionApplied = true; + if (action.type === "tool") { + setActiveTool(action.tool); + if (action.tool !== "select") setSelectedAnnotationId(null); + } else if (action.panel === "crop") { + openCrop(); + } else if (action.panel !== "aspect") { + setActivePopover(action.panel); + } + }); + + return ( + + ); +} + +function AppearanceOption(props: { label: string; children: JSX.Element }) { + return ( +
+ {props.children} + + {props.label} + +
+ ); +} + +function ScreenshotExportActions() { + const { exportImage, isExporting } = useScreenshotExport(); + const { editorInstance, selectedAnnotationId } = useScreenshotEditorContext(); + const path = () => editorInstance()?.path ?? ""; + createEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (event.defaultPrevented || !(event.metaKey || event.ctrlKey)) return; + const target = event.target as HTMLElement | null; + if ( + target?.tagName === "INPUT" || + target?.tagName === "TEXTAREA" || + target?.isContentEditable + ) + return; + const key = event.key.toLowerCase(); + if (key === "c" && !selectedAnnotationId()) { + const selection = window.getSelection(); + if (selection && !selection.isCollapsed && selection.toString()) return; + event.preventDefault(); + if (!isExporting()) exportImage("clipboard"); + } else if (key === "s") { + event.preventDefault(); + if (!isExporting()) exportImage("file"); + } + }; + window.addEventListener("keydown", handleKeyDown); + onCleanup(() => window.removeEventListener("keydown", handleKeyDown)); + }); + return ( +
+ + + + + + + More + + + + void revealItemInDir(path())} + > + + Open folder + + + void (async () => { + if ( + await ask( + "Are you sure you want to delete this screenshot?", + ) + ) { + await remove(path()); + await getCurrentWindow().close(); + } + })() + } + > + + Delete screenshot + + + + +
+ ); +} diff --git a/apps/desktop/src/routes/screenshot-editor/ui.tsx b/apps/desktop/src/routes/screenshot-editor/ui.tsx index 5493775e57f..986963dd2ed 100644 --- a/apps/desktop/src/routes/screenshot-editor/ui.tsx +++ b/apps/desktop/src/routes/screenshot-editor/ui.tsx @@ -418,7 +418,7 @@ export function EditorButton( } export const dropdownContainerClasses = - "z-10 flex flex-col rounded-xl border border-gray-3 bg-gray-1 shadow-s overflow-y-hidden outline-hidden"; + "z-50 flex flex-col rounded-xl border border-gray-3 bg-gray-1 shadow-s overflow-y-hidden outline-hidden"; export const topLeftAnimateClasses = "data-expanded:animate-in data-expanded:fade-in data-expanded:zoom-in-95 data-closed:animate-out data-closed:fade-out data-closed:zoom-out-95 origin-top-left"; diff --git a/apps/desktop/src/routes/screenshot-editor/useScreenshotExport.ts b/apps/desktop/src/routes/screenshot-editor/useScreenshotExport.ts index ddaba791124..c377f6a2673 100644 --- a/apps/desktop/src/routes/screenshot-editor/useScreenshotExport.ts +++ b/apps/desktop/src/routes/screenshot-editor/useScreenshotExport.ts @@ -258,5 +258,5 @@ export function useScreenshotExport() { } }; - return { exportImage, exportStatus, isExporting }; + return { exportImage, exportStatus, isExporting, renderExportCanvas }; } diff --git a/apps/desktop/src/styles/theme.css b/apps/desktop/src/styles/theme.css index 166b2ebf933..b2172bf1127 100644 --- a/apps/desktop/src/styles/theme.css +++ b/apps/desktop/src/styles/theme.css @@ -30,6 +30,7 @@ --track-keyboard: #f97316; --track-style: #ec4899; --track-image: #f59e0b; + --track-video: #a855f7; --track-text: #14b8a6; --track-mask: #ef4444; --track-scene: #8b5cf6; diff --git a/apps/desktop/src/utils/importMedia.ts b/apps/desktop/src/utils/importMedia.ts index 16127675c84..3de51a0d6bb 100644 --- a/apps/desktop/src/utils/importMedia.ts +++ b/apps/desktop/src/utils/importMedia.ts @@ -1,4 +1,3 @@ -import { invoke } from "@tauri-apps/api/core"; import * as dialog from "@tauri-apps/plugin-dialog"; import { hideCurrentWindow } from "~/utils/hide-window"; import { commands } from "~/utils/tauri"; @@ -39,7 +38,7 @@ export const importVideoPath = async ( sourcePath: string, options?: ImportOptions, ) => { - const projectPath = await commands.startVideoImport(sourcePath); + const projectPath = await commands.createMediaProjectFromVideo(sourcePath); await commands.showWindow({ Editor: { project_path: projectPath } }); await maybeHideCurrentWindow(options); return projectPath; @@ -49,8 +48,8 @@ export const importImagePath = async ( sourcePath: string, options?: ImportOptions, ) => { - const imagePath = await invoke("start_image_import", { sourcePath }); - await commands.showWindow({ ScreenshotEditor: { path: imagePath } }); + const imagePath = await commands.createMediaProjectFromImage(sourcePath); + await commands.showWindow({ Editor: { project_path: imagePath } }); await maybeHideCurrentWindow(options); return imagePath; }; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 6f27d9856cb..7696ad6b061 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -190,9 +190,21 @@ async generateExportPreviewFast(frameTime: number, settings: ExportPreviewSettin async startVideoImport(sourcePath: string) : Promise { return await TAURI_INVOKE("start_video_import", { sourcePath }); }, +async createMediaProjectFromVideo(sourcePath: string) : Promise { + return await TAURI_INVOKE("create_media_project_from_video", { sourcePath }); +}, +async createMediaProjectFromImage(sourcePath: string) : Promise { + return await TAURI_INVOKE("create_media_project_from_image", { sourcePath }); +}, async addExistingRecordingToEditor(sourcePath: string) : Promise { return await TAURI_INVOKE("add_existing_recording_to_editor", { sourcePath }); }, +async importEditorImage(sourcePath: string) : Promise { + return await TAURI_INVOKE("import_editor_image", { sourcePath }); +}, +async importEditorVideo(sourcePath: string) : Promise { + return await TAURI_INVOKE("import_editor_video", { sourcePath }); +}, async startImageImport(sourcePath: string) : Promise { return await TAURI_INVOKE("start_image_import", { sourcePath }); }, @@ -247,6 +259,12 @@ async getMicWaveforms() : Promise { async getSystemAudioWaveforms() : Promise { return await TAURI_INVOKE("get_system_audio_waveforms"); }, +async getImportedWaveform(path: string) : Promise { + return await TAURI_INVOKE("get_imported_waveform", { path }); +}, +async cancelImportedWaveforms() : Promise { + await TAURI_INVOKE("cancel_imported_waveforms"); +}, async listAudioLibrary() : Promise { return await TAURI_INVOKE("list_audio_library"); }, @@ -322,6 +340,18 @@ async uploadRenderedScreenshot(imageBytes: number[], contentType: string, projec async createScreenshotEditorInstance() : Promise { return await TAURI_INVOKE("create_screenshot_editor_instance"); }, +async createImageDrawingInstance(imageIndex: number) : Promise { + return await TAURI_INVOKE("create_image_drawing_instance", { imageIndex }); +}, +async closeImageDrawingInstance() : Promise { + return await TAURI_INVOKE("close_image_drawing_instance"); +}, +async commitImageDrawing(imageIndex: number, pngPath: string) : Promise { + return await TAURI_INVOKE("commit_image_drawing", { imageIndex, pngPath }); +}, +async imageDrawingTempPath() : Promise { + return await TAURI_INVOKE("image_drawing_temp_path"); +}, async updateScreenshotConfig(config: ProjectConfiguration, save: boolean, revision: number) : Promise { return await TAURI_INVOKE("update_screenshot_config", { config, save, revision }); }, @@ -1123,13 +1153,16 @@ export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; export type HotkeyAction = "startStudioRecording" | "startInstantRecording" | "stopRecording" | "restartRecording" | "togglePauseRecording" | "cycleRecordingMode" | "openRecordingPicker" | "openRecordingPickerDisplay" | "openRecordingPickerWindow" | "openRecordingPickerArea" | "screenshotDisplay" | "screenshotWindow" | "screenshotArea" | "other" export type HotkeysConfiguration = { show: boolean } export type HotkeysStore = { hotkeys: { [key in HotkeyAction]: Hotkey } } -export type ImageSegment = { start: number; end: number; track: number; enabled: boolean; path: string; name: string; center: XY; size: XY; opacity: number; rotation: number; rounding: number; flipX: boolean; flipY: boolean; lockAspect: boolean } +export type ImageDrawingCommit = { imageIndex: number; path: string; sourcePath: string; annotations: Annotation[] } +export type ImageSegment = { start: number; end: number; track: number; enabled: boolean; path: string; sourcePath?: string | null; annotations: Annotation[]; name: string; center: XY; size: XY; opacity: number; rotation: number; rounding: number; flipX: boolean; flipY: boolean; lockAspect: boolean } export type ImportStage = "Probing" | "Converting" | "Finalizing" | "Complete" | "Failed" export type ImportedAudioTrack = { /** * Path relative to the project directory, e.g. `assets/audio/`. */ path: string; name: string; duration: number } +export type ImportedEditorImage = { path: string; name: string; width: number; height: number } +export type ImportedEditorVideo = { path: string; name: string; duration: number; fps: number; width: number; height: number; hasAudio: boolean } export type IncompleteRecordingInfo = { projectPath: string; prettyName: string; segmentCount: number; estimatedDurationSecs: number } export type InstantRecordingMeta = { recording: boolean } | { error: string } | { fps: number; sample_rate: number | null } export type JsonValue = [T] @@ -1179,7 +1212,7 @@ export type OnEscapePress = null export type Organization = { id: string; name: string; ownerId: string; role?: string; canEditBrand?: boolean; iconUrl?: string | null; brandColors?: OrganizationBrandColors } export type OrganizationBrandColors = { primary: string | null; secondary: string | null; accent: string | null; background: string | null } export type OverlayTrack = { kind: OverlayTrackKind; track: number } -export type OverlayTrackKind = "mask" | "image" | "text" +export type OverlayTrackKind = "mask" | "image" | "video" | "text" export type Phase = "awaitingShortcut" | "starting" | "recording" | "pausing" | "paused" | "resuming" | "resumeFailed" | "restarting" | "stopping" | "restoring" export type PhysicalSize = { width: number; height: number } export type Plan = { upgraded: boolean; manual: boolean; last_checked: number } @@ -1350,7 +1383,7 @@ letterSpacing?: number; lineHeight?: number; opacity?: number; shadow?: number; * segment edges when `layout` is not `Overlay`. */ layoutTransition?: number } -export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; styleSegments: StyleSegment[]; imageSegments: ImageSegment[]; camera3dSegments?: Camera3DSegment[] } +export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; styleSegments: StyleSegment[]; imageSegments: ImageSegment[]; videoSegments: VideoSegment[]; camera3dSegments?: Camera3DSegment[] } export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number; name?: string | null; speedAudioMode?: ClipSpeedAudioMode | null; volume?: number | null; hideCursor?: boolean | null } export type TranscriptionEngine = "Whisper" | "Parakeet" export type Trigger = "screenshotTaken" | "studioRecordingFinished" | "instantRecordingFinished" | "recordingStarted" | "uploadCompleted" | "videoImported" | "recordingDeleted" @@ -1367,6 +1400,7 @@ export type Video = { duration: number; width: number; height: number; fps: numb export type VideoImportProgress = { project_path: string; stage: ImportStage; progress: number; message: string } export type VideoMeta = { path: string; fps?: number; start_time?: number | null; device_id?: string | null } export type VideoRecordingMetadata = { duration: number; size: number } +export type VideoSegment = { start: number; end: number; track: number; enabled: boolean; path: string; name: string; sourceStart: number; sourceDuration: number; muted: boolean; volumeDb: number; center: XY; size: XY; opacity: number; rotation: number; rounding: number; flipX: boolean; flipY: boolean; lockAspect: boolean } export type VideoUploadInfo = { id: string; link: string; config: S3UploadMeta } export type VoiceIsolation = "light" | "balanced" | "strong" export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: string | null; windowTitle?: string | null } diff --git a/crates/audio/src/imported_waveform.rs b/crates/audio/src/imported_waveform.rs new file mode 100644 index 00000000000..c5207bc6186 --- /dev/null +++ b/crates/audio/src/imported_waveform.rs @@ -0,0 +1,231 @@ +use crate::{AudioStream, ChunkRead}; +use std::{ + collections::hash_map::DefaultHasher, + fs::{self, File}, + hash::{Hash, Hasher}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::{Arc, OnceLock, atomic::AtomicBool}, + time::UNIX_EPOCH, +}; + +const MAGIC: &[u8; 8] = b"CAPWAVE2"; +const SAMPLE_RATE: usize = crate::AudioData::SAMPLE_RATE as usize; +const SAMPLES_PER_PEAK: usize = SAMPLE_RATE / 10; +const MAX_CACHED_PEAKS: usize = 10_000_000; + +pub fn imported_waveform_slots() -> &'static tokio::sync::Semaphore { + static SLOTS: OnceLock = OnceLock::new(); + SLOTS.get_or_init(|| tokio::sync::Semaphore::new(2)) +} + +fn source_stamp(path: &Path) -> Result<(u64, u64, u32), String> { + let metadata = + fs::metadata(path).map_err(|error| format!("Cannot inspect audio source: {error}"))?; + if !metadata.is_file() { + return Err("Audio source is not a file".into()); + } + let modified = metadata + .modified() + .map_err(|error| format!("Cannot inspect audio source time: {error}"))? + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("Invalid audio source time: {error}"))?; + Ok((metadata.len(), modified.as_secs(), modified.subsec_nanos())) +} + +fn cache_path(project_root: &Path, source: &Path) -> PathBuf { + let mut hasher = DefaultHasher::new(); + source.hash(&mut hasher); + project_root + .join("cache/waveforms") + .join(format!("{:016x}.cawf", hasher.finish())) +} + +fn read_cache(path: &Path, stamp: (u64, u64, u32)) -> Option> { + let mut file = File::open(path).ok()?; + let mut header = [0u8; 36]; + file.read_exact(&mut header).ok()?; + if &header[..8] != MAGIC + || u64::from_le_bytes(header[8..16].try_into().ok()?) != stamp.0 + || u64::from_le_bytes(header[16..24].try_into().ok()?) != stamp.1 + || u32::from_le_bytes(header[24..28].try_into().ok()?) != stamp.2 + { + return None; + } + let count = u64::from_le_bytes(header[28..36].try_into().ok()?) as usize; + if count > MAX_CACHED_PEAKS { + return None; + } + let mut peaks = vec![0u8; count]; + file.read_exact(&mut peaks).ok()?; + let mut trailing = [0u8; 1]; + if file.read(&mut trailing).ok()? != 0 { + return None; + } + Some(peaks.into()) +} + +fn write_cache(path: &Path, stamp: (u64, u64, u32), peaks: &[u8]) -> Result<(), String> { + let parent = path.parent().ok_or("Invalid waveform cache path")?; + fs::create_dir_all(parent).map_err(|error| format!("Cannot create waveform cache: {error}"))?; + static NEXT_TEMP: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let sequence = NEXT_TEMP.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let temporary = path.with_extension(format!("{}.{}.tmp", std::process::id(), sequence)); + let result = (|| { + let mut file = File::create(&temporary) + .map_err(|error| format!("Cannot write waveform cache: {error}"))?; + file.write_all(MAGIC) + .and_then(|_| file.write_all(&stamp.0.to_le_bytes())) + .and_then(|_| file.write_all(&stamp.1.to_le_bytes())) + .and_then(|_| file.write_all(&stamp.2.to_le_bytes())) + .and_then(|_| file.write_all(&(peaks.len() as u64).to_le_bytes())) + .and_then(|_| file.write_all(peaks)) + .and_then(|_| file.sync_all()) + .map_err(|error| format!("Cannot save waveform cache: {error}"))?; + let finished = match fs::rename(&temporary, path) { + Ok(()) => Ok(()), + Err(_) if read_cache(path, stamp).is_some() => Ok(()), + Err(_) if cfg!(windows) && path.exists() => { + fs::remove_file(path).and_then(|_| fs::rename(&temporary, path)) + } + Err(error) => Err(error), + }; + finished.map_err(|error| format!("Cannot finish waveform cache: {error}")) + })(); + if temporary.exists() { + let _ = fs::remove_file(&temporary); + } + result +} + +fn quantize(mean: f32) -> u8 { + if mean <= 0.0 || !mean.is_finite() { + return 0; + } + let db = (20.0 * mean.log10()).clamp(-60.0, 0.0); + ((db + 60.0) * (255.0 / 60.0)).round() as u8 +} + +fn decode_peaks(source: &Path, cancellation: Arc) -> Result, String> { + let mut stream = match AudioStream::open_waveform(source, cancellation) { + Ok(stream) => stream, + Err(error) if error.stage == "stream" && error.detail == "No Stream" => { + return Ok(Vec::new()); + } + Err(error) => return Err(format!("Cannot decode imported audio: {error}")), + }; + let mut peaks = Vec::new(); + let mut sum = 0.0_f32; + let mut samples = 0usize; + while let ChunkRead::Chunk(chunk) = stream + .read_chunk(SAMPLE_RATE) + .map_err(|error| format!("Cannot read imported audio: {error}"))? + { + for sample in chunk.samples { + sum += sample.abs(); + samples += 1; + if samples == SAMPLES_PER_PEAK { + peaks.push(quantize(sum / samples as f32)); + sum = 0.0; + samples = 0; + } + } + if peaks.len() > MAX_CACHED_PEAKS { + return Err("Imported audio is too long for a timeline waveform".into()); + } + } + if samples > 0 { + peaks.push(quantize(sum / samples as f32)); + } + Ok(peaks) +} + +pub fn imported_waveform( + project_root: &Path, + relative_path: &str, + cancellation: Arc, +) -> Result, String> { + let project_root = fs::canonicalize(project_root) + .map_err(|error| format!("Cannot open editor project: {error}"))?; + let source = fs::canonicalize(project_root.join(relative_path)) + .map_err(|error| format!("Cannot open imported media: {error}"))?; + if !source.starts_with(&project_root) { + return Err("Imported media is outside the editor project".into()); + } + let stamp = source_stamp(&source)?; + let cache = cache_path(&project_root, &source); + if let Some(peaks) = read_cache(&cache, stamp) { + return Ok(peaks); + } + let peaks = decode_peaks(&source, cancellation)?; + if source_stamp(&source)? != stamp { + return Err("Imported media changed while its waveform was generated".into()); + } + write_cache(&cache, stamp, &peaks)?; + Ok(peaks.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn wav(path: &Path, samples: &[i16]) { + let bytes = (samples.len() * 2) as u32; + let mut out = Vec::new(); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&(36 + bytes).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&48_000u32.to_le_bytes()); + out.extend_from_slice(&96_000u32.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&bytes.to_le_bytes()); + for sample in samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + fs::write(path, out).unwrap(); + } + + #[test] + fn waveform_cache_is_compact_and_invalidates_when_media_changes() { + let project = tempfile::tempdir().unwrap(); + let source = project.path().join("music.wav"); + let mut samples = vec![0i16; 48_000]; + samples.extend(vec![20_000i16; 48_000]); + wav(&source, &samples); + let cancelled = Arc::new(AtomicBool::new(false)); + let first = imported_waveform(project.path(), "music.wav", cancelled.clone()).unwrap(); + assert_eq!(first.len(), 20); + assert!(first[..10].iter().all(|value| *value == 0)); + assert!(first[10..].iter().all(|value| *value > 0)); + assert_eq!( + imported_waveform(project.path(), "music.wav", cancelled.clone()) + .unwrap() + .as_ref(), + first.as_ref() + ); + wav(&source, &vec![0i16; 96_000]); + let changed = imported_waveform(project.path(), "music.wav", cancelled).unwrap(); + assert!(changed.iter().all(|value| *value == 0)); + + let audible: Vec = (0..48_000) + .map(|index| { + (20_000.0 * (2.0 * std::f64::consts::PI * 8_000.0 * index as f64 / 48_000.0).sin()) + as i16 + }) + .collect(); + wav(&source, &audible); + let high_frequency = imported_waveform( + project.path(), + "music.wav", + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + assert_eq!(high_frequency.len(), 10); + assert!(high_frequency.iter().all(|value| *value > 100)); + } +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index ef0e71c45e3..ac2ae3a4781 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -1,5 +1,6 @@ mod audio_data; mod calibration_store; +mod imported_waveform; mod latency; mod progressive; mod renderer; @@ -11,6 +12,7 @@ mod voice_level; pub use audio_data::*; pub use calibration_store::*; +pub use imported_waveform::*; pub use latency::*; pub use progressive::*; pub use renderer::*; diff --git a/crates/audio/src/streaming.rs b/crates/audio/src/streaming.rs index 830243b9716..62121828196 100644 --- a/crates/audio/src/streaming.rs +++ b/crates/audio/src/streaming.rs @@ -135,6 +135,23 @@ impl AudioStream { user: cancellation, abort: None, }, + crate::AudioData::SAMPLE_RATE, + false, + ) + } + + pub fn open_waveform( + path: &Path, + cancellation: Arc, + ) -> Result { + Self::open_controlled( + path, + StreamCancellation { + user: cancellation, + abort: None, + }, + crate::AudioData::SAMPLE_RATE, + true, ) } @@ -149,16 +166,23 @@ impl AudioStream { user, abort: Some(abort), }, + crate::AudioData::SAMPLE_RATE, + false, ) } fn open_controlled( path: &Path, cancellation: StreamCancellation, + sample_rate: u32, + mono: bool, ) -> Result { - Self::open_from(cancellation, |cancellation| { - open_input(path, cancellation).map(StreamInput::File) - }) + Self::open_from( + cancellation, + |cancellation| open_input(path, cancellation).map(StreamInput::File), + sample_rate, + mono, + ) } pub fn open_relocatable<'a>( @@ -190,21 +214,28 @@ impl AudioStream { paths: impl IntoIterator, cancellation: StreamCancellation, ) -> Result { - Self::open_from(cancellation, |cancellation| { - let cancellation = cancellation.clone(); - cap_enc_ffmpeg::SegmentedInput::open_relocatable_interruptible( - source, - paths, - Arc::new(move || cancellation.is_cancelled()), - ) - .map(StreamInput::Relocatable) - .map_err(|error| error.to_string()) - }) + Self::open_from( + cancellation, + |cancellation| { + let cancellation = cancellation.clone(); + cap_enc_ffmpeg::SegmentedInput::open_relocatable_interruptible( + source, + paths, + Arc::new(move || cancellation.is_cancelled()), + ) + .map(StreamInput::Relocatable) + .map_err(|error| error.to_string()) + }, + crate::AudioData::SAMPLE_RATE, + false, + ) } fn open_from( cancellation: StreamCancellation, open: impl FnOnce(&Arc) -> Result, + sample_rate: u32, + mono: bool, ) -> Result { let cancellation = Arc::new(cancellation); let at_open = |stage, detail: String| { @@ -238,9 +269,9 @@ impl AudioStream { decoder.set_channel_layout(ChannelLayout::default(source_channels as i32)); } decoder.set_packet_time_base(stream.time_base()); - let channels = if source_channels <= 1 { 1 } else { 2 }; + let channels = if mono || source_channels <= 1 { 1 } else { 2 }; let mut options = ffmpeg::Dictionary::new(); - options.set("filter_size", "128"); + options.set("filter_size", if mono { "16" } else { "128" }); options.set("cutoff", "0.97"); let resampler = resampling::Context::get_with( decoder.format(), @@ -248,7 +279,7 @@ impl AudioStream { decoder.rate(), crate::AudioData::SAMPLE_FORMAT, ChannelLayout::default(channels as i32), - crate::AudioData::SAMPLE_RATE, + sample_rate, options, ) .map_err(|e| at_open("resampler-open", e.to_string()))?; diff --git a/crates/editor/examples/editor-playback-benchmark.rs b/crates/editor/examples/editor-playback-benchmark.rs index 14305d6c61a..1c3d8611296 100644 --- a/crates/editor/examples/editor-playback-benchmark.rs +++ b/crates/editor/examples/editor-playback-benchmark.rs @@ -389,6 +389,7 @@ async fn load_recording( scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/editor/examples/playback-pipeline-benchmark.rs b/crates/editor/examples/playback-pipeline-benchmark.rs index 8e772d10372..47fbad7024e 100644 --- a/crates/editor/examples/playback-pipeline-benchmark.rs +++ b/crates/editor/examples/playback-pipeline-benchmark.rs @@ -315,6 +315,7 @@ async fn load_recording( scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 73433bd3657..3d1de8772fe 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -208,6 +208,10 @@ impl AudioRenderer { self } + pub fn set_music(&mut self, music: MusicTracks) { + self.music = music; + } + pub fn set_playhead(&mut self, playhead: f64, project: &ProjectConfiguration) { self.elapsed_samples = self.playhead_to_samples(playhead); self.speed_audio_processors = [None, None]; @@ -261,9 +265,25 @@ impl AudioRenderer { // Capture the output-time playhead before the recording mix advances // it, so timeline-positioned music is aligned to the same grid. let frame_start = self.elapsed_samples; - let (written, mut buf) = self.render_timeline_frame_raw(samples, project, timeline)?; + let (written, mut buf) = + match self.render_timeline_frame_raw(samples, project, timeline) { + Some(rendered) => rendered, + None => { + let remaining = self + .playhead_to_samples(timeline.duration()) + .saturating_sub(self.elapsed_samples); + let written = samples.min(remaining); + if written == 0 { + return None; + } + self.elapsed_samples += written; + (written, vec![0.0; written * 2]) + } + }; - if !self.music.is_empty() && !timeline.audio_segments.is_empty() { + if !self.music.is_empty() + && (!timeline.audio_segments.is_empty() || !timeline.video_segments.is_empty()) + { mix_music(&self.music, timeline, frame_start, written, &mut buf); } @@ -313,34 +333,52 @@ impl AudioRenderer { }; while written < samples { sources.check_cancelled()?; - let (mapping, span) = if !timeline.transitions.is_empty() + let mapping_span = if !timeline.transitions.is_empty() || !timeline.hold_windows().is_empty() { - let Some((mapping, output_end_samples)) = self.next_transition_mapping(timeline) - else { - break; - }; - ( - mapping, - output_end_samples - .saturating_sub(self.elapsed_samples) - .min(samples - written), - ) + self.next_transition_mapping(timeline) + .map(|(mapping, output_end_samples)| { + ( + mapping, + output_end_samples + .saturating_sub(self.elapsed_samples) + .min(samples - written), + ) + }) } else { - let Some(cursor) = self.timeline_cursor(timeline) else { - break; - }; - ( - TimelineFrameMapping::Single { - source: TimelineSource { - source_time: cursor.segment_time, - segment_index: cursor.segment_index, - segment: cursor.segment, + self.timeline_cursor(timeline).map(|cursor| { + ( + TimelineFrameMapping::Single { + source: TimelineSource { + source_time: cursor.segment_time, + segment_index: cursor.segment_index, + segment: cursor.segment, + }, + output_end: 0.0, }, - output_end: 0.0, - }, - (cursor.segment_end_samples - self.elapsed_samples).min(samples - written), - ) + (cursor.segment_end_samples - self.elapsed_samples).min(samples - written), + ) + }) + }; + let Some((mapping, span)) = mapping_span else { + let remaining = self + .playhead_to_samples(timeline.duration()) + .saturating_sub(self.elapsed_samples); + let count = (samples - written) + .min(remaining) + .min(EXPORT_AUDIO_BLOCK_SAMPLES); + if count == 0 { + break; + } + let block = &mut output[..count * 2]; + block.fill(0.0); + if !self.music.is_empty() { + mix_music(&self.music, timeline, self.elapsed_samples, count, block); + } + emit(written, block)?; + self.elapsed_samples += count; + written += count; + continue; }; if span == 0 { break; @@ -408,6 +446,15 @@ impl AudioRenderer { ); } } + if !self.music.is_empty() { + mix_music( + &self.music, + timeline, + self.elapsed_samples + span_offset, + count, + output, + ); + } emit(written + span_offset, output)?; span_offset += count; } @@ -1306,7 +1353,7 @@ fn mix_music( let frame_start = frame_start as i64; let frame_end = frame_start + samples as i64; - for segment in &timeline.audio_segments { + for segment in crate::segments::mixed_audio_segments(timeline) { if !segment.enabled || segment.end <= segment.start { continue; } @@ -2648,6 +2695,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -2784,6 +2832,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3072,6 +3121,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3188,6 +3238,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3419,6 +3470,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3461,6 +3513,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3604,6 +3657,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -3820,4 +3874,50 @@ mod tests { assert!((left_at_second(&stream, 0) - expected(4_000)).abs() < 0.02); assert!((left_at_second(&stream, 1) - expected(10_000)).abs() < 0.02); } + + #[test] + fn video_only_timeline_decodes_source_audio_and_respects_mute() { + let _ = ffmpeg::init(); + let project_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let path = "apps/media-server/src/__tests__/fixtures/test-with-audio.mp4"; + let video = cap_project::VideoSegment { + start: 0.0, + end: 0.75, + path: path.to_string(), + source_start: 0.25, + source_duration: 1.0, + ..Default::default() + }; + let mut project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + video_segments: vec![video], + ..Default::default() + }), + ..Default::default() + }; + let mut cache = MusicTracks::new(); + let music = crate::load_music_tracks(&project, &project_path, &mut cache); + let data = music.get(path).unwrap(); + assert_eq!( + data.source_start_sample(), + AudioData::SAMPLE_RATE as usize / 4 + ); + assert_eq!(data.sample_count(), AudioData::SAMPLE_RATE as usize * 3 / 4); + + let mut renderer = AudioRenderer::new(vec![]).with_music(music); + let stream = render_export_audio(&mut renderer, &project, 30, 30); + let audible = stream[..AudioData::SAMPLE_RATE as usize * 3 / 2] + .iter() + .map(|sample| sample.abs()) + .sum::() + / (AudioData::SAMPLE_RATE as f32 * 1.5); + assert!(audible > 0.01, "source audio was silent: {audible}"); + + project.timeline.as_mut().unwrap().video_segments[0].muted = true; + let muted = crate::load_music_tracks(&project, &project_path, &mut cache); + assert!(muted.is_empty()); + renderer.set_music(muted); + let stream = render_export_audio(&mut renderer, &project, 30, 30); + assert!(stream.iter().all(|sample| *sample == 0.0)); + } } diff --git a/crates/editor/src/audio_output/native_tests.rs b/crates/editor/src/audio_output/native_tests.rs index d64b46ebca6..765064792c4 100644 --- a/crates/editor/src/audio_output/native_tests.rs +++ b/crates/editor/src/audio_output/native_tests.rs @@ -419,6 +419,7 @@ fn preparing_test_sources( audio_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), transitions: Vec::new(), }), diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 16f906b3e3b..079963e3e09 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -252,12 +252,13 @@ impl Renderer { continue; } Some(RendererMessage::RenderThumbnail { input, finished }) => { + let render_display = input.segment_frames.screen_frame.is_some(); let result = frame_renderer .render_immediate( input.segment_frames, input.uniforms, &input.cursor, - true, + render_display, &mut layers, ) .await @@ -314,12 +315,13 @@ impl Renderer { while let Ok(msg) = rx.try_recv() { match msg { RendererMessage::RenderThumbnail { input, finished } => { + let render_display = input.segment_frames.screen_frame.is_some(); let result = frame_renderer .render_immediate( input.segment_frames, input.uniforms, &input.cursor, - true, + render_display, &mut layers, ) .await @@ -412,22 +414,25 @@ impl Renderer { let input_frame_number = current.input.uniforms().frame_number; let frame_layout = current.input.uniforms().frame_layout(); let render_result = match (output_format, current.input) { - (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => frame_renderer - .render_immediate_with_timings( - input.segment_frames, - input.uniforms, - &input.cursor, - true, - &mut layers, - ) - .await - .map(|(frame, timings)| { - ( - EditorFrameOutput::Rgba(frame), - PlaybackRenderOutputFormat::Rgba, - timings, + (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => { + let render_display = input.segment_frames.screen_frame.is_some(); + frame_renderer + .render_immediate_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + render_display, + &mut layers, ) - }), + .await + .map(|(frame, timings)| { + ( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + timings, + ) + }) + } ( EditorFrameFormat::Rgba, PendingRenderInput::Transition { @@ -439,16 +444,16 @@ impl Renderer { ) => frame_renderer .render_transition_immediate( TransitionRenderInput { + render_display: outgoing.segment_frames.screen_frame.is_some(), segment_frames: outgoing.segment_frames, uniforms: outgoing.uniforms, cursor: &outgoing.cursor, - render_display: true, }, TransitionRenderInput { + render_display: incoming.segment_frames.screen_frame.is_some(), segment_frames: incoming.segment_frames, uniforms: incoming.uniforms, cursor: &incoming.cursor, - render_display: true, }, kind, progress, @@ -464,12 +469,13 @@ impl Renderer { }), #[cfg(target_os = "macos")] (EditorFrameFormat::BgraSurface, PendingRenderInput::Single(input)) => { + let render_display = input.segment_frames.screen_frame.is_some(); frame_renderer .render_immediate_bgra_surface( input.segment_frames, input.uniforms, &input.cursor, - true, + render_display, &mut layers, ) .await @@ -493,16 +499,16 @@ impl Renderer { ) => frame_renderer .render_transition_bgra_surface( TransitionRenderInput { + render_display: outgoing.segment_frames.screen_frame.is_some(), segment_frames: outgoing.segment_frames, uniforms: outgoing.uniforms, cursor: &outgoing.cursor, - render_display: true, }, TransitionRenderInput { + render_display: incoming.segment_frames.screen_frame.is_some(), segment_frames: incoming.segment_frames, uniforms: incoming.uniforms, cursor: &incoming.cursor, - render_display: true, }, kind, progress, diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index f670c27a4fb..dc8fcd47b7f 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -1,15 +1,19 @@ use crate::completed_audio::{CompletedAudioHandoff, CompletedAudioSegment}; use crate::editor; use crate::playback::{self, PlaybackHandle, PlaybackStartError}; -use cap_project::StudioRecordingMeta; use cap_project::{ CursorEvents, ProjectConfiguration, RecordingMeta, RecordingMetaInner, TimelineConfiguration, TimelineFrameMapping, TimelineSegment, XY, }; +use cap_project::{StudioRecordingMeta, StudioRecordingStatus}; +use cap_rendering::media_project::{ + add_still_image_to_timeline, media_canvas_size, still_image_path, +}; use cap_rendering::{ - PrecomputedCursorTimeline, ProjectRecordingsMeta, ProjectUniforms, RecordingSegmentDecoders, - RenderVideoConstants, SegmentVideoPaths, SharedWgpuDevice, Video, ZoomTransformTimeline, - get_duration, spring_mass_damper::SpringMassDamperSimulationConfig, + BackgroundTextureCache, DecodedSegmentFrames, PrecomputedCursorTimeline, ProjectRecordingsMeta, + ProjectUniforms, RecordingSegmentDecoders, RenderOptions, RenderVideoConstants, + SegmentVideoPaths, SharedWgpuDevice, Video, ZoomTransformTimeline, get_duration, + spring_mass_damper::SpringMassDamperSimulationConfig, }; use std::{ path::{Path, PathBuf}, @@ -351,19 +355,30 @@ impl EditorInstance { }; meta.ensure_ordinary_media_access(&project_path)?; + let still_image = still_image_path(meta); - let segment_count = match meta.as_ref() { - StudioRecordingMeta::SingleSegment { .. } => 1, - StudioRecordingMeta::MultipleSegments { inner } => inner.segments.len(), + let segment_count = if still_image.is_some() { + 0 + } else { + match meta.as_ref() { + StudioRecordingMeta::SingleSegment { .. } => 1, + StudioRecordingMeta::MultipleSegments { inner } => inner.segments.len(), + } }; - if segment_count == 0 { + if segment_count == 0 && !matches!(meta.status(), StudioRecordingStatus::Complete) { return Err( "Recording has no segments. It may need to be recovered first.".to_string(), ); } let mut project = recording_meta.project_config(); + if let Some(path) = &still_image + && add_still_image_to_timeline(&mut project, path) + && let Err(error) = project.write(&recording_meta.project_path) + { + warn!(%error, "Failed to save image timeline"); + } if project.timeline.is_none() { warn!("Project config has no timeline, creating one from recording segments"); @@ -428,6 +443,7 @@ impl EditorInstance { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -442,7 +458,14 @@ impl EditorInstance { } } - if project.clips.is_empty() { + if segment_count == 0 && project.timeline.is_none() { + project.timeline = Some(TimelineConfiguration::default()); + if let Err(error) = project.write(&recording_meta.project_path) { + warn!(%error, "Failed to save media timeline"); + } + } + + if segment_count > 0 && project.clips.is_empty() { project.clips = initial_clip_configuration(&recording_meta.project_path, meta); if let Err(e) = project.write(&recording_meta.project_path) { @@ -462,7 +485,7 @@ impl EditorInstance { tracing::info!("Using FFmpeg decoder for editor preview"); } - let completed_audio = completed_audio.and_then(|handoff| { + let completed_audio = completed_audio.filter(|_| segment_count > 0).and_then(|handoff| { let matching = handoff.into_matching(&recording_meta, meta); if matching.is_none() { tracing::debug!("Completed preparing audio did not match finalized metadata; decoding ordinary sources"); @@ -476,6 +499,9 @@ impl EditorInstance { let recording_meta = recording_meta.clone(); let studio_meta = (**meta).clone(); async move { + if segment_count == 0 { + return Ok(Vec::new()); + } create_segments_with_audio( &recording_meta, &studio_meta, @@ -500,7 +526,7 @@ impl EditorInstance { let has_music = project .timeline .as_ref() - .map(|t| !t.audio_segments.is_empty()) + .map(|t| !t.audio_segments.is_empty() || !t.video_segments.is_empty()) .unwrap_or(false); if has_declared_audio || has_music { audio_output.prewarm(); @@ -523,12 +549,18 @@ impl EditorInstance { }); } - let recordings = match preloaded_recordings { - Some(recordings) => recordings, - None => Arc::new(ProjectRecordingsMeta::new( - &recording_meta.project_path, - meta.as_ref(), - )?), + let recordings = if segment_count == 0 { + Arc::new(ProjectRecordingsMeta { + segments: Vec::new(), + }) + } else { + match preloaded_recordings { + Some(recordings) => recordings, + None => Arc::new(ProjectRecordingsMeta::new( + &recording_meta.project_path, + meta.as_ref(), + )?), + } }; cap_project::synchronize_legacy_keyboard(&recording_meta, &mut project); @@ -541,22 +573,46 @@ impl EditorInstance { .collect::>(), ); + let media_options = (segment_count == 0).then(|| RenderOptions { + camera_size: None, + screen_size: media_canvas_size(&project_path, &project, still_image.as_deref()), + preserve_screen_alpha: still_image.is_some(), + }); let render_constants = if let Some(shared) = shared_device { - let rc = RenderVideoConstants::new_with_device( - shared, - &recordings.segments, - recording_meta.clone(), - (**meta).clone(), - ) - .map_err(|e| format!("Failed to create render constants: {e}"))?; + let rc = if let Some(options) = media_options { + RenderVideoConstants::from_shared_device( + shared, + options, + (**meta).clone(), + recording_meta.clone(), + Arc::new(BackgroundTextureCache::default()), + ) + } else { + RenderVideoConstants::new_with_device( + shared, + &recordings.segments, + recording_meta.clone(), + (**meta).clone(), + ) + .map_err(|e| format!("Failed to create render constants: {e}"))? + }; Arc::new(rc) } else { - let rc = RenderVideoConstants::new( - &recordings.segments, - recording_meta.clone(), - (**meta).clone(), - ) - .await + let rc = if let Some(options) = media_options { + RenderVideoConstants::new_with_options( + options, + recording_meta.clone(), + (**meta).clone(), + ) + .await + } else { + RenderVideoConstants::new( + &recordings.segments, + recording_meta.clone(), + (**meta).clone(), + ) + .await + } .map_err(|e| format!("Failed to create render constants: {e}"))?; Arc::new(rc) }; @@ -1024,6 +1080,51 @@ impl EditorInstance { } }); + if project.get_segment_time(frame_time).is_none() { + let duration = project + .timeline + .as_ref() + .map(TimelineConfiguration::duration) + .unwrap_or(0.0); + if frame_time > duration { + break; + } + let size = self.render_constants.options.screen_size; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: frame_time as f32, + recording_time: frame_time as f32, + segment_has_camera: false, + }; + let cursor = Arc::new(CursorEvents::default()); + let zoom = + ZoomTransformTimeline::from_project(&project, &cursor, duration, size); + let uniforms = ProjectUniforms::new( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &cursor, + &frames, + duration, + &zoom, + ); + if preview_rx.has_changed().unwrap_or(false) { + continue; + } + if !self + .renderer + .render_frame_confirmed(frames, uniforms, cursor) + .await + { + warn!(frame_number, "Preview renderer: media frame failed"); + } + break; + } + let Some((segment_time, segment)) = project.get_segment_time(frame_time) else { warn!( "Preview renderer: no segment found for frame {}", @@ -1844,6 +1945,58 @@ mod tests { use super::*; use cap_project::{AudioGapSummary, CursorClickEvent, CursorConfiguration, CursorMoveEvent}; + #[tokio::test] + async fn screenshot_bundle_opens_in_video_editor_and_renders_original_image() { + let directory = tempfile::tempdir().unwrap(); + let project_path = directory.path().join("Screenshot.cap"); + std::fs::create_dir_all(&project_path).unwrap(); + let image_path = project_path.join("original.png"); + image::RgbaImage::from_pixel(160, 90, image::Rgba([0, 0, 255, 255])) + .save(&image_path) + .unwrap(); + let original = std::fs::read(&image_path).unwrap(); + let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name": "Screenshot", + "display": { "path": "original.png", "fps": 0 }, + "camera": null, "audio": null, "cursor": null + })) + .unwrap(); + meta.project_path = project_path.clone(); + meta.save_for_project().unwrap(); + ProjectConfiguration::default() + .write(&project_path) + .unwrap(); + + let (tx, rx) = std::sync::mpsc::channel(); + let instance = EditorInstance::new( + project_path.clone(), + |_| {}, + Box::new(move |frame, _| { + if let crate::EditorFrameOutput::Rgba(frame) = frame { + let _ = tx.send(frame); + } + }), + None, + ) + .await + .unwrap(); + assert!(instance.segment_medias.is_empty()); + assert_eq!(instance.get_total_frames(30), 150); + instance + .preview_tx + .send(Some((0, 30, XY::new(160, 90)))) + .unwrap(); + let frame = tokio::task::spawn_blocking(move || { + rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap() + }) + .await + .unwrap(); + let center = 45 * frame.padded_bytes_per_row as usize + 80 * 4; + assert_eq!(&frame.data[center..center + 3], &[0, 0, 255]); + assert_eq!(std::fs::read(&image_path).unwrap(), original); + instance.dispose().await; + } + #[tokio::test] async fn completed_pcm_loader_reuses_arc_without_opening_a_missing_file() { let audio = crate::completed_audio::tests::audio(); diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rs index 8df5e4df7f3..0a023e177a6 100644 --- a/crates/editor/src/export_audio.rs +++ b/crates/editor/src/export_audio.rs @@ -284,6 +284,10 @@ impl ExportAudioValidation { } impl ExportAudioRenderer { + pub fn set_music(&mut self, music: crate::MusicTracks) { + self.renderer.set_music(music); + } + pub fn eligible(project: &ProjectConfiguration, meta: &StudioRecordingMeta) -> bool { let source_count = match meta { StudioRecordingMeta::SingleSegment { segment } => usize::from(segment.audio.is_some()), @@ -1090,6 +1094,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 95696fa680e..30d04be38e1 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -1,5 +1,5 @@ use cap_project::{ - ClipOffsets, ClipTransitionType, ProjectConfiguration, TimelineFrameMapping, XY, + ClipOffsets, ClipTransitionType, CursorEvents, ProjectConfiguration, TimelineFrameMapping, XY, }; use cap_rendering::{ DecodedFrame, DecodedSegmentFrames, PrecomputedCursorTimeline, ProjectUniforms, @@ -48,6 +48,7 @@ const INITIAL_PARALLEL_DECODE_TASKS: usize = 4; const FRAME_CACHE_SIZE: usize = 4; const MAX_FRAME_CACHE_BYTES: usize = 64 * 1024 * 1024; const RAMP_UP_FRAME_COUNT: u32 = 15; +const MEDIA_ONLY_SEGMENT_INDEX: u32 = u32::MAX; fn clip_audio_changed(previous: &ProjectConfiguration, next: &ProjectConfiguration) -> bool { let settings = |segment: &cap_project::TimelineSegment| { @@ -645,6 +646,7 @@ impl Playback { .map(|duration| (duration, true)) .unwrap_or((0.0, false)); let segment_media_count = self.segment_medias.len(); + let blank_screen_size = self.render_constants.options.screen_size; tokio::spawn(async move { if !has_timeline { @@ -813,6 +815,31 @@ impl Playback { is_initial, transition, }))); + } else if cached_project.get_segment_time(prefetch_time).is_none() { + let frame = PrefetchedFrame { + seek_generation, + frame_number: frame_num, + segment_frames: DecodedSegmentFrames { + screen_size: blank_screen_size, + screen_frame: None, + camera_frame: None, + segment_time: prefetch_time as f32, + recording_time: prefetch_time as f32, + segment_has_camera: false, + }, + segment_index: MEDIA_ONLY_SEGMENT_INDEX, + transition: None, + }; + next_prefetch_frame += 1; + match prefetch_tx.try_send(frame) { + Ok(()) => {} + Err(std_mpsc::TrySendError::Full(frame)) => { + pending_frame = Some(frame); + break; + } + Err(std_mpsc::TrySendError::Disconnected(_)) => return, + } + continue; } next_prefetch_frame += 1; @@ -1110,6 +1137,13 @@ impl Playback { let mut cursor_timelines = build_cursor_timelines(&cached_project); let mut zoom_timelines = build_zoom_timelines(&cached_project); let mut outgoing_zoom_timelines = build_outgoing_zoom_timelines(&cached_project); + let media_only_cursor = Arc::new(CursorEvents::default()); + let mut media_only_zoom = ZoomTransformTimeline::from_project( + &cached_project, + &media_only_cursor, + duration, + self.render_constants.options.screen_size, + ); if !*stop_rx.borrow() && let Some(prefetched_idx) = prefetch_buffer @@ -1432,6 +1466,12 @@ impl Playback { cursor_timelines = build_cursor_timelines(&cached_project); zoom_timelines = build_zoom_timelines(&cached_project); outgoing_zoom_timelines = build_outgoing_zoom_timelines(&cached_project); + media_only_zoom = ZoomTransformTimeline::from_project( + &cached_project, + &media_only_cursor, + duration, + self.render_constants.options.screen_size, + ); } let frame_offset = frame_number.saturating_sub(clock_anchor_frame) as f64; @@ -1761,128 +1801,169 @@ impl Playback { let frame_acquire_duration = frame_acquire_start.elapsed(); if let Some((segment_frames, segment_index, transition)) = segment_frames_opt { - let Some(segment_media) = self.segment_medias.get(segment_index as usize) - else { - if adopted.is_some() { - break; - } - frame_number = frame_number.saturating_add(1); - continue; - }; - - if !was_cached { - frame_cache.insert( + if segment_index == MEDIA_ONLY_SEGMENT_INDEX { + media_only_zoom + .ensure_precomputed_until((frame_number as f32 + 1.0) / fps as f32); + let cursor = Arc::clone(&media_only_cursor); + let uniforms_start = Instant::now(); + let uniforms = ProjectUniforms::new( + &self.render_constants, + &cached_project, frame_number, - Arc::clone(&segment_frames), - segment_index, - transition.as_ref().map( - |(frames, transition_index, kind, progress)| { - (Arc::clone(frames), *transition_index, *kind, *progress) - }, - ), + fps, + resolution_base, + &cursor, + &segment_frames, + duration, + &media_only_zoom, ); - } - - let zoom_until = (frame_number as f32 + 1.0) / fps as f32; - if let Some(timeline) = zoom_timelines.get_mut(segment_index as usize) { - timeline.ensure_precomputed_until(zoom_until); - } - if let Some(timeline) = outgoing_zoom_timelines.get_mut(segment_index as usize) - { - timeline.ensure_precomputed_until(zoom_until); - } - let zoom_timeline = zoom_timelines.get(segment_index as usize); + let uniforms_duration = uniforms_start.elapsed(); + let submit_start = Instant::now(); + self.renderer.render_frame( + Arc::unwrap_or_clone(segment_frames), + uniforms, + cursor, + ); + let submit_duration = submit_start.elapsed(); + if let Some(telemetry) = &self.telemetry { + telemetry.emit(PlaybackTelemetryEvent::FrameSubmitted { + frame_number, + source: frame_source, + schedule_overshoot: overshoot, + frame_acquire_duration, + uniforms_duration, + submit_duration, + prefetch_buffer_len: prefetch_buffer.len(), + total_frames_skipped, + }); + } + total_frames_rendered += 1; + } else { + let Some(segment_media) = self.segment_medias.get(segment_index as usize) + else { + if adopted.is_some() { + break; + } + frame_number = frame_number.saturating_add(1); + continue; + }; - let empty_timeline; - let zoom_ref = match zoom_timeline { - Some(timeline) => timeline, - None => { - empty_timeline = ZoomTransformTimeline::new( - &[], - None, - &segment_media.cursor, - cached_project.screen_movement_spring, - duration, - None, + if !was_cached { + frame_cache.insert( + frame_number, + Arc::clone(&segment_frames), + segment_index, + transition.as_ref().map( + |(frames, transition_index, kind, progress)| { + (Arc::clone(frames), *transition_index, *kind, *progress) + }, + ), ); - &empty_timeline } - }; - - let precomputed_cursor = &cursor_timelines[segment_index as usize]; - let uniforms_start = Instant::now(); - let uniforms = ProjectUniforms::new_with_precomputed_cursor( - &self.render_constants, - &cached_project, - frame_number, - fps, - resolution_base, - &segment_media.cursor, - &segment_frames, - duration, - zoom_ref, - precomputed_cursor, - ); - let uniforms_duration = uniforms_start.elapsed(); - let submit_start = Instant::now(); - let submitted_frame_number = frame_number; - if let Some((outgoing_frames, outgoing_index, kind, progress)) = transition { - let outgoing_media = &self.segment_medias[outgoing_index as usize]; + let zoom_until = (frame_number as f32 + 1.0) / fps as f32; + if let Some(timeline) = zoom_timelines.get_mut(segment_index as usize) { + timeline.ensure_precomputed_until(zoom_until); + } if let Some(timeline) = - outgoing_zoom_timelines.get_mut(outgoing_index as usize) + outgoing_zoom_timelines.get_mut(segment_index as usize) { timeline.ensure_precomputed_until(zoom_until); } - let outgoing_uniforms = ProjectUniforms::new_with_precomputed_cursor( + let zoom_timeline = zoom_timelines.get(segment_index as usize); + + let empty_timeline; + let zoom_ref = match zoom_timeline { + Some(timeline) => timeline, + None => { + empty_timeline = ZoomTransformTimeline::new( + &[], + None, + &segment_media.cursor, + cached_project.screen_movement_spring, + duration, + None, + ); + &empty_timeline + } + }; + + let precomputed_cursor = &cursor_timelines[segment_index as usize]; + + let uniforms_start = Instant::now(); + let uniforms = ProjectUniforms::new_with_precomputed_cursor( &self.render_constants, &cached_project, frame_number, fps, resolution_base, - &outgoing_media.cursor, - &outgoing_frames, + &segment_media.cursor, + &segment_frames, duration, - &outgoing_zoom_timelines[outgoing_index as usize], - &cursor_timelines[outgoing_index as usize], + zoom_ref, + precomputed_cursor, ); - self.renderer.render_transition_frame( - editor::RendererTransitionInput { - segment_frames: Arc::unwrap_or_clone(outgoing_frames), - uniforms: outgoing_uniforms, - cursor: outgoing_media.cursor.clone(), - }, - editor::RendererTransitionInput { - segment_frames: Arc::unwrap_or_clone(segment_frames), + let uniforms_duration = uniforms_start.elapsed(); + let submit_start = Instant::now(); + let submitted_frame_number = frame_number; + if let Some((outgoing_frames, outgoing_index, kind, progress)) = transition + { + let outgoing_media = &self.segment_medias[outgoing_index as usize]; + if let Some(timeline) = + outgoing_zoom_timelines.get_mut(outgoing_index as usize) + { + timeline.ensure_precomputed_until(zoom_until); + } + let outgoing_uniforms = ProjectUniforms::new_with_precomputed_cursor( + &self.render_constants, + &cached_project, + frame_number, + fps, + resolution_base, + &outgoing_media.cursor, + &outgoing_frames, + duration, + &outgoing_zoom_timelines[outgoing_index as usize], + &cursor_timelines[outgoing_index as usize], + ); + self.renderer.render_transition_frame( + editor::RendererTransitionInput { + segment_frames: Arc::unwrap_or_clone(outgoing_frames), + uniforms: outgoing_uniforms, + cursor: outgoing_media.cursor.clone(), + }, + editor::RendererTransitionInput { + segment_frames: Arc::unwrap_or_clone(segment_frames), + uniforms, + cursor: segment_media.cursor.clone(), + }, + kind, + progress, + ); + } else { + self.renderer.render_frame( + Arc::unwrap_or_clone(segment_frames), uniforms, - cursor: segment_media.cursor.clone(), - }, - kind, - progress, - ); - } else { - self.renderer.render_frame( - Arc::unwrap_or_clone(segment_frames), - uniforms, - segment_media.cursor.clone(), - ); - } - let submit_duration = submit_start.elapsed(); + segment_media.cursor.clone(), + ); + } + let submit_duration = submit_start.elapsed(); - if let Some(telemetry) = &self.telemetry { - telemetry.emit(PlaybackTelemetryEvent::FrameSubmitted { - frame_number: submitted_frame_number, - source: frame_source, - schedule_overshoot: overshoot, - frame_acquire_duration, - uniforms_duration, - submit_duration, - prefetch_buffer_len: prefetch_buffer.len(), - total_frames_skipped, - }); - } + if let Some(telemetry) = &self.telemetry { + telemetry.emit(PlaybackTelemetryEvent::FrameSubmitted { + frame_number: submitted_frame_number, + source: frame_source, + schedule_overshoot: overshoot, + frame_acquire_duration, + uniforms_duration, + submit_duration, + prefetch_buffer_len: prefetch_buffer.len(), + total_frames_skipped, + }); + } - total_frames_rendered += 1; + total_frames_rendered += 1; + } } if last_stats_time.elapsed() >= stats_interval { diff --git a/crates/editor/src/preparing_audio/mixer.rs b/crates/editor/src/preparing_audio/mixer.rs index 9a309333db2..d605c7a8d22 100644 --- a/crates/editor/src/preparing_audio/mixer.rs +++ b/crates/editor/src/preparing_audio/mixer.rs @@ -493,6 +493,7 @@ mod tests { audio_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), }), clips: (0..count) diff --git a/crates/editor/src/preparing_audio/output_tests.rs b/crates/editor/src/preparing_audio/output_tests.rs index c17d4391d5d..dd49583033b 100644 --- a/crates/editor/src/preparing_audio/output_tests.rs +++ b/crates/editor/src/preparing_audio/output_tests.rs @@ -26,6 +26,7 @@ fn sources(duration: f64, loader: Option) -> PreparingAudioSou audio_segments: vec![], style_segments: vec![], image_segments: vec![], + video_segments: vec![], camera3d_segments: vec![], }), clips: vec![ClipConfiguration::default()], diff --git a/crates/editor/src/segments.rs b/crates/editor/src/segments.rs index 60953e588f5..61149b22d12 100644 --- a/crates/editor/src/segments.rs +++ b/crates/editor/src/segments.rs @@ -1,7 +1,12 @@ -use std::{collections::HashMap, path::Path, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + sync::Arc, +}; use cap_audio::AudioData; -use cap_project::ProjectConfiguration; +use cap_project::{AudioTrackSegment, ProjectConfiguration, TimelineConfiguration}; +use cap_rendering::media_project::checked_project_video_source; use tracing::warn; use crate::{ @@ -18,6 +23,33 @@ fn resolve_music_path(project_path: &Path, path: &str) -> std::path::PathBuf { } } +pub(crate) fn mixed_audio_segments( + timeline: &TimelineConfiguration, +) -> impl Iterator + '_ { + timeline + .audio_segments + .iter() + .cloned() + .chain( + timeline + .video_segments + .iter() + .map(|video| AudioTrackSegment { + start: video.start, + end: video.end, + track: video.track, + path: video.path.clone(), + name: Some(video.name.clone()), + enabled: video.enabled && !video.muted, + trim_start: video.source_start, + volume_db: video.volume_db, + fade_in: 0.0, + fade_out: 0.0, + duration: Some(video.source_duration), + }), + ) +} + /// Decodes every distinct music/imported-audio file referenced by the project's /// timeline audio segments, reusing `cache` so repeated playback/export starts /// don't re-decode. Returns a snapshot keyed by the config path string for the @@ -34,10 +66,10 @@ pub fn load_music_tracks( return result; }; - let mut ranges: HashMap<&str, (usize, usize)> = HashMap::new(); + let mut ranges: HashMap = HashMap::new(); let sample_rate = AudioData::SAMPLE_RATE as f64; - for segment in &timeline.audio_segments { + for segment in mixed_audio_segments(timeline) { if !segment.enabled || segment.end <= segment.start || segment.volume_db <= MUSIC_SILENCE_DB { continue; @@ -53,7 +85,7 @@ pub fn load_music_tracks( let trim_end = trim_start.saturating_add(duration); ranges - .entry(segment.path.as_str()) + .entry(segment.path) .and_modify(|(source_start, source_end)| { *source_start = (*source_start).min(trim_start); *source_end = (*source_end).max(trim_end); @@ -61,20 +93,35 @@ pub fn load_music_tracks( .or_insert((trim_start, trim_end)); } + let video_paths: HashSet<_> = timeline + .video_segments + .iter() + .map(|video| video.path.as_str()) + .collect(); for (path, (source_start, source_end)) in ranges { - if let Some(data) = cache.get(path) + let resolved = if video_paths.contains(path.as_str()) { + match checked_project_video_source(project_path, &path) { + Ok((_, resolved)) => resolved, + Err(error) => { + warn!(path, %error, "Failed to load imported video audio; skipping"); + continue; + } + } + } else { + resolve_music_path(project_path, &path) + }; + if let Some(data) = cache.get(&path) && data.covers_source_range(source_start, source_end) { - result.insert(path.to_string(), Arc::clone(data)); + result.insert(path, Arc::clone(data)); continue; } - let resolved = resolve_music_path(project_path, path); match AudioData::from_file_range(&resolved, source_start, source_end) { Ok(data) => { let data = Arc::new(data); - cache.insert(path.to_string(), Arc::clone(&data)); - result.insert(path.to_string(), data); + cache.insert(path.clone(), Arc::clone(&data)); + result.insert(path, data); } Err(error) => { warn!( @@ -99,6 +146,40 @@ pub fn load_music_tracks_uncached( load_music_tracks(project, project_path, &mut cache) } +#[cfg(test)] +mod imported_video_path_tests { + use super::*; + use cap_project::VideoSegment; + + #[test] + fn video_audio_loader_skips_a_path_outside_the_project() { + let _ = ffmpeg::init(); + let root = tempfile::tempdir().unwrap(); + let project_path = root.path().join("project"); + std::fs::create_dir(&project_path).unwrap(); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../apps/media-server/src/__tests__/fixtures/test-with-audio.mp4"); + std::fs::copy(fixture, root.path().join("outside.mp4")).unwrap(); + let project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + video_segments: vec![VideoSegment { + start: 0.0, + end: 0.5, + source_duration: 1.0, + path: "../outside.mp4".into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + let mut cache = MusicTracks::new(); + + assert!(load_music_tracks(&project, &project_path, &mut cache).is_empty()); + assert!(cache.is_empty()); + } +} + /// Waits for a segment track's background decode, degrading a failed track to /// "no audio" (with a warning) so playback never hard-fails on a corrupt file. /// Export validates loaders strictly before reaching this point. diff --git a/crates/export/src/estimates.rs b/crates/export/src/estimates.rs index 5fad082b1a6..56cf3091ffd 100644 --- a/crates/export/src/estimates.rs +++ b/crates/export/src/estimates.rs @@ -228,11 +228,26 @@ pub async fn estimate_export( let setup_started = Instant::now(); let render_constants = tokio::select! { _ = wait_for_stop(&editor, &cancel, deadline) => return Err("Export estimate cancelled or timed out".into()), - result = cap_rendering::RenderVideoConstants::new( - &editor.recordings.segments, - editor.meta().clone(), - editor.meta().studio_meta().ok_or("Cannot estimate this recording")?.clone(), - ) => Arc::new(result.map_err(|error| error.to_string())?), + result = async { + let recording_meta = editor.meta().clone(); + let studio_meta = editor.meta().studio_meta().ok_or("Cannot estimate this recording")?.clone(); + if editor.recordings.segments.is_empty() { + cap_rendering::RenderVideoConstants::new_with_options( + editor.render_constants.options, + recording_meta, + studio_meta, + ) + .await + } else { + cap_rendering::RenderVideoConstants::new( + &editor.recordings.segments, + recording_meta, + studio_meta, + ) + .await + } + .map_err(|error| error.to_string()) + } => Arc::new(result?), }; let sample_medias = tokio::select! { _ = wait_for_stop(&editor, &cancel, deadline) => return Err("Export estimate cancelled or timed out".into()), @@ -773,6 +788,73 @@ fn summarize_pass(measurement: &PassMeasurement) -> Result 0.0); + assert!(estimate.estimated_time_seconds > 0.0); + } + fn packet(bytes: u64, key: bool) -> EncodedPacket { EncodedPacket { bytes, key } } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 9ea2e3abaa5..e9e8d24d203 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -10,7 +10,10 @@ use cap_project::{ BackgroundSource, ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, }; -use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants}; +use cap_rendering::media_project::{ + add_still_image_to_timeline, media_canvas_size, still_image_path, +}; +use cap_rendering::{ProjectRecordingsMeta, RenderOptions, RenderVideoConstants}; use std::{ path::PathBuf, sync::{ @@ -120,13 +123,21 @@ impl ExporterBuilder { let studio_meta = recording_meta .studio_meta() .ok_or(Error::NotStudioRecording)?; + let still_image = still_image_path(studio_meta); - let recordings = Arc::new( + let recordings = Arc::new(if still_image.is_some() { + ProjectRecordingsMeta { + segments: Vec::new(), + } + } else { ProjectRecordingsMeta::new(&recording_meta.project_path, studio_meta) - .map_err(Error::RecordingsMeta)?, - ); + .map_err(Error::RecordingsMeta)? + }); synthesize_default_timeline(&mut project_config, &recordings); + if let Some(path) = &still_image { + add_still_image_to_timeline(&mut project_config, path); + } cap_project::synchronize_legacy_keyboard(&recording_meta, &mut project_config); cap_project::synchronize_captions( @@ -147,22 +158,40 @@ impl ExporterBuilder { ); let stream_audio = streaming_output.is_some(); - let render_constants = Arc::new( + let render_constants = Arc::new(if recordings.segments.is_empty() { + RenderVideoConstants::new_with_options( + RenderOptions { + screen_size: media_canvas_size( + &recording_meta.project_path, + &project_config, + still_image.as_deref(), + ), + camera_size: None, + preserve_screen_alpha: still_image.is_some(), + }, + recording_meta.clone(), + studio_meta.clone(), + ) + .await + .map_err(Error::RendererSetup)? + } else { RenderVideoConstants::new( &recordings.segments, recording_meta.clone(), studio_meta.clone(), ) .await - .map_err(Error::RendererSetup)?, - ); + .map_err(Error::RendererSetup)? + }); let audio_cancellation = if stream_audio { cancellation.map(ExportAudioCancellation::new) } else { None }; - let (segments, streaming_audio) = if let Some(control) = &audio_cancellation { + let (segments, streaming_audio) = if recordings.segments.is_empty() { + (Vec::new(), None) + } else if let Some(control) = &audio_cancellation { let recording = recording_meta.clone(); let studio = studio_meta.clone(); let cancellation = control.user.clone(); @@ -276,6 +305,7 @@ pub fn synthesize_default_timeline( scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -335,6 +365,98 @@ pub fn make_cursor_only_project(mut project_config: ProjectConfiguration) -> Pro project_config } +#[cfg(test)] +mod media_only_tests { + use super::*; + use cap_project::{AudioTrackSegment, VideoSegment, XY}; + + #[tokio::test] + async fn video_only_project_exports_full_mp4_with_source_audio() { + let directory = tempfile::tempdir().unwrap(); + let project_path = directory.path().join("VideoOnly.cap"); + std::fs::create_dir_all(project_path.join("content/videos")).unwrap(); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../apps/media-server/src/__tests__/fixtures/test-with-audio.mp4"); + let imported = project_path.join("content/videos/clip.mp4"); + std::fs::copy(&source, &imported).unwrap(); + let original = std::fs::read(&imported).unwrap(); + let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name": "Video Only", + "segments": [] + })) + .unwrap(); + meta.project_path = project_path.clone(); + meta.save_for_project().unwrap(); + let mut project = ProjectConfiguration::default(); + project.background.padding = 0.0; + project.background.shadow = 0.0; + project.timeline = Some(TimelineConfiguration { + video_segments: vec![VideoSegment { + start: 0.0, + end: 1.0, + path: "content/videos/clip.mp4".into(), + source_duration: 1.0, + size: XY::new(1.0, 1.0), + ..Default::default() + }], + ..Default::default() + }); + project.write(&project_path).unwrap(); + let output = directory.path().join("output.mp4"); + let base = ExporterBase::builder(project_path) + .with_output_path(output.clone()) + .build() + .await + .unwrap(); + assert_eq!(base.total_frames(30), 30); + let settings = crate::mp4::Mp4ExportSettings { + fps: 30, + resolution_base: XY::new(160, 90), + compression: crate::mp4::ExportCompression::Social, + custom_bpp: None, + force_ffmpeg_decoder: false, + optimize_filesize: true, + }; + settings.export(base, |_| true).await.unwrap(); + let export = ffmpeg::format::input(&output).unwrap(); + assert!(export.streams().best(ffmpeg::media::Type::Video).is_some()); + assert!(export.streams().best(ffmpeg::media::Type::Audio).is_some()); + assert!(export.duration() >= 900_000); + let audio_project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + audio_segments: vec![AudioTrackSegment { + start: 0.0, + end: 1.0, + track: 0, + path: "output.mp4".into(), + name: None, + enabled: true, + trim_start: 0.0, + volume_db: 0.0, + fade_in: 0.0, + fade_out: 0.0, + duration: Some(1.0), + }], + ..Default::default() + }), + ..Default::default() + }; + let tracks = cap_editor::load_music_tracks_uncached(&audio_project, directory.path()); + let decoded = tracks.get("output.mp4").unwrap(); + let amplitude = decoded + .samples() + .iter() + .map(|sample| sample.abs()) + .sum::() + / decoded.samples().len() as f32; + assert!( + amplitude > 0.01, + "exported source audio was silent: {amplitude}" + ); + assert_eq!(std::fs::read(imported).unwrap(), original); + } +} + fn prepare_streaming_output( output: &std::path::Path, eligible: bool, diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 863246473bf..309b5bc5b1d 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -368,6 +368,9 @@ impl Mp4ExportSettings { let audio_segments = get_audio_segments(&base.segments).await; let music = load_music_tracks_uncached(&base.project_config, &base.project_path); + if let Some(audio) = &mut streaming_audio { + audio.set_music(music.clone()); + } let has_recording_audio = audio_segments .first() diff --git a/crates/export/src/preview.rs b/crates/export/src/preview.rs index 3257d07e731..1e9960a54ff 100644 --- a/crates/export/src/preview.rs +++ b/crates/export/src/preview.rs @@ -4,8 +4,9 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use cap_editor::EditorInstance; use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, TimelineFrameMapping, XY}; use cap_rendering::{ - FrameRenderer, ProjectUniforms, RecordingSegmentDecoders, RenderVideoConstants, RenderedFrame, - RendererLayers, TransitionRenderInput, ZoomTransformTimeline, + DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RecordingSegmentDecoders, + RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput, + ZoomTransformTimeline, }; use image::{ Rgba, @@ -28,6 +29,54 @@ pub struct ExportPreviewSettings { pub cursor_only: bool, } +#[cfg(test)] +mod media_only_preview_tests { + use super::*; + + #[tokio::test] + async fn image_only_project_has_export_preview() { + let directory = tempfile::tempdir().unwrap(); + let bundle = cap_project::create_media_project(directory.path(), "Image preview").unwrap(); + let images = bundle.join("content/images"); + std::fs::create_dir_all(&images).unwrap(); + image::RgbaImage::from_pixel(640, 360, Rgba([20, 80, 220, 255])) + .save(images.join("source.png")) + .unwrap(); + let mut config = ProjectConfiguration::load(&bundle).unwrap(); + config + .timeline + .as_mut() + .unwrap() + .image_segments + .push(cap_project::ImageSegment { + end: 3.0, + path: "content/images/source.png".to_string(), + size: XY::new(1.0, 1.0), + ..Default::default() + }); + config.write(&bundle).unwrap(); + let preview = render_preview_with_config( + bundle, + config, + 0.5, + ExportPreviewSettings { + fps: 30, + resolution_base: XY::new(640, 360), + compression_bpp: 0.15, + cursor_only: false, + }, + false, + ) + .await + .unwrap(); + assert_eq!(preview.total_frames, 90); + let jpeg = STANDARD.decode(preview.jpeg_base64).unwrap(); + let frame = image::load_from_memory(&jpeg).unwrap().to_rgb8(); + let pixel = frame.get_pixel(frame.width() / 2, frame.height() / 2); + assert!(pixel[2] > pixel[0] + 50); + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct ExportPreviewResult { pub jpeg_base64: String, @@ -220,6 +269,15 @@ async fn render_preview_frame( segments, total_duration, } = source; + if !frame_time.is_finite() + || frame_time < 0.0 + || frame_time >= total_duration + || settings.fps == 0 + { + return Err(ExportError::Other( + "Frame time is outside video duration".to_string(), + )); + } let transition_mapping = project_config.timeline.as_ref().and_then(|timeline| { if timeline.transitions.is_empty() { return None; @@ -235,9 +293,14 @@ async fn render_preview_frame( } }); let Some((segment_time, segment)) = project_config.get_segment_time(frame_time) else { - return Err(ExportError::Other( - "Frame time is outside video duration".to_string(), - )); + return render_blank_preview_frame( + project_config, + render_constants, + total_duration, + frame_time, + settings, + ) + .await; }; let segment_media = segments @@ -364,7 +427,68 @@ async fn render_preview_frame( .await? }; - let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0; + preview_result_from_frame( + frame, + render_start.elapsed().as_secs_f64() * 1000.0, + total_duration, + settings, + ) +} + +async fn render_blank_preview_frame( + project_config: &ProjectConfiguration, + render_constants: &RenderVideoConstants, + total_duration: f64, + frame_time: f64, + settings: ExportPreviewSettings, +) -> Result { + let render_start = std::time::Instant::now(); + let frame_number = (frame_time * f64::from(settings.fps)).floor() as u32; + let size = render_constants.options.screen_size; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: frame_time as f32, + recording_time: frame_time as f32, + segment_has_camera: false, + }; + let cursor = CursorEvents::default(); + let zoom = ZoomTransformTimeline::from_project(project_config, &cursor, total_duration, size); + let uniforms = ProjectUniforms::new( + render_constants, + project_config, + frame_number, + settings.fps, + settings.resolution_base, + &cursor, + &frames, + total_duration, + &zoom, + ); + let mut renderer = FrameRenderer::new(render_constants); + let mut layers = RendererLayers::new_with_options( + &render_constants.device, + &render_constants.queue, + render_constants.is_software_adapter, + ); + let frame = renderer + .render_immediate(frames, uniforms, &cursor, false, &mut layers) + .await?; + preview_result_from_frame( + frame, + render_start.elapsed().as_secs_f64() * 1000.0, + total_duration, + settings, + ) +} + +fn preview_result_from_frame( + frame: RenderedFrame, + frame_render_time_ms: f64, + total_duration: f64, + settings: ExportPreviewSettings, +) -> Result { let width = frame.width; let height = frame.height; diff --git a/crates/media-info/Cargo.toml b/crates/media-info/Cargo.toml index 40350902d39..8d11cf91123 100644 --- a/crates/media-info/Cargo.toml +++ b/crates/media-info/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" ffmpeg.workspace = true thiserror.workspace = true cpal.workspace = true +uuid = { version = "1.10.0", features = ["v4"] } workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/crates/media-info/src/lib.rs b/crates/media-info/src/lib.rs index 133da5e3509..169ed140c92 100644 --- a/crates/media-info/src/lib.rs +++ b/crates/media-info/src/lib.rs @@ -1,5 +1,6 @@ use cpal::{SampleFormat, SupportedBufferSize, SupportedStreamConfig}; use ffmpeg::frame; +pub mod video_import; pub use ffmpeg::{ format::{ pixel::Pixel, diff --git a/crates/media-info/src/video_import.rs b/crates/media-info/src/video_import.rs new file mode 100644 index 00000000000..b877bad1816 --- /dev/null +++ b/crates/media-info/src/video_import.rs @@ -0,0 +1,234 @@ +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, +}; + +pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "wmv", "m4v", "flv"]; + +#[derive(Clone, Debug)] +pub struct ImportedVideo { + pub path: String, + pub name: String, + pub duration: f64, + pub fps: u32, + pub width: u32, + pub height: u32, + pub has_audio: bool, +} + +pub fn is_supported_video_path(path: &Path) -> bool { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + VIDEO_EXTENSIONS + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }) +} + +fn probe_video(path: &Path) -> Result<(f64, u32, u32, u32, bool), String> { + let input = + ffmpeg::format::input(path).map_err(|error| format!("Cannot open video: {error}"))?; + let stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or_else(|| "The file has no video track".to_string())?; + let decoder = ffmpeg::codec::Context::from_parameters(stream.parameters()) + .map_err(|error| format!("Cannot inspect video: {error}"))? + .decoder() + .video() + .map_err(|error| format!("Cannot decode video: {error}"))?; + let (width, height) = (decoder.width(), decoder.height()); + if width == 0 + || height == 0 + || width > 16_384 + || height > 16_384 + || u64::from(width) * u64::from(height) > 33_554_432 + { + return Err( + "Videos must be at most 16,384 pixels per side and 33,554,432 pixels per frame".into(), + ); + } + let duration = if input.duration() > 0 { + input.duration() as f64 / 1_000_000.0 + } else { + let time_base = stream.time_base(); + if stream.duration() <= 0 || time_base.denominator() <= 0 { + return Err("Cannot determine video duration".into()); + } + stream.duration() as f64 * time_base.numerator() as f64 / time_base.denominator() as f64 + }; + if !duration.is_finite() || duration <= 0.0 { + return Err("Cannot determine video duration".into()); + } + let rate = stream.avg_frame_rate(); + let fps = if rate.denominator() > 0 { + (rate.numerator() as f64 / rate.denominator() as f64).round() + } else { + 30.0 + }; + let fps = if fps.is_finite() && (1.0..=240.0).contains(&fps) { + fps as u32 + } else { + 30 + }; + let has_audio = input.streams().best(ffmpeg::media::Type::Audio).is_some(); + Ok((duration, fps, width, height, has_audio)) +} + +pub fn import_video(project_path: &Path, source: &Path) -> Result { + if !project_path.is_dir() { + return Err("The editor project is unavailable".into()); + } + if !is_supported_video_path(source) { + return Err("Choose an MP4, MOV, AVI, MKV, WebM, WMV, M4V or FLV video".into()); + } + let mut source_file = std::fs::File::open(source) + .map_err(|error| format!("Cannot open source video: {error}"))?; + let source_metadata = source_file + .metadata() + .map_err(|error| format!("Cannot inspect source video: {error}"))?; + if !source_metadata.is_file() || source_metadata.len() == 0 { + return Err("The source video is empty or unavailable".into()); + } + let extension = source + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or("mp4") + .to_ascii_lowercase(); + let id = uuid::Uuid::new_v4(); + let directory = project_path.join("content/videos"); + for candidate in [project_path.join("content"), directory.clone()] { + match std::fs::symlink_metadata(&candidate) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("Video assets cannot use linked project directories".into()); + } + Ok(_) => {} + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(format!("Cannot inspect video assets: {error}")), + } + } + std::fs::create_dir_all(&directory) + .map_err(|error| format!("Cannot create video assets: {error}"))?; + let root = project_path + .canonicalize() + .map_err(|error| format!("Cannot inspect editor project: {error}"))?; + let resolved_directory = directory + .canonicalize() + .map_err(|error| format!("Cannot inspect video assets: {error}"))?; + if !resolved_directory.starts_with(&root) { + return Err("Video assets must stay inside the editor project".into()); + } + let temporary = directory.join(format!(".{id}.import")); + let result = (|| { + let mut writable_temporary = None; + let copied = if cfg!(windows) && source_metadata.permissions().readonly() { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| format!("Cannot create video asset: {error}"))?; + let copied = std::io::copy(&mut source_file, &mut file) + .map_err(|error| format!("Cannot copy video into project: {error}"))?; + writable_temporary = Some(file); + copied + } else { + std::fs::copy(source, &temporary) + .map_err(|error| format!("Cannot copy video into project: {error}"))? + }; + if copied != source_metadata.len() { + return Err("The source video changed during import. Drop it again.".into()); + } + match writable_temporary { + Some(file) => file.sync_all(), + None => std::fs::OpenOptions::new() + .read(true) + .write(cfg!(windows)) + .open(&temporary) + .and_then(|file| file.sync_all()), + } + .map_err(|error| format!("Cannot save video asset: {error}"))?; + let (duration, fps, width, height, has_audio) = probe_video(&temporary)?; + let path = format!("content/videos/{id}.{extension}"); + std::fs::rename(&temporary, project_path.join(&path)) + .map_err(|error| format!("Cannot finish video import: {error}"))?; + Ok(ImportedVideo { + path, + name: source + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or("Video") + .to_string(), + duration, + fps, + width, + height, + has_audio, + }) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result +} + +pub fn resolved_video_path(project_path: &Path, video: &ImportedVideo) -> PathBuf { + project_path.join(&video.path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn video_import_keeps_source_and_creates_distinct_project_assets() { + let directory = + std::env::temp_dir().join(format!("cap-video-import-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../video-decode/tests/fixtures/h264-decoder-lifecycle.mp4"); + let original = std::fs::read(&source).unwrap(); + let first = import_video(&directory, &source).unwrap(); + let second = import_video(&directory, &source).unwrap(); + assert_ne!(first.path, second.path); + assert!(first.duration > 0.0); + assert!(first.width > 0 && first.height > 0 && first.fps > 0); + assert!(!Path::new(&first.path).is_absolute()); + assert_eq!( + std::fs::read(resolved_video_path(&directory, &first)).unwrap(), + original + ); + assert_eq!(std::fs::read(&source).unwrap(), original); + let damaged = directory.join("damaged.mp4"); + std::fs::write(&damaged, b"invalid video").unwrap(); + assert!(import_video(&directory, &damaged).is_err()); + assert_eq!( + std::fs::read_dir(directory.join("content/videos")) + .unwrap() + .count(), + 2 + ); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[cfg(unix)] + #[test] + fn video_import_rejects_a_linked_asset_directory_without_writing_outside() { + let root = + std::env::temp_dir().join(format!("cap-video-link-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let project = root.join("project"); + let outside = root.join("outside"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&outside).unwrap(); + std::os::unix::fs::symlink(&outside, project.join("content")).unwrap(); + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../video-decode/tests/fixtures/h264-decoder-lifecycle.mp4"); + + assert!(import_video(&project, &source).is_err()); + assert_eq!(std::fs::read_dir(&outside).unwrap().count(), 0); + std::fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 4e107005f4d..029756c14e9 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -1704,6 +1704,10 @@ pub struct ImageSegment { pub track: u32, pub enabled: bool, pub path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_path: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub annotations: Vec, pub name: String, pub center: XY, pub size: XY, @@ -1723,6 +1727,8 @@ impl Default for ImageSegment { track: 0, enabled: true, path: String::new(), + source_path: None, + annotations: Vec::new(), name: "Image".to_string(), center: XY::new(0.5, 0.5), size: XY::new(0.3, 0.3), @@ -1742,6 +1748,73 @@ impl ImageSegment { } } +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase", default)] +pub struct VideoSegment { + pub start: f64, + pub end: f64, + pub track: u32, + pub enabled: bool, + pub path: String, + pub name: String, + pub source_start: f64, + pub source_duration: f64, + pub muted: bool, + pub volume_db: f32, + pub center: XY, + pub size: XY, + pub opacity: f32, + pub rotation: f32, + pub rounding: f32, + pub flip_x: bool, + pub flip_y: bool, + pub lock_aspect: bool, +} + +impl Default for VideoSegment { + fn default() -> Self { + Self { + start: 0.0, + end: 0.0, + track: 0, + enabled: true, + path: String::new(), + name: "Video".to_string(), + source_start: 0.0, + source_duration: 0.0, + muted: false, + volume_db: 0.0, + center: XY::new(0.5, 0.5), + size: XY::new(1.0, 1.0), + opacity: 1.0, + rotation: 0.0, + rounding: 0.0, + flip_x: false, + flip_y: false, + lock_aspect: true, + } + } +} + +impl VideoSegment { + pub fn is_active_at(&self, time: f64) -> bool { + self.enabled && is_timeline_interval_active(self.start, self.end, time) + } + + pub fn source_time_at(&self, time: f64) -> Option { + if !self.is_active_at(time) + || !self.source_start.is_finite() + || !self.source_duration.is_finite() + || self.source_start < 0.0 + || self.source_duration <= self.source_start + { + return None; + } + let source_time = self.source_start + time - self.start; + (source_time < self.source_duration).then_some(source_time) + } +} + fn is_timeline_interval_active(start: f64, end: f64, time: f64) -> bool { time.is_finite() && start.is_finite() @@ -1786,7 +1859,7 @@ where Ok(by_segment.into_values().collect()) } -#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] #[serde(rename_all = "camelCase")] pub struct TimelineConfiguration { pub segments: Vec, @@ -1813,6 +1886,8 @@ pub struct TimelineConfiguration { pub style_segments: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub image_segments: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub video_segments: Vec, // Explicit rename: the digit boundary makes rename_all's camelCase output // easy to second-guess, and the editor TypeScript hardcodes this name. #[serde(default, rename = "camera3dSegments")] @@ -2120,7 +2195,20 @@ impl TimelineConfiguration { .sum::() }; - segment_duration + self.held_duration() + let recording_duration = segment_duration + self.held_duration(); + let media_duration = self + .image_segments + .iter() + .filter(|segment| segment.enabled && segment.end.is_finite()) + .map(|segment| segment.end) + .chain( + self.video_segments + .iter() + .filter(|segment| segment.enabled && segment.end.is_finite()) + .map(|segment| segment.end), + ) + .fold(0.0, f64::max); + recording_duration.max(media_duration) } /// Gapless-time windows covered by clips that hide the cursor, with @@ -2563,6 +2651,7 @@ impl Annotation { pub enum OverlayTrackKind { Mask, Image, + Video, Text, } @@ -2655,6 +2744,45 @@ impl Default for ProjectConfiguration { } } +pub fn create_media_project(base: &Path, pretty_name: &str) -> Result { + std::fs::create_dir_all(base) + .map_err(|error| format!("Cannot create media library: {error}"))?; + let project_path = base.join(format!("{}.cap", uuid::Uuid::new_v4())); + std::fs::create_dir(&project_path) + .map_err(|error| format!("Cannot create media project: {error}"))?; + let result = (|| { + let meta = crate::RecordingMeta { + platform: Some(crate::Platform::default()), + project_path: project_path.clone(), + pretty_name: pretty_name.to_string(), + sharing: None, + inner: crate::RecordingMetaInner::Studio(Box::new( + crate::StudioRecordingMeta::MultipleSegments { + inner: crate::MultipleSegments { + segments: Vec::new(), + cursors: Default::default(), + status: Some(crate::StudioRecordingStatus::Complete), + }, + }, + )), + upload: None, + }; + meta.save_for_project() + .map_err(|error| format!("Cannot save media metadata: {error}"))?; + ProjectConfiguration { + timeline: Some(TimelineConfiguration::default()), + ..Default::default() + } + .write(&project_path) + .map_err(|error| format!("Cannot save media timeline: {error}"))?; + Ok(project_path.clone()) + })(); + if result.is_err() { + let _ = std::fs::remove_dir_all(&project_path); + } + result +} + impl ProjectConfiguration { pub fn resolved_overlay_order(&self, available: &[OverlayTrack]) -> Vec { let mut ordered = Vec::with_capacity(available.len()); @@ -2693,6 +2821,14 @@ impl ProjectConfiguration { .map(|segment| segment.track) .collect::>(), ), + ( + OverlayTrackKind::Video, + timeline + .video_segments + .iter() + .map(|segment| segment.track) + .collect::>(), + ), ( OverlayTrackKind::Mask, timeline @@ -2799,6 +2935,13 @@ impl ProjectConfiguration { for annotation in &self.annotations { annotation.validate()?; } + if let Some(timeline) = &self.timeline { + for segment in &timeline.image_segments { + for annotation in &segment.annotations { + annotation.validate()?; + } + } + } Ok(()) } @@ -3026,6 +3169,25 @@ mod notch_tests { mod tests { use super::*; + #[test] + fn media_project_has_a_complete_empty_studio_timeline() { + let directory = tempfile::tempdir().unwrap(); + let project_path = create_media_project(directory.path(), "Imported video").unwrap(); + let meta = crate::RecordingMeta::load_for_project(&project_path).unwrap(); + assert_eq!(meta.pretty_name, "Imported video"); + let studio = meta.studio_meta().unwrap(); + assert!(matches!( + studio.status(), + crate::StudioRecordingStatus::Complete + )); + assert!(matches!( + studio, + crate::StudioRecordingMeta::MultipleSegments { inner } if inner.segments.is_empty() + )); + let config = ProjectConfiguration::load(&project_path).unwrap(); + assert_eq!(config.timeline.unwrap().duration(), 0.0); + } + #[test] fn studio_sound_defaults_old_projects_to_balanced_and_round_trips_tiers() { let legacy: AudioConfiguration = @@ -3105,6 +3267,7 @@ mod tests { audio_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), } } @@ -4261,6 +4424,7 @@ mod tests { audio_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), }), ..Default::default() @@ -4370,6 +4534,7 @@ mod tests { audio_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), camera3d_segments: Vec::new(), }), ..Default::default() diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index 9eddbcbeb2e..5e60cd0b7f9 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -2350,6 +2350,16 @@ impl RecoveryManager { } Err(error) => return Err(error.into()), }; + let (image_segments, video_segments) = config + .timeline + .as_ref() + .map(|timeline| { + ( + timeline.image_segments.clone(), + timeline.video_segments.clone(), + ) + }) + .unwrap_or_default(); config.timeline = Some(TimelineConfiguration { segments: timeline_segments, @@ -2357,7 +2367,8 @@ impl RecoveryManager { zoom_segments: Vec::new(), scene_segments: Vec::new(), style_segments: Vec::new(), - image_segments: Vec::new(), + image_segments, + video_segments, mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 1ff44d06f1b..2e9ceb57db5 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -2850,6 +2850,7 @@ async fn stop_recording( scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/recording/src/track_heal.rs b/crates/recording/src/track_heal.rs index 9ebfba3046f..659c52868db 100644 --- a/crates/recording/src/track_heal.rs +++ b/crates/recording/src/track_heal.rs @@ -1067,6 +1067,7 @@ mod tests { scene_segments: vec![], style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: vec![], text_segments: vec![], caption_segments: vec![], @@ -1116,6 +1117,7 @@ mod tests { scene_segments: vec![], style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: vec![], text_segments: vec![], caption_segments: vec![], @@ -1214,6 +1216,7 @@ mod tests { scene_segments: vec![], style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: vec![], text_segments: vec![], caption_segments: vec![], diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 897b82a2d19..707ebac0641 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -14,6 +14,7 @@ mod keyboard; mod mask; mod notch; mod text; +mod video; use std::sync::OnceLock; @@ -92,6 +93,7 @@ pub use keyboard::*; pub use mask::*; pub use notch::*; pub use text::*; +pub use video::*; #[cfg(test)] mod font_tests { diff --git a/crates/rendering/src/layers/video.rs b/crates/rendering/src/layers/video.rs new file mode 100644 index 00000000000..a9735f05b12 --- /dev/null +++ b/crates/rendering/src/layers/video.rs @@ -0,0 +1,268 @@ +use std::{collections::HashMap, path::Path, sync::Arc}; + +use cap_project::{VideoSegment, XY}; + +use crate::{ + DecodedSegmentFrames, ProjectUniforms, RenderVideoConstants, RenderingError, + composite_frame::{CompositeVideoFramePipeline, CompositeVideoFrameUniforms}, + decoder::{ManagedVideoDecoder, spawn_managed_decoder}, + media_project::checked_project_video_source, + yuv_converter::YuvConverterPipelines, +}; + +use super::DisplayLayer; + +const MAX_ACTIVE_VIDEO_OVERLAYS: usize = 8; + +struct VideoInstance { + path: String, + decoder: ManagedVideoDecoder, + display: DisplayLayer, +} + +pub struct VideoLayer { + instances: HashMap, + draw_order: Vec<(u32, usize)>, + yuv_pipelines: Arc, + composite_pipeline: Arc, + prefer_cpu_conversion: bool, +} + +impl VideoLayer { + pub fn new( + yuv_pipelines: Arc, + composite_pipeline: Arc, + prefer_cpu_conversion: bool, + ) -> Self { + Self { + instances: HashMap::new(), + draw_order: Vec::new(), + yuv_pipelines, + composite_pipeline, + prefer_cpu_conversion, + } + } + + async fn frames( + &mut self, + constants: &RenderVideoConstants, + uniforms: &ProjectUniforms, + ) -> Result< + Vec<( + usize, + u32, + DecodedSegmentFrames, + CompositeVideoFrameUniforms, + )>, + RenderingError, + > { + self.draw_order.clear(); + let Some(timeline) = uniforms.project.timeline.as_ref() else { + self.instances.clear(); + return Ok(Vec::new()); + }; + let time = f64::from(uniforms.frame_number) / f64::from(uniforms.frame_rate.max(1)); + let visible: Vec<_> = timeline + .video_segments + .iter() + .enumerate() + .filter_map(|(index, segment)| { + let source_time = segment.source_time_at(time)?; + (segment.opacity.is_finite() + && segment.opacity > 0.0 + && segment.rotation.is_finite() + && segment.rounding.is_finite() + && !segment.path.is_empty()) + .then_some((index, segment, source_time)) + }) + .collect(); + if visible.len() > MAX_ACTIVE_VIDEO_OVERLAYS { + return Err(RenderingError::VideoOverlayDecodeFailed( + "At most eight video layers may overlap at one time".into(), + )); + } + self.instances.retain(|index, instance| { + timeline + .video_segments + .get(*index) + .is_some_and(|segment| segment.path == instance.path) + }); + let mut frames = Vec::with_capacity(visible.len()); + for (index, segment, source_time) in visible { + if !self.instances.contains_key(&index) { + let (source, _) = checked_project_video_source( + &constants.recording_meta.project_path, + &segment.path, + ) + .map_err(|error| RenderingError::VideoOverlayDecodeFailed(error.to_string()))?; + let decoder = spawn_managed_decoder( + "video-overlay", + source, + [Path::new(&segment.path)], + uniforms.frame_rate.max(1), + 0.0, + true, + ) + .map_err(|error| RenderingError::VideoOverlayDecodeFailed(error.to_string()))?; + let display = DisplayLayer::new_with_all_shared_pipelines( + &constants.device, + self.yuv_pipelines.clone(), + self.composite_pipeline.clone(), + self.prefer_cpu_conversion, + ); + self.instances.insert( + index, + VideoInstance { + path: segment.path.clone(), + decoder, + display, + }, + ); + } + let instance = self.instances.get_mut(&index).unwrap(); + instance.decoder.wait_ready().await.map_err(|error| { + RenderingError::VideoOverlayDecodeFailed(format!("{}: {error}", segment.name)) + })?; + let frame = instance + .decoder + .get_frame(source_time as f32) + .await + .map_err(|error| { + RenderingError::VideoOverlayDecodeFailed(format!( + "{} at {source_time:.2}s: {error}", + segment.name + )) + })?; + let size = XY::new(frame.width(), frame.height()); + let composite = Self::uniforms_for(segment, size, uniforms.output_size); + frames.push(( + index, + segment.track, + DecodedSegmentFrames { + screen_size: size, + screen_frame: Some(frame), + camera_frame: None, + segment_time: source_time as f32, + recording_time: source_time as f32, + segment_has_camera: false, + }, + composite, + )); + } + Ok(frames) + } + + fn uniforms_for( + segment: &VideoSegment, + frame_size: XY, + output_size: (u32, u32), + ) -> CompositeVideoFrameUniforms { + let output_width = output_size.0 as f32; + let output_height = output_size.1 as f32; + let center_x = segment.center.x as f32 * output_width; + let center_y = segment.center.y as f32 * output_height; + let width = segment.size.x as f32 * output_width; + let height = segment.size.y as f32 * output_height; + CompositeVideoFrameUniforms { + crop_bounds: [0.0, 0.0, frame_size.x as f32, frame_size.y as f32], + target_bounds: [ + center_x - width * 0.5, + center_y - height * 0.5, + center_x + width * 0.5, + center_y + height * 0.5, + ], + output_size: [output_width, output_height], + frame_size: [frame_size.x as f32, frame_size.y as f32], + target_size: [width, height], + rounding_px: segment.rounding.clamp(0.0, 100.0) * 0.005 * width.min(height), + mirror_x: u8::from(segment.flip_x) as f32, + opacity: segment.opacity.clamp(0.0, 1.0), + _padding1: [ + segment.rotation.to_radians(), + u8::from(segment.flip_y) as f32, + 0.0, + ], + ..Default::default() + } + } + + pub async fn prepare( + &mut self, + constants: &RenderVideoConstants, + uniforms: &ProjectUniforms, + ) -> Result<(), RenderingError> { + for (index, track, frame, composite) in self.frames(constants, uniforms).await? { + let display = &mut self.instances.get_mut(&index).unwrap().display; + let (ready, _, _) = display.prepare( + &constants.device, + &constants.queue, + &frame, + frame.screen_size, + composite, + ); + if !ready { + return Err(RenderingError::VideoOverlayDecodeFailed( + "Unable to upload an imported video frame".into(), + )); + } + self.draw_order.push((track, index)); + } + self.draw_order.sort_unstable(); + Ok(()) + } + + pub async fn prepare_with_encoder( + &mut self, + constants: &RenderVideoConstants, + uniforms: &ProjectUniforms, + encoder: &mut wgpu::CommandEncoder, + ) -> Result<(), RenderingError> { + for (index, track, frame, composite) in self.frames(constants, uniforms).await? { + let display = &mut self.instances.get_mut(&index).unwrap().display; + if !display.prepare_with_encoder( + &constants.device, + &constants.queue, + &frame, + composite, + encoder, + ) { + return Err(RenderingError::VideoOverlayDecodeFailed( + "Unable to upload an imported video frame".into(), + )); + } + self.draw_order.push((track, index)); + } + self.draw_order.sort_unstable(); + Ok(()) + } + + pub fn copy_to_texture(&mut self, encoder: &mut wgpu::CommandEncoder) { + for (_, index) in &self.draw_order { + if let Some(instance) = self.instances.get_mut(index) { + instance.display.copy_to_texture(encoder); + } + } + } + + pub fn has_track(&self, track: u32) -> bool { + self.draw_order.iter().any(|(draw_track, index)| { + *draw_track == track + && self + .instances + .get(index) + .is_some_and(|instance| instance.display.has_valid_frame()) + }) + } + + pub fn render_track(&self, pass: &mut wgpu::RenderPass<'_>, track: u32) { + for (_, index) in self + .draw_order + .iter() + .filter(|(draw_track, _)| *draw_track == track) + { + if let Some(instance) = self.instances.get(index) { + instance.display.render(pass); + } + } + } +} diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index a3e2c0f9ae5..346b0329bc6 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -48,6 +48,7 @@ pub mod iosurface_texture; mod layers; mod managed_segment; mod mask; +pub mod media_project; pub mod notch_shape; mod overlay_layers; mod project_recordings; @@ -589,6 +590,8 @@ pub enum RenderingError { ChannelSendNv12FrameFailed(#[from] mpsc::error::SendError<(Nv12RenderedFrame, u32)>), #[error("Failed to load image: {0}")] ImageLoadError(String), + #[error("Failed to decode imported video: {0}")] + VideoOverlayDecodeFailed(String), #[error("Error polling wgpu: {0}")] PollError(#[from] wgpu::PollError), #[error("Failed to upload display frame {frame_number} at recording time {recording_time}")] @@ -747,6 +750,41 @@ pub async fn render_video_to_channel( } }); + if project.get_segment_time(frame_time).is_none() { + next_frame = frame_windows.next_after(current_frame_number); + last_frame_number = current_frame_number; + frames_rendered += 1; + let size = constants.options.screen_size; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: frame_time as f32, + recording_time: frame_time as f32, + segment_has_camera: false, + }; + let cursor = CursorEvents::default(); + let zoom = ZoomTransformTimeline::from_project(project, &cursor, duration, size); + let uniforms = ProjectUniforms::new( + constants, + project, + current_frame_number, + fps, + resolution_base, + &cursor, + &frames, + duration, + &zoom, + ); + if let Some(frame) = frame_renderer + .render(frames, uniforms, &cursor, false, &mut layers) + .await? + { + last_successful_frame = Some(frame.clone()); + sender.send((frame, current_frame_number)).await?; + } + continue; + } let Some((segment_time, segment)) = project.get_segment_time(frame_time) else { break; }; @@ -1222,6 +1260,46 @@ pub async fn render_video_to_channel_nv12( } }); + if project.get_segment_time(frame_time).is_none() { + next_frame = frame_windows.next_after(current_frame_number); + last_frame_number = current_frame_number; + frames_rendered += 1; + let size = constants.options.screen_size; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: frame_time as f32, + recording_time: frame_time as f32, + segment_has_camera: false, + }; + let cursor = CursorEvents::default(); + let zoom = ZoomTransformTimeline::from_project(project, &cursor, duration, size); + let uniforms = ProjectUniforms::new( + constants, + project, + current_frame_number, + fps, + resolution_base, + &cursor, + &frames, + duration, + &zoom, + ); + if let Some(frame) = frame_renderer + .render_nv12(frames, uniforms, &cursor, false, &mut layers) + .await? + { + last_successful_frame = Some(frame.clone_metadata_with_data()); + sender.send((frame, current_frame_number)).await?; + channel_frames_sent += 1; + if stop_after_frames_sent.is_some_and(|limit| channel_frames_sent >= limit) { + stopped_after_frame_limit = true; + break; + } + } + continue; + } let Some((segment_time, segment)) = project.get_segment_time(frame_time) else { break; }; @@ -5395,7 +5473,93 @@ pub struct FrameRenderStageTimings { #[cfg(test)] mod style_image_tests { use super::*; - use cap_project::{BackgroundSource, ImageSegment, StyleOverrides, StyleSegment}; + use cap_project::{ + BackgroundSource, ImageSegment, StyleOverrides, StyleSegment, TimelineConfiguration, + VideoSegment, + }; + use std::path::Path; + + #[tokio::test] + async fn imported_video_renders_without_a_recording_and_respects_rotation() { + let mut recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name": "video-overlay-test", + "display": { "path": "display.mp4", "fps": 30 }, + "camera": null, "audio": null, "cursor": null + })) + .unwrap(); + recording_meta.project_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let meta = recording_meta.studio_meta().unwrap().clone(); + let size = XY::new(160, 90); + let constants = RenderVideoConstants::new_with_options( + RenderOptions { + screen_size: size, + camera_size: None, + preserve_screen_alpha: false, + }, + recording_meta, + meta, + ) + .await + .unwrap(); + let mut project = ProjectConfiguration::default(); + project.background.source = BackgroundSource::Color { + value: [255, 255, 255], + alpha: 255, + }; + project.background.padding = 0.0; + project.background.shadow = 0.0; + project.timeline = Some(TimelineConfiguration { + video_segments: vec![VideoSegment { + start: 0.0, + end: 1.0, + path: "crates/video-decode/tests/fixtures/h264-decoder-lifecycle.mp4".into(), + source_duration: 1.0, + size: XY::new(1.0, 1.0), + ..Default::default() + }], + ..Default::default() + }); + let cursor = CursorEvents::default(); + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: 0.2, + recording_time: 0.2, + segment_has_camera: false, + }; + let zoom = ZoomTransformTimeline::from_project(&project, &cursor, 1.0, size); + let mut layers = RendererLayers::new(&constants.device, &constants.queue); + let mut renderer = FrameRenderer::new(&constants); + let uniforms = ProjectUniforms::new( + &constants, &project, 6, 30, size, &cursor, &frames, 1.0, &zoom, + ); + layers + .prepare(&constants, &uniforms, &frames, &cursor, false) + .await + .unwrap(); + let normal = renderer + .render_immediate(frames.clone(), uniforms, &cursor, false, &mut layers) + .await + .unwrap(); + + project.timeline.as_mut().unwrap().video_segments[0].rotation = 90.0; + let uniforms = ProjectUniforms::new( + &constants, &project, 6, 30, size, &cursor, &frames, 1.0, &zoom, + ); + layers + .prepare(&constants, &uniforms, &frames, &cursor, false) + .await + .unwrap(); + let rotated = renderer + .render_immediate(frames, uniforms, &cursor, false, &mut layers) + .await + .unwrap(); + + assert_ne!(normal.data, rotated.data); + let center = (45 * normal.padded_bytes_per_row as usize) + 80 * 4; + assert_ne!(&normal.data[center..center + 3], &[255, 255, 255]); + } #[tokio::test] async fn camera_only_ignores_cutout_and_preserves_background_blur() { @@ -6866,12 +7030,20 @@ impl RendererLayers { camera_only: readiness::measure("layers.camera_only", || { CameraLayer::new_with_all_shared_pipelines( device, + shared_yuv_pipelines.clone(), + shared_composite_pipeline.clone(), + ) + }), + mask: readiness::measure("layers.mask", || MaskLayer::new(device)), + overlays: include_overlays.then(|| { + OverlayLayers::new( + device, + queue, shared_yuv_pipelines, shared_composite_pipeline, + prefer_cpu_conversion, ) }), - mask: readiness::measure("layers.mask", || MaskLayer::new(device)), - overlays: include_overlays.then(|| OverlayLayers::new(device, queue)), camera3d: readiness::measure("layers.camera3d", || Camera3DLayer::new(device)), camera_blur_processor: None, camera_blur_init_failed: false, @@ -7132,6 +7304,7 @@ impl RendererLayers { if let Some(overlays) = &mut self.overlays { overlays.images.prepare(constants, uniforms).await; + overlays.videos.prepare(constants, uniforms).await?; if uniforms.project.overlay_order.is_empty() { overlays.text.prepare( @@ -7311,6 +7484,10 @@ impl RendererLayers { if let Some(overlays) = &mut self.overlays { let start = Instant::now(); overlays.images.prepare(constants, uniforms).await; + overlays + .videos + .prepare_with_encoder(constants, uniforms, encoder) + .await?; if uniforms.project.overlay_order.is_empty() { overlays.text.prepare( @@ -7388,6 +7565,9 @@ impl RendererLayers { } self.camera.copy_to_texture(encoder); self.camera_only.copy_to_texture(encoder); + if let Some(overlays) = &mut self.overlays { + overlays.videos.copy_to_texture(encoder); + } self.background.render_surface(encoder); { @@ -7535,7 +7715,17 @@ impl RendererLayers { } if let Some(overlays) = &self.overlays { - if render_display && overlays.images.has_content() { + for overlay in uniforms.project.overlay_tracks() { + if overlay.kind == OverlayTrackKind::Video + && overlays.videos.has_track(overlay.track) + { + let mut pass = + render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + overlays.videos.render_track(&mut pass, overlay.track); + } + } + + if overlays.images.has_content() { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); overlays.images.render(&mut pass); } @@ -7557,9 +7747,7 @@ impl RendererLayers { self.mask.render(device, queue, session, encoder, mask); } } - OverlayTrackKind::Image - if render_display && overlays.images.has_track(overlay.track) => - { + OverlayTrackKind::Image if overlays.images.has_track(overlay.track) => { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); overlays.images.render_track(&mut pass, overlay.track); @@ -7569,7 +7757,12 @@ impl RendererLayers { render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); overlays.text.render_track(&mut pass, overlay.track); } - OverlayTrackKind::Text | OverlayTrackKind::Image => {} + OverlayTrackKind::Video if overlays.videos.has_track(overlay.track) => { + let mut pass = + render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + overlays.videos.render_track(&mut pass, overlay.track); + } + OverlayTrackKind::Text | OverlayTrackKind::Image | OverlayTrackKind::Video => {} } } } diff --git a/crates/rendering/src/media_project.rs b/crates/rendering/src/media_project.rs new file mode 100644 index 00000000000..c4b98beb33b --- /dev/null +++ b/crates/rendering/src/media_project.rs @@ -0,0 +1,203 @@ +use std::{ + io, + path::{Path, PathBuf}, +}; + +use cap_enc_ffmpeg::{RelocatableSource, SegmentedInput}; +use cap_project::{ + ImageSegment, ProjectConfiguration, StudioRecordingMeta, TimelineConfiguration, XY, +}; + +use crate::Video; + +pub fn checked_project_video_source( + project_path: &Path, + relative: &str, +) -> io::Result<(RelocatableSource, PathBuf)> { + let source = RelocatableSource::new(project_path.to_path_buf())?; + let relative = Path::new(relative); + source.reader(relative)?; + let root = project_path.canonicalize()?; + let resolved = root.join(relative).canonicalize()?; + if !resolved.starts_with(&root) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Imported video is outside the editor project", + )); + } + Ok((source, resolved)) +} + +pub fn still_image_path(meta: &StudioRecordingMeta) -> Option { + let StudioRecordingMeta::SingleSegment { segment } = meta else { + return None; + }; + (segment.display.fps == 0).then(|| segment.display.path.to_string()) +} + +pub fn add_still_image_to_timeline(project: &mut ProjectConfiguration, path: &str) -> bool { + let legacy_annotations = project.annotations.clone(); + let timeline = project + .timeline + .get_or_insert_with(TimelineConfiguration::default); + if let Some(segment) = timeline + .image_segments + .iter_mut() + .find(|segment| segment.path == path || segment.source_path.as_deref() == Some(path)) + { + let mut changed = false; + if segment.name == "Image" { + segment.name = "Screenshot".to_string(); + changed = true; + } + if segment.source_path.is_none() + && segment.annotations.is_empty() + && !legacy_annotations.is_empty() + { + segment.source_path = Some(path.to_string()); + segment.annotations = legacy_annotations; + changed = true; + } + return changed; + } + timeline.image_segments.push(ImageSegment { + start: 0.0, + end: 5.0, + path: path.to_string(), + source_path: (!legacy_annotations.is_empty()).then(|| path.to_string()), + annotations: legacy_annotations, + name: "Screenshot".to_string(), + size: XY::new(1.0, 1.0), + ..Default::default() + }); + true +} + +pub fn media_canvas_size( + project_path: &Path, + project: &ProjectConfiguration, + still_image: Option<&str>, +) -> XY { + let image_path = still_image.or_else(|| { + project + .timeline + .as_ref()? + .image_segments + .first() + .map(|segment| segment.path.as_str()) + }); + if let Some(image_path) = image_path + && let Ok((width, height)) = image::image_dimensions(project_path.join(image_path)) + && width > 0 + && height > 0 + { + return XY::new(width, height); + } + if let Some(video) = project + .timeline + .as_ref() + .and_then(|timeline| timeline.video_segments.first()) + && let Ok((source, _)) = checked_project_video_source(project_path, &video.path) + && let Ok(input) = SegmentedInput::open_relocatable(&source, [Path::new(&video.path)]) + && let Ok(source) = Video::from_input(input.input(), 0.0) + { + return XY::new(source.width, source.height); + } + XY::new(1920, 1080) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_project::VideoSegment; + + #[test] + fn legacy_screenshot_annotations_transfer_only_once() { + let annotation = serde_json::from_value(serde_json::json!({ + "id": "legacy", "type": "rectangle", "x": 10.0, "y": 20.0, + "width": 30.0, "height": 40.0, "strokeColor": "#f05656", + "strokeWidth": 4.0, "fillColor": "transparent", "opacity": 1.0, + "rotation": 0.0 + })) + .unwrap(); + let mut project = ProjectConfiguration::default(); + project.annotations = vec![annotation]; + + assert!(add_still_image_to_timeline(&mut project, "original.png")); + let segment = &mut project.timeline.as_mut().unwrap().image_segments[0]; + assert_eq!(segment.name, "Screenshot"); + assert_eq!(segment.source_path.as_deref(), Some("original.png")); + assert_eq!(segment.annotations.len(), 1); + segment.annotations.clear(); + + assert!(!add_still_image_to_timeline(&mut project, "original.png")); + let segment = &mut project.timeline.as_mut().unwrap().image_segments[0]; + assert!(segment.annotations.is_empty()); + segment.path = "content/images/drawing.png".to_string(); + + assert!(!add_still_image_to_timeline(&mut project, "original.png")); + assert_eq!(project.timeline.unwrap().image_segments.len(), 1); + } + + #[test] + fn video_source_accepts_project_files_and_rejects_escape_paths() { + let root = tempfile::tempdir().unwrap(); + let project = root.path().join("project"); + std::fs::create_dir_all(project.join("content/videos")).unwrap(); + std::fs::write(project.join("content/videos/video.mp4"), b"video").unwrap(); + std::fs::write(root.path().join("outside.mp4"), b"outside").unwrap(); + + assert!(checked_project_video_source(&project, "content/videos/video.mp4").is_ok()); + assert!(checked_project_video_source(&project, "../outside.mp4").is_err()); + assert!( + checked_project_video_source( + &project, + root.path().join("outside.mp4").to_str().unwrap() + ) + .is_err() + ); + + #[cfg(unix)] + { + std::os::unix::fs::symlink( + root.path().join("outside.mp4"), + project.join("content/videos/linked.mp4"), + ) + .unwrap(); + assert!(checked_project_video_source(&project, "content/videos/linked.mp4").is_err()); + } + } + + #[test] + fn video_canvas_uses_imported_source_and_ignores_escape_path() { + let _ = ffmpeg::init(); + let root = tempfile::tempdir().unwrap(); + let project_path = root.path().join("project"); + std::fs::create_dir_all(project_path.join("content/videos")).unwrap(); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../video-decode/tests/fixtures/h264-decoder-lifecycle.mp4"); + let imported = project_path.join("content/videos/video.mp4"); + std::fs::copy(&fixture, &imported).unwrap(); + let expected = Video::new(&fixture, 0.0).unwrap(); + let mut project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + video_segments: vec![VideoSegment { + path: "content/videos/video.mp4".into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + assert_eq!( + media_canvas_size(&project_path, &project, None), + XY::new(expected.width, expected.height) + ); + project.timeline.as_mut().unwrap().video_segments[0].path = "../video.mp4".into(); + assert_eq!( + media_canvas_size(&project_path, &project, None), + XY::new(1920, 1080) + ); + } +} diff --git a/crates/rendering/src/overlay_layers.rs b/crates/rendering/src/overlay_layers.rs index 513de7ab66a..4bc6b76dd82 100644 --- a/crates/rendering/src/overlay_layers.rs +++ b/crates/rendering/src/overlay_layers.rs @@ -1,21 +1,32 @@ use crate::{ ProjectConfiguration, RenderingError, - layers::{CaptionsLayer, ImageLayer, KeyboardLayer, TextLayer}, + composite_frame::CompositeVideoFramePipeline, + layers::{CaptionsLayer, ImageLayer, KeyboardLayer, TextLayer, VideoLayer}, readiness, + yuv_converter::YuvConverterPipelines, }; +use std::sync::Arc; pub(super) struct OverlayLayers { pub(super) text: TextLayer, pub(super) images: ImageLayer, + pub(super) videos: VideoLayer, pub(super) captions: CaptionsLayer, pub(super) keyboard: KeyboardLayer, } impl OverlayLayers { - pub(super) fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Self { + pub(super) fn new( + device: &wgpu::Device, + queue: &wgpu::Queue, + yuv_pipelines: Arc, + composite_pipeline: Arc, + prefer_cpu_conversion: bool, + ) -> Self { Self { text: readiness::measure("layers.text", || TextLayer::new(device, queue)), images: readiness::measure("layers.images", || ImageLayer::new(device)), + videos: VideoLayer::new(yuv_pipelines, composite_pipeline, prefer_cpu_conversion), captions: readiness::measure("layers.captions", || CaptionsLayer::new(device, queue)), keyboard: readiness::measure("layers.keyboard", || KeyboardLayer::new(device, queue)), } @@ -34,6 +45,7 @@ impl OverlayLayers { || project.timeline.as_ref().is_some_and(|timeline| { !timeline.text_segments.is_empty() || !timeline.image_segments.is_empty() + || !timeline.video_segments.is_empty() || !timeline.caption_segments.is_empty() || !timeline.keyboard_segments.is_empty() }) diff --git a/crates/rendering/src/shaders/composite-video-frame.wgsl b/crates/rendering/src/shaders/composite-video-frame.wgsl index cf6b0c01eaa..a39847b6f0d 100644 --- a/crates/rendering/src/shaders/composite-video-frame.wgsl +++ b/crates/rendering/src/shaders/composite-video-frame.wgsl @@ -18,8 +18,8 @@ struct Uniforms { border_enabled: f32, border_width: f32, preserve_source_alpha: f32, - _padding1a: f32, - _padding1b: f32, + rotation: f32, + flip_y: f32, _padding1c: f32, border_color: vec4, // Per-corner multipliers on rounding_px: (tl, tr, bl, br). All 1s keeps @@ -234,8 +234,15 @@ fn fs_main(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { let p = frag_coord.xy; let center = (uniforms.target_bounds.xy + uniforms.target_bounds.zw) * 0.5; let size = (uniforms.target_bounds.zw - uniforms.target_bounds.xy) * 0.5; - - let dist = sdf_rounded_rect(p - center, size, corner_radius_for(p - center), uniforms.rounding_type); + let offset = p - center; + let cosine = cos(uniforms.rotation); + let sine = sin(uniforms.rotation); + let local = vec2( + cosine * offset.x + sine * offset.y, + -sine * offset.x + cosine * offset.y + ); + + let dist = sdf_rounded_rect(local, size, corner_radius_for(local), uniforms.rounding_type); let min_frame_size = min(size.x, size.y); let shadow_enabled = uniforms.shadow > 0.0; @@ -268,7 +275,11 @@ fn fs_main(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { let shadow_strength_final = smoothstep(shadow_size + shadow_blur, -shadow_blur, abs(shadow_dist)); let shadow_color = vec4(0.0, 0.0, 0.0, shadow_strength_final * shadow_opacity); - let target_uv = (p - uniforms.target_bounds.xy) / uniforms.target_size; + let target_uv = select( + (p - uniforms.target_bounds.xy) / uniforms.target_size, + (local + size) / uniforms.target_size, + uniforms.rotation != 0.0 + ); let crop_bounds_uv = vec4(uniforms.crop_bounds.xy / uniforms.frame_size, uniforms.crop_bounds.zw / uniforms.frame_size); let edge_padding = max(2.0, uniforms.border_width + 2.0); let edge_padding_uv = edge_padding / uniforms.target_size; @@ -289,17 +300,17 @@ fn fs_main(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { // Evaluate coverage before the apron return so fwidth retains every helper lane. let shape_coverage = rounded_rect_coverage( - p - center, + local, size, - corner_radius_for(p - center), + corner_radius_for(local), uniforms.rounding_type ); var border_coverage = 0.0; if (uniforms.border_enabled > 0.0) { let border_outer_coverage = rounded_rect_coverage( - p - center, + local, size + vec2(uniforms.border_width), - corner_radius_for(p - center) + uniforms.border_width, + corner_radius_for(local) + uniforms.border_width, uniforms.rounding_type ); border_coverage = clamp(border_outer_coverage - shape_coverage, 0.0, 1.0); @@ -425,6 +436,9 @@ fn sample_texture(uv: vec2, crop_bounds_uv: vec4) -> vec4 { if uniforms.mirror_x != 0.0 { sample_uv.x = 1.0 - sample_uv.x; } + if uniforms.flip_y != 0.0 { + sample_uv.y = 1.0 - sample_uv.y; + } let crop_size = crop_bounds_uv.zw - crop_bounds_uv.xy; var cropped_uv = sample_uv * crop_size + crop_bounds_uv.xy; diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 462c931d3d7..83062d038e9 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -1118,6 +1118,7 @@ mod tests { scene_segments: Vec::new(), style_segments, image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), @@ -2115,6 +2116,7 @@ mod tests { scene_segments: vec![], style_segments: vec![], image_segments: vec![], + video_segments: vec![], mask_segments: vec![], text_segments: vec![], caption_segments: vec![], @@ -2191,6 +2193,7 @@ mod tests { scene_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), + video_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), caption_segments: Vec::new(), diff --git a/crates/utils/src/export_resources.rs b/crates/utils/src/export_resources.rs index 810b1b506fc..9420c52fc66 100644 --- a/crates/utils/src/export_resources.rs +++ b/crates/utils/src/export_resources.rs @@ -13,7 +13,9 @@ const STOP_PREFIX: &str = "Export stopped to protect your computer: "; enum MemoryPressure { #[default] Unknown, + #[cfg(any(target_os = "macos", test))] Normal, + #[cfg(any(target_os = "macos", test))] Warning, Critical, } @@ -118,10 +120,14 @@ impl ExportResources { disk.description )); } - if matches!( + #[cfg(any(target_os = "macos", test))] + let memory_warning = matches!( sample.memory, MemoryPressure::Warning | MemoryPressure::Critical - ) { + ); + #[cfg(not(any(target_os = "macos", test)))] + let memory_warning = sample.memory == MemoryPressure::Critical; + if memory_warning { warnings.push("Your computer is already under memory pressure. Close other applications before exporting.".to_string()); } if warnings.is_empty() {