From 748bac3725b2f200ed82ec1b8f70b709bbb820c3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:13:56 +0100 Subject: [PATCH 01/47] feat: bring desktop editor to web --- .../workflows/docker-build-editor-worker.yml | 91 + Cargo.lock | 2 + Cargo.toml | 12 + apps/chrome-extension/e2e/overlay-ui.spec.ts | 464 +++-- .../e2e/recording-upload.spec.ts | 327 ++- .../chrome-extension/e2e/webcam-start.spec.ts | 43 + .../src/background/service-worker.ts | 152 +- .../src/content/countdown-overlay.tsx | 2 + apps/chrome-extension/src/content/overlay.tsx | 264 ++- .../src/content/recording-error-toast.css | 27 + .../src/offscreen/display-capture.test.ts | 15 + .../src/offscreen/display-capture.ts | 2 +- .../src/offscreen/recorder-start-sync.test.ts | 29 + .../src/offscreen/recorder-start-sync.ts | 30 + .../src/offscreen/recorder.ts | 537 ++++- .../src/preview/camera-preview.tsx | 81 +- .../src/shared/storage-bridge.test.ts | 160 ++ .../src/shared/storage-bridge.ts | 117 ++ apps/chrome-extension/src/shared/storage.ts | 120 +- apps/desktop/src-tauri/src/clip_thumbnails.rs | 231 +-- apps/desktop/src-tauri/src/import.rs | 971 +-------- apps/desktop/src-tauri/src/recording.rs | 10 +- .../desktop/src/routes/editor/CaptionsTab.tsx | 569 +++--- .../src/routes/editor/ConfigSidebar.tsx | 48 +- apps/desktop/src/routes/editor/ExportPage.tsx | 33 +- apps/desktop/src/routes/editor/Player.tsx | 3 + .../routes/editor/Timeline/TrackManager.tsx | 2 +- .../src/routes/editor/Timeline/index.tsx | 16 +- apps/desktop/src/routes/editor/context.ts | 30 + .../src/utils/organization-branding.ts | 110 + apps/media-server/Dockerfile.editor | 87 + .../Dockerfile.editor.dockerignore | 41 + apps/media-server/EDITOR_WORKER.md | 51 + apps/media-server/package.json | 6 + apps/media-server/railway.editor.toml | 9 + .../src/__tests__/editor-worker-app.test.ts | 77 + .../fixtures/editor-clips/camera-green.webm | Bin 0 -> 2180 bytes .../fixtures/editor-clips/clip-blue-audio.mp4 | Bin 0 -> 23322 bytes .../fixtures/editor-clips/display-red.webm | Bin 0 -> 24419 bytes .../__tests__/fixtures/exif-orientation-6.jpg | Bin 0 -> 825 bytes .../src/__tests__/fixtures/rgb-tiff.tiff | Bin 0 -> 630 bytes apps/media-server/src/__tests__/index.test.ts | 1 + .../src/__tests__/lib/editor-assets.test.ts | 106 + .../lib/editor-audio-library.test.ts | 49 + .../__tests__/lib/editor-cap-assets.test.ts | 38 + .../__tests__/lib/editor-cap-bundle.test.ts | 73 + .../lib/editor-clip-thumbnails.test.ts | 106 + .../lib/editor-command-socket.test.ts | 438 ++++ .../__tests__/lib/editor-config-paths.test.ts | 110 + .../__tests__/lib/editor-image-assets.test.ts | 179 ++ .../lib/editor-project-bundles.test.ts | 156 ++ .../__tests__/lib/editor-video-assets.test.ts | 94 + .../__tests__/lib/editor-wallpapers.test.ts | 70 + .../routes/editor-1080p-benchmark.ts | 1230 +++++++++++ .../routes/editor-camera-effects-benchmark.ts | 575 ++++++ .../routes/editor-cap-caption-benchmark.ts | 292 +++ .../routes/editor-cap-import-replay.ts | 886 ++++++++ .../routes/editor-clips-benchmark.ts | 264 +++ .../__tests__/routes/editor-clips-replay.ts | 4 + .../__tests__/routes/editor-clips-scenario.ts | 357 ++++ .../src/__tests__/routes/editor-clips.test.ts | 11 + .../routes/editor-legacy-replay.test.ts | 172 ++ .../routes/editor-project-bundle-replay.ts | 241 +++ .../routes/editor-solid-ui-replay.ts | 537 +++++ .../src/__tests__/routes/editor.test.ts | 1676 +++++++++++++++ apps/media-server/src/app.ts | 12 +- apps/media-server/src/editor-worker-app.ts | 34 + apps/media-server/src/editor-worker.ts | 32 + apps/media-server/src/index.ts | 13 +- apps/media-server/src/lib/editor-assets.ts | 109 + .../src/lib/editor-audio-library.ts | 135 ++ .../media-server/src/lib/editor-cap-assets.ts | 76 + .../media-server/src/lib/editor-cap-bundle.ts | 116 ++ .../src/lib/editor-cap-captions.ts | 385 ++++ .../src/lib/editor-cap-imports.ts | 186 ++ .../src/lib/editor-clip-thumbnails.ts | 60 + .../src/lib/editor-command-socket.ts | 277 +++ .../src/lib/editor-config-paths.ts | 163 ++ .../src/lib/editor-export-estimates.ts | 191 ++ .../src/lib/editor-export-previews.ts | 172 ++ apps/media-server/src/lib/editor-exports.ts | 450 ++++ .../src/lib/editor-image-assets.ts | 99 + apps/media-server/src/lib/editor-media.ts | 207 ++ apps/media-server/src/lib/editor-native.ts | 470 +++++ .../src/lib/editor-project-bundles.ts | 324 +++ apps/media-server/src/lib/editor-sessions.ts | 325 +++ .../src/lib/editor-signed-assets.ts | 94 + .../src/lib/editor-socket-tickets.ts | 121 ++ .../src/lib/editor-video-assets.ts | 140 ++ .../src/lib/editor-video-imports.ts | 112 + .../media-server/src/lib/editor-wallpapers.ts | 86 + apps/media-server/src/lib/editor-websocket.ts | 415 ++++ apps/media-server/src/routes/editor.ts | 964 +++++++++ apps/web/.gitignore | 1 + .../__tests__/editor-caption-access.test.ts | 84 + ...editor-caption-assemblyai-stream-replay.ts | 106 + .../__tests__/editor-caption-client.test.ts | 160 ++ apps/web/__tests__/editor-caption-job.test.ts | 212 ++ .../__tests__/editor-caption-native-replay.ts | 31 + .../__tests__/editor-caption-sources.test.ts | 308 +++ .../editor-caption-transport.test.ts | 94 + .../__tests__/editor-caption-workflow.test.ts | 792 ++++++++ apps/web/__tests__/editor-captions.test.ts | 84 + .../__tests__/editor-legacy-sources.test.ts | 210 ++ .../__tests__/editor-project-storage.test.ts | 67 + .../__tests__/editor-session-policy.test.ts | 36 + .../web/__tests__/editor-video-upload.test.ts | 94 + .../__tests__/editor-worker-routing.test.ts | 69 + .../__tests__/unit/editor-cap-bundle.test.ts | 107 + .../unit/editor-cap-import-client.test.ts | 75 + .../unit/editor-clip-recorder.test.ts | 249 +++ apps/web/__tests__/unit/editor-clips.test.ts | 133 ++ .../unit/editor-export-upload-client.test.ts | 277 +++ .../unit/editor-host-lifecycle.test.ts | 1286 ++++++++++++ .../web/__tests__/unit/editor-imports.test.ts | 107 + .../__tests__/unit/editor-local-draft.test.ts | 101 + .../unit/editor-recorded-clips.test.ts | 69 + .../unit/editor-studio-recovery.test.ts | 250 +++ .../unit/media-processing-workflows.test.ts | 98 +- .../__tests__/unit/multipart-presign.test.ts | 283 ++- .../unit/multipart-upload-utils.test.ts | 19 + .../trigger-instant-recording-processing.ts | 25 +- .../CameraPreviewWindow.tsx | 44 +- .../web-recorder-dialog/HowItWorksPanel.tsx | 4 +- .../InProgressRecordingBar.tsx | 33 +- .../web-recorder-dialog/recording-spool.md | 28 +- .../web-recorder-dialog/useWebRecorder.ts | 593 +++++- .../web-recorder-dialog.tsx | 11 + .../app/api/editor/preparations/[id]/route.ts | 129 ++ apps/web/app/api/editor/preparations/route.ts | 122 ++ .../api/editor/sessions/[id]/assets/route.ts | 438 ++++ .../editor/sessions/[id]/cap-imports/route.ts | 195 ++ .../editor/sessions/[id]/captions/route.ts | 356 ++++ .../api/editor/sessions/[id]/clips/route.ts | 289 +++ .../api/editor/sessions/[id]/config/route.ts | 205 ++ .../[id]/exports/[exportId]/chunk/route.ts | 112 + .../[exportId]/download-ticket/route.ts | 109 + .../sessions/[id]/exports/[exportId]/route.ts | 129 ++ .../api/editor/sessions/[id]/exports/route.ts | 112 + .../api/editor/sessions/[id]/file/route.ts | 114 ++ .../api/editor/sessions/[id]/meta/route.ts | 120 ++ .../api/editor/sessions/[id]/plan/route.ts | 48 + .../project-bundle/download-ticket/route.ts | 105 + .../web/app/api/editor/sessions/[id]/route.ts | 83 + .../sessions/[id]/share-status/route.ts | 99 + .../api/editor/sessions/[id]/tickets/route.ts | 123 ++ .../[id]/video-assets/[jobId]/route.ts | 6 + .../sessions/[id]/video-assets/abort/route.ts | 5 + .../[id]/video-assets/complete/route.ts | 6 + .../[id]/video-assets/presign-part/route.ts | 5 + .../sessions/[id]/video-assets/route.ts | 7 + .../api/upload/[...route]/multipart-utils.ts | 6 + .../app/api/upload/[...route]/multipart.ts | 254 ++- apps/web/app/s/[videoId]/edit/page.tsx | 32 +- .../edit/studio/EditorClipRecorder.tsx | 351 ++++ .../edit/studio/StudioEditorClient.tsx | 529 +++++ .../s/[videoId]/edit/studio/editor-host.ts | 1808 +++++++++++++++++ apps/web/app/s/[videoId]/edit/studio/page.tsx | 67 + apps/web/lib/editor-cap-import-client.ts | 265 +++ apps/web/lib/editor-caption-access.ts | 71 + apps/web/lib/editor-caption-client.ts | 189 ++ apps/web/lib/editor-caption-job.ts | 292 +++ apps/web/lib/editor-caption-sources.ts | 262 +++ apps/web/lib/editor-caption-transcripts.ts | 63 + apps/web/lib/editor-caption-transport.ts | 91 + apps/web/lib/editor-caption-workflow.ts | 468 +++++ apps/web/lib/editor-captions.ts | 155 ++ apps/web/lib/editor-clip-recorder.ts | 284 +++ apps/web/lib/editor-clips.ts | 115 ++ apps/web/lib/editor-export-upload-client.ts | 262 +++ apps/web/lib/editor-imports.ts | 126 ++ apps/web/lib/editor-local-draft.ts | 136 ++ apps/web/lib/editor-project-storage.ts | 62 + apps/web/lib/editor-recording-title.ts | 12 + apps/web/lib/editor-session.ts | 603 ++++++ apps/web/lib/editor-video-import-client.ts | 271 +++ apps/web/lib/editor-video-upload-api.ts | 753 +++++++ apps/web/lib/editor-video-upload.ts | 199 ++ apps/web/lib/editor-worker-routing.ts | 104 + apps/web/package.json | 7 +- apps/web/workflows/process-video.ts | 89 +- bun.lock | 45 +- crates/editor/Cargo.toml | 10 +- crates/editor/src/auto_zoom.rs | 226 +++ crates/editor/src/bin/web-editor-service.rs | 1095 ++++++++++ .../bin/web-editor-service/preview_h264.rs | 201 ++ crates/editor/src/clip_thumbnails_shared.rs | 225 ++ .../editor/src/existing_recording_import.rs | 1219 +++++++++++ crates/editor/src/lib.rs | 8 + .../editor/src/screen_recording_defaults.rs | 18 + crates/export/Cargo.toml | 3 + crates/export/src/bin/web-editor.rs | 755 +++++++ crates/project/Cargo.toml | 3 + crates/project/src/configuration.rs | 5 +- crates/rendering/Cargo.toml | 3 + crates/rendering/src/lib.rs | 151 +- .../cap-editor-probe/editor-free-captions.png | Bin 0 -> 96496 bytes .../cap-editor-probe/editor-pro-captions.png | Bin 0 -> 110908 bytes output/playwright/cap-editor-probe/editor.png | Bin 0 -> 59464 bytes packages/database/types/metadata.ts | 92 + packages/editor-cap-bundle/package.json | 8 + packages/editor-cap-bundle/src/index.ts | 186 ++ packages/editor-solid-web/index.html | 12 + packages/editor-solid-web/package.json | 26 + packages/editor-solid-web/postcss.config.cjs | 6 + .../src/audio-packets.test.ts | 32 + .../editor-solid-web/src/audio-packets.ts | 39 + packages/editor-solid-web/src/audio-player.ts | 71 + .../editor-solid-web/src/audio-processor.js | 56 + packages/editor-solid-web/src/channels.ts | 35 + .../src/editor-file-mapping.test.ts | 44 + .../src/editor-file-mapping.ts | 61 + packages/editor-solid-web/src/entry.tsx | 131 ++ .../editor-solid-web/src/tauri-bridge.test.ts | 294 +++ packages/editor-solid-web/src/tauri-bridge.ts | 378 ++++ .../editor-solid-web/src/tauri-clipboard.ts | 3 + packages/editor-solid-web/src/tauri-core.ts | 52 + .../editor-solid-web/src/tauri-dialog.test.ts | 92 + packages/editor-solid-web/src/tauri-dialog.ts | 207 ++ packages/editor-solid-web/src/tauri-event.ts | 47 + packages/editor-solid-web/src/tauri-fs.ts | 94 + packages/editor-solid-web/src/tauri-menu.ts | 169 ++ packages/editor-solid-web/src/tauri-opener.ts | 5 + packages/editor-solid-web/src/tauri-os.ts | 48 + packages/editor-solid-web/src/tauri-path.ts | 50 + packages/editor-solid-web/src/tauri-store.ts | 165 ++ .../editor-solid-web/src/tauri-window.test.ts | 66 + packages/editor-solid-web/src/tauri-window.ts | 113 ++ .../editor-solid-web/src/websocket.test.ts | 59 + packages/editor-solid-web/src/websocket.ts | 513 +++++ packages/editor-solid-web/tsconfig.json | 27 + packages/editor-solid-web/vite.config.ts | 100 + packages/env/server.ts | 9 + turbo.json | 9 + 234 files changed, 40505 insertions(+), 1946 deletions(-) create mode 100644 .github/workflows/docker-build-editor-worker.yml create mode 100644 apps/chrome-extension/src/content/recording-error-toast.css create mode 100644 apps/chrome-extension/src/offscreen/recorder-start-sync.test.ts create mode 100644 apps/chrome-extension/src/offscreen/recorder-start-sync.ts create mode 100644 apps/chrome-extension/src/shared/storage-bridge.test.ts create mode 100644 apps/chrome-extension/src/shared/storage-bridge.ts create mode 100644 apps/media-server/Dockerfile.editor create mode 100644 apps/media-server/Dockerfile.editor.dockerignore create mode 100644 apps/media-server/EDITOR_WORKER.md create mode 100644 apps/media-server/railway.editor.toml create mode 100644 apps/media-server/src/__tests__/editor-worker-app.test.ts create mode 100644 apps/media-server/src/__tests__/fixtures/editor-clips/camera-green.webm create mode 100644 apps/media-server/src/__tests__/fixtures/editor-clips/clip-blue-audio.mp4 create mode 100644 apps/media-server/src/__tests__/fixtures/editor-clips/display-red.webm create mode 100644 apps/media-server/src/__tests__/fixtures/exif-orientation-6.jpg create mode 100644 apps/media-server/src/__tests__/fixtures/rgb-tiff.tiff create mode 100644 apps/media-server/src/__tests__/lib/editor-assets.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-audio-library.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-cap-assets.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-cap-bundle.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-clip-thumbnails.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-command-socket.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-config-paths.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-image-assets.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-project-bundles.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-video-assets.test.ts create mode 100644 apps/media-server/src/__tests__/lib/editor-wallpapers.test.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-1080p-benchmark.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-camera-effects-benchmark.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-cap-caption-benchmark.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-cap-import-replay.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-clips-benchmark.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-clips-replay.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-clips-scenario.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-clips.test.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-legacy-replay.test.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-project-bundle-replay.ts create mode 100644 apps/media-server/src/__tests__/routes/editor-solid-ui-replay.ts create mode 100644 apps/media-server/src/__tests__/routes/editor.test.ts create mode 100644 apps/media-server/src/editor-worker-app.ts create mode 100644 apps/media-server/src/editor-worker.ts create mode 100644 apps/media-server/src/lib/editor-assets.ts create mode 100644 apps/media-server/src/lib/editor-audio-library.ts create mode 100644 apps/media-server/src/lib/editor-cap-assets.ts create mode 100644 apps/media-server/src/lib/editor-cap-bundle.ts create mode 100644 apps/media-server/src/lib/editor-cap-captions.ts create mode 100644 apps/media-server/src/lib/editor-cap-imports.ts create mode 100644 apps/media-server/src/lib/editor-clip-thumbnails.ts create mode 100644 apps/media-server/src/lib/editor-command-socket.ts create mode 100644 apps/media-server/src/lib/editor-config-paths.ts create mode 100644 apps/media-server/src/lib/editor-export-estimates.ts create mode 100644 apps/media-server/src/lib/editor-export-previews.ts create mode 100644 apps/media-server/src/lib/editor-exports.ts create mode 100644 apps/media-server/src/lib/editor-image-assets.ts create mode 100644 apps/media-server/src/lib/editor-media.ts create mode 100644 apps/media-server/src/lib/editor-native.ts create mode 100644 apps/media-server/src/lib/editor-project-bundles.ts create mode 100644 apps/media-server/src/lib/editor-sessions.ts create mode 100644 apps/media-server/src/lib/editor-signed-assets.ts create mode 100644 apps/media-server/src/lib/editor-socket-tickets.ts create mode 100644 apps/media-server/src/lib/editor-video-assets.ts create mode 100644 apps/media-server/src/lib/editor-video-imports.ts create mode 100644 apps/media-server/src/lib/editor-wallpapers.ts create mode 100644 apps/media-server/src/lib/editor-websocket.ts create mode 100644 apps/media-server/src/routes/editor.ts create mode 100644 apps/web/__tests__/editor-caption-access.test.ts create mode 100644 apps/web/__tests__/editor-caption-assemblyai-stream-replay.ts create mode 100644 apps/web/__tests__/editor-caption-client.test.ts create mode 100644 apps/web/__tests__/editor-caption-job.test.ts create mode 100644 apps/web/__tests__/editor-caption-native-replay.ts create mode 100644 apps/web/__tests__/editor-caption-sources.test.ts create mode 100644 apps/web/__tests__/editor-caption-transport.test.ts create mode 100644 apps/web/__tests__/editor-caption-workflow.test.ts create mode 100644 apps/web/__tests__/editor-captions.test.ts create mode 100644 apps/web/__tests__/editor-legacy-sources.test.ts create mode 100644 apps/web/__tests__/editor-project-storage.test.ts create mode 100644 apps/web/__tests__/editor-session-policy.test.ts create mode 100644 apps/web/__tests__/editor-video-upload.test.ts create mode 100644 apps/web/__tests__/editor-worker-routing.test.ts create mode 100644 apps/web/__tests__/unit/editor-cap-bundle.test.ts create mode 100644 apps/web/__tests__/unit/editor-cap-import-client.test.ts create mode 100644 apps/web/__tests__/unit/editor-clip-recorder.test.ts create mode 100644 apps/web/__tests__/unit/editor-clips.test.ts create mode 100644 apps/web/__tests__/unit/editor-export-upload-client.test.ts create mode 100644 apps/web/__tests__/unit/editor-host-lifecycle.test.ts create mode 100644 apps/web/__tests__/unit/editor-imports.test.ts create mode 100644 apps/web/__tests__/unit/editor-local-draft.test.ts create mode 100644 apps/web/__tests__/unit/editor-recorded-clips.test.ts create mode 100644 apps/web/__tests__/unit/editor-studio-recovery.test.ts create mode 100644 apps/web/app/api/editor/preparations/[id]/route.ts create mode 100644 apps/web/app/api/editor/preparations/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/assets/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/cap-imports/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/captions/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/clips/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/config/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/exports/[exportId]/chunk/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/exports/[exportId]/download-ticket/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/exports/[exportId]/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/exports/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/file/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/meta/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/plan/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/project-bundle/download-ticket/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/share-status/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/tickets/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/video-assets/[jobId]/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/video-assets/abort/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/video-assets/complete/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/video-assets/presign-part/route.ts create mode 100644 apps/web/app/api/editor/sessions/[id]/video-assets/route.ts create mode 100644 apps/web/app/s/[videoId]/edit/studio/EditorClipRecorder.tsx create mode 100644 apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx create mode 100644 apps/web/app/s/[videoId]/edit/studio/editor-host.ts create mode 100644 apps/web/app/s/[videoId]/edit/studio/page.tsx create mode 100644 apps/web/lib/editor-cap-import-client.ts create mode 100644 apps/web/lib/editor-caption-access.ts create mode 100644 apps/web/lib/editor-caption-client.ts create mode 100644 apps/web/lib/editor-caption-job.ts create mode 100644 apps/web/lib/editor-caption-sources.ts create mode 100644 apps/web/lib/editor-caption-transcripts.ts create mode 100644 apps/web/lib/editor-caption-transport.ts create mode 100644 apps/web/lib/editor-caption-workflow.ts create mode 100644 apps/web/lib/editor-captions.ts create mode 100644 apps/web/lib/editor-clip-recorder.ts create mode 100644 apps/web/lib/editor-clips.ts create mode 100644 apps/web/lib/editor-export-upload-client.ts create mode 100644 apps/web/lib/editor-imports.ts create mode 100644 apps/web/lib/editor-local-draft.ts create mode 100644 apps/web/lib/editor-project-storage.ts create mode 100644 apps/web/lib/editor-recording-title.ts create mode 100644 apps/web/lib/editor-session.ts create mode 100644 apps/web/lib/editor-video-import-client.ts create mode 100644 apps/web/lib/editor-video-upload-api.ts create mode 100644 apps/web/lib/editor-video-upload.ts create mode 100644 apps/web/lib/editor-worker-routing.ts create mode 100644 crates/editor/src/auto_zoom.rs create mode 100644 crates/editor/src/bin/web-editor-service.rs create mode 100644 crates/editor/src/bin/web-editor-service/preview_h264.rs create mode 100644 crates/editor/src/clip_thumbnails_shared.rs create mode 100644 crates/editor/src/existing_recording_import.rs create mode 100644 crates/editor/src/screen_recording_defaults.rs create mode 100644 crates/export/src/bin/web-editor.rs create mode 100644 output/playwright/cap-editor-probe/editor-free-captions.png create mode 100644 output/playwright/cap-editor-probe/editor-pro-captions.png create mode 100644 output/playwright/cap-editor-probe/editor.png create mode 100644 packages/editor-cap-bundle/package.json create mode 100644 packages/editor-cap-bundle/src/index.ts create mode 100644 packages/editor-solid-web/index.html create mode 100644 packages/editor-solid-web/package.json create mode 100644 packages/editor-solid-web/postcss.config.cjs create mode 100644 packages/editor-solid-web/src/audio-packets.test.ts create mode 100644 packages/editor-solid-web/src/audio-packets.ts create mode 100644 packages/editor-solid-web/src/audio-player.ts create mode 100644 packages/editor-solid-web/src/audio-processor.js create mode 100644 packages/editor-solid-web/src/channels.ts create mode 100644 packages/editor-solid-web/src/editor-file-mapping.test.ts create mode 100644 packages/editor-solid-web/src/editor-file-mapping.ts create mode 100644 packages/editor-solid-web/src/entry.tsx create mode 100644 packages/editor-solid-web/src/tauri-bridge.test.ts create mode 100644 packages/editor-solid-web/src/tauri-bridge.ts create mode 100644 packages/editor-solid-web/src/tauri-clipboard.ts create mode 100644 packages/editor-solid-web/src/tauri-core.ts create mode 100644 packages/editor-solid-web/src/tauri-dialog.test.ts create mode 100644 packages/editor-solid-web/src/tauri-dialog.ts create mode 100644 packages/editor-solid-web/src/tauri-event.ts create mode 100644 packages/editor-solid-web/src/tauri-fs.ts create mode 100644 packages/editor-solid-web/src/tauri-menu.ts create mode 100644 packages/editor-solid-web/src/tauri-opener.ts create mode 100644 packages/editor-solid-web/src/tauri-os.ts create mode 100644 packages/editor-solid-web/src/tauri-path.ts create mode 100644 packages/editor-solid-web/src/tauri-store.ts create mode 100644 packages/editor-solid-web/src/tauri-window.test.ts create mode 100644 packages/editor-solid-web/src/tauri-window.ts create mode 100644 packages/editor-solid-web/src/websocket.test.ts create mode 100644 packages/editor-solid-web/src/websocket.ts create mode 100644 packages/editor-solid-web/tsconfig.json create mode 100644 packages/editor-solid-web/vite.config.ts diff --git a/.github/workflows/docker-build-editor-worker.yml b/.github/workflows/docker-build-editor-worker.yml new file mode 100644 index 00000000000..b6885bb5507 --- /dev/null +++ b/.github/workflows/docker-build-editor-worker.yml @@ -0,0 +1,91 @@ +name: Docker Build Editor Worker + +on: + pull_request: + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + push: + branches: + - main + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + +permissions: {} + +jobs: + build: + name: Build and test editor worker (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: amd64 + runner: ubuntu-24.04 + - platform: arm64 + runner: ubuntu-24.04-arm + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + with: + context: . + file: apps/media-server/Dockerfile.editor + platforms: linux/${{ matrix.platform }} + load: true + tags: cap-editor-worker:verification + cache-from: type=gha,scope=editor-worker-${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=editor-worker-${{ matrix.platform }} + - name: Check native worker readiness + run: | + docker run --rm --network none \ + -e CAP_WEB_EDITOR_PUBLIC_ORIGIN=http://127.0.0.1:3457 \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification \ + -e 'import app from "./src/editor-worker-app.ts"; const response = await app.request("/health"); if (!response.ok) process.exit(1)' + - name: Exercise paired native editor routes + run: | + docker run --rm --network none --cpus 4 --memory 4g \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification test \ + src/__tests__/editor-worker-app.test.ts \ + src/__tests__/lib/editor-assets.test.ts \ + src/__tests__/lib/editor-clip-thumbnails.test.ts \ + src/__tests__/lib/editor-config-paths.test.ts \ + src/__tests__/lib/editor-image-assets.test.ts \ + src/__tests__/lib/editor-wallpapers.test.ts \ + src/__tests__/routes/editor-clips.test.ts \ + src/__tests__/routes/editor.test.ts diff --git a/Cargo.lock b/Cargo.lock index 4d08eb8ddc6..db5e3d86b58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1657,6 +1657,7 @@ dependencies = [ "futures", "image 0.25.8", "lru", + "relative-path", "ringbuf", "sentry", "serde", @@ -1667,6 +1668,7 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "uuid", "windows 0.60.0", "workspace-hack", ] diff --git a/Cargo.toml b/Cargo.toml index 8a9be0ffb64..e8d0146f07a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,18 @@ opt-level = 2 [profile.dev.package.cap-audio] opt-level = 2 +[profile.dev.package.cap-editor] +opt-level = 3 + +[profile.dev.package.image] +opt-level = 3 + +[profile.dev.package.png] +opt-level = 3 + +[profile.dev.package.flate2] +opt-level = 3 + # Optimize for smaller binary size [profile.release] panic = "unwind" diff --git a/apps/chrome-extension/e2e/overlay-ui.spec.ts b/apps/chrome-extension/e2e/overlay-ui.spec.ts index 307609c637a..c0e55a5846b 100644 --- a/apps/chrome-extension/e2e/overlay-ui.spec.ts +++ b/apps/chrome-extension/e2e/overlay-ui.spec.ts @@ -202,7 +202,7 @@ const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-close-e2e-")); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ `--disable-extensions-except=${extensionPath}`, `--load-extension=${extensionPath}`, @@ -379,9 +379,9 @@ const getClosedShadowNodeId = async ( for (let index = 0; index < attributes.length; index += 2) { if (attributes[index] !== attribute) continue; if (!classToken) return true; - return ( - attributes[index + 1]?.split(/\s+/).includes(classToken) === true - ); + return attribute === "class" + ? attributes[index + 1]?.split(/\s+/).includes(classToken) === true + : attributes[index + 1] === classToken; } return false; })?.nodeId ?? null @@ -484,7 +484,7 @@ test("idle bootstrap and tab activation never open the camera preview", async () } }); -test("dismissing or closing the panel also closes the idle camera preview", async () => { +test("closing extension UI removes the ready bar and camera preview", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer(); const extension = await launchExtensionContext(); @@ -502,30 +502,63 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn await targetPage.goto(`${mockServer.origin}/capture.html`); await targetPage.bringToFront(); - await sendServiceWorkerMessage(messengerPage, { + const openResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "open-recorder-panel", }); - + expect(openResponse).toMatchObject({ ok: true }); + const activeTabUrl = await worker.evaluate(async () => { + const tabs = await chrome.tabs.query({ + active: true, + currentWindow: true, + }); + return tabs[0]?.url; + }); + expect(activeTabUrl).toBe(targetPage.url()); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) + .poll(() => + worker.evaluate(async () => { + const items = await chrome.storage.session.get( + "cap-extension-shared-ui-state", + ); + const state = items["cap-extension-shared-ui-state"] as + | { panelOpen?: boolean } + | undefined; + return state?.panelOpen ?? false; + }), + ) .toBe(true); + + const devtools = await extension.context.newCDPSession(targetPage); + await expect + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - await targetPage.screenshot({ path: "test-results/close-ui-before.png" }); - await targetPage.mouse.click(400, 300); - + const closeResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); + expect(closeResponse).toMatchObject({ ok: true }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, @@ -537,36 +570,52 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn type: "open-recorder-panel", }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await panelFrame - .locator('button[aria-label="Close Cap and hide all recorder UI"]') - .click(); + const hideButton = await getClosedShadowElementBox( + devtools, + "aria-label", + "Hide recording bar", + ); + if (!hideButton) throw new Error("Hide recording bar button missing"); + await targetPage.mouse.click( + hideButton.x + hideButton.width / 2, + hideButton.y + hideButton.height / 2, + ); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); + expect(frameWithUrl(targetPage, "camera-preview.html")).not.toBeNull(); + await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - await targetPage.waitForTimeout(2_500); expect(frameWithUrl(targetPage, "camera-preview.html")).toBeNull(); - await targetPage.screenshot({ path: "test-results/close-ui-after.png" }); } finally { await extension.cleanup(); await mockServer.close(); @@ -594,13 +643,17 @@ test("an abandoned recording start clears without leaving controls behind", asyn target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); - const devtools = await extension.context.newCDPSession(targetPage); await writeStaleCreatingState(worker); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("creating"); await targetPage.waitForTimeout(500); @@ -608,7 +661,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); @@ -623,15 +676,19 @@ test("an abandoned recording start clears without leaving controls behind", asyn expect(statusResponse.status?.phase).toBe("idle"); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("idle"); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); expect( await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); } finally { @@ -640,7 +697,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn } }); -test("a failed recording start reopens the panel with the error", async () => { +test("a failed recording start shows the error on the recorded tab", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer({ failInstantRecordings: true }); const extension = await launchExtensionContext(); @@ -676,22 +733,32 @@ test("a failed recording start reopens the panel with the error", async () => { const statusResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "get-recording-status", - })) as { status?: { phase?: string } }; + })) as { status?: { phase?: string; message?: string } }; expect(statusResponse.status?.phase).toBe("error"); - // The panel should reopen in the page to show the failure. + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 10_000, - }) - .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await expect(panelFrame.getByText("Recording failed.")).toBeVisible({ - timeout: 10_000, - }); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-error-toast", + ), + ) + .not.toBeNull(); + const toastNodeId = await getClosedShadowNodeId( + devtools, + "class", + "cap-extension-recording-error-toast", + ); + if (!toastNodeId) throw new Error("recording error toast missing"); + const { outerHTML } = (await devtools.send("DOM.getOuterHTML", { + nodeId: toastNodeId, + })) as { outerHTML: string }; + expect(outerHTML).toContain("Recording failed."); + expect(outerHTML).toContain(statusResponse.status?.message); await targetPage.screenshot({ - path: "test-results/start-error-panel.png", + path: "test-results/start-error-toast.png", }); } finally { await extension.cleanup(); @@ -724,9 +791,7 @@ test("the countdown appears while recording setup is still pending", async () => type: "start-recording", mode: "fullscreen", }); - await expect - .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) - .toBe(true); + void startPromise.catch(() => undefined); const devtools = await extension.context.newCDPSession(targetPage); await expect @@ -737,9 +802,12 @@ test("the countdown appears while recording setup is still pending", async () => "class", "cap-extension-countdown", )) !== null, - { timeout: 1_000 }, + { timeout: 5_000 }, ) .toBe(true); + await expect + .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) + .toBe(true); await targetPage.screenshot({ path: "test-results/countdown-during-setup.png", }); @@ -869,11 +937,77 @@ test("recording controls stay stable and the camera resizes directly", async () target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + )) !== null, + { timeout: 15_000 }, + ) + .toBe(true); + await expect + .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + timeout: 10_000, }) .toBe(true); + const initialCameraBox = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + if (!initialCameraBox) throw new Error("camera preview missing"); + await targetPage.mouse.move( + initialCameraBox.x + initialCameraBox.width / 2, + initialCameraBox.y + initialCameraBox.height / 2, + ); + await expect + .poll(() => + getClosedShadowComputedStyle( + devtools, + "data-camera-resize-ne", + "opacity", + ), + ) + .toBe("1"); + const cameraResizeHandle = await getClosedShadowElementBox( + devtools, + "data-camera-resize-ne", + ); + if (!cameraResizeHandle) throw new Error("camera resize handle missing"); + await targetPage.screenshot({ + path: "test-results/camera-resize-handles.png", + }); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2, + cameraResizeHandle.y + cameraResizeHandle.height / 2, + ); + await targetPage.mouse.down(); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, + cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, + { steps: 8 }, + ); + await targetPage.mouse.up(); + await expect + .poll(async () => { + const resizedCamera = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + return ( + resizedCamera !== null && + resizedCamera.width - initialCameraBox.width > 50 && + resizedCamera.height - initialCameraBox.height > 50 + ); + }) + .toBe(true); + await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); + await targetPage.screenshot({ + path: "test-results/camera-resized.png", + }); const startResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", @@ -882,104 +1016,86 @@ test("recording controls stay stable and the camera resizes directly", async () })) as { ok: boolean }; expect(startResponse.ok).toBe(true); await expect - .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - const devtools = await extension.context.newCDPSession(targetPage); await expect .poll( async () => (await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", )) !== null, { timeout: 10_000 }, ) .toBe(true); + await expect + .poll(async () => { + const box = await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-active-recording-container", + ); + return box?.height ?? 0; + }) + .toBeGreaterThanOrEqual(47); const initialBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); if (!initialBox) throw new Error("recording bar missing"); expect(initialBox.x).toBeLessThanOrEqual(20); - expect(initialBox.width).toBeGreaterThan(100); - expect(initialBox.width).toBeLessThanOrEqual(128); - expect(initialBox.height).toBeLessThanOrEqual(44); - const handleBox = await getClosedShadowElementBox( - devtools, - "data-drag-handle", - ); - if (!handleBox) throw new Error("recording bar drag handle missing"); + expect(initialBox.width).toBeGreaterThan(70); + expect(initialBox.height).toBeGreaterThanOrEqual(47); const timeBox = await getClosedShadowElementBox( devtools, - "data-recording-time", + "class", + "cap-extension-recording-badge-time", ); - if (!timeBox || timeBox.width < 30) { + if (!timeBox || timeBox.width < 20) { throw new Error("recording time is not visible at rest"); } - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("0"); - await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) !== null, ) - .toBe("1"); - const actionsBox = await getClosedShadowElementBox( - devtools, - "data-recording-actions", - ); - if (!actionsBox) throw new Error("recording action capsule missing"); - const actionGap = actionsBox.x - (initialBox.x + initialBox.width); - expect(actionGap).toBeGreaterThanOrEqual(4); - expect(actionGap).toBeLessThanOrEqual(8); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "border-top-left-radius", - ), - ).toBe("14px"); + .toBe(true); const pauseBox = await getClosedShadowElementBox( devtools, - "data-recording-pause", + "aria-label", + "Pause recording", ); - if (!pauseBox) throw new Error("pause action missing on hover"); + if (!pauseBox) throw new Error("pause control missing on hover"); await targetPage.mouse.move( pauseBox.x + pauseBox.width / 2, pauseBox.y + pauseBox.height / 2, ); - await targetPage.waitForTimeout(300); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); await targetPage.screenshot({ path: "test-results/recording-bar-expanded.png", }); + const pauseClickBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Pause recording", + ); + if (!pauseClickBox) + throw new Error("pause control disappeared before click"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + pauseClickBox.x + pauseClickBox.width / 2, + pauseClickBox.y + pauseClickBox.height / 2, ); await expect .poll(async () => { @@ -990,16 +1106,25 @@ test("recording controls stay stable and the camera resizes directly", async () return response.status?.phase; }) .toBe("paused"); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); + await expect + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + )) !== null, + ) + .toBe(true); + const resumeBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + ); + if (!resumeBox) throw new Error("resume control missing"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + resumeBox.x + resumeBox.width / 2, + resumeBox.y + resumeBox.height / 2, ); await expect .poll(async () => { @@ -1011,13 +1136,13 @@ test("recording controls stay stable and the camera resizes directly", async () }) .toBe("recording"); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await targetPage.mouse.down(); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2 + 120, - handleBox.y + handleBox.height / 2 - 80, + initialBox.x + initialBox.width / 2 + 120, + initialBox.y + initialBox.height / 2 - 80, { steps: 8 }, ); await targetPage.mouse.up(); @@ -1027,7 +1152,7 @@ test("recording controls stay stable and the camera resizes directly", async () const movedBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); return ( movedBox !== null && @@ -1066,72 +1191,17 @@ test("recording controls stay stable and the camera resizes directly", async () await targetPage.mouse.move(760, 40); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ) - .toBe("0"); - await targetPage.waitForTimeout(500); - await targetPage.screenshot({ - path: "test-results/recording-bar-visible.png", - }); - - const initialCameraBox = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - if (!initialCameraBox) throw new Error("camera preview missing"); - await targetPage.mouse.move( - initialCameraBox.x + initialCameraBox.width / 2, - initialCameraBox.y + initialCameraBox.height / 2, - ); - await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-camera-resize-ne", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) === null, ) - .toBe("1"); - const cameraResizeHandle = await getClosedShadowElementBox( - devtools, - "data-camera-resize-ne", - ); - if (!cameraResizeHandle) throw new Error("camera resize handle missing"); - await targetPage.screenshot({ - path: "test-results/camera-resize-handles.png", - }); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2, - cameraResizeHandle.y + cameraResizeHandle.height / 2, - ); - await targetPage.mouse.down(); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, - cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, - { steps: 8 }, - ); - await targetPage.mouse.up(); - await expect - .poll(async () => { - const resizedCamera = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - return ( - resizedCamera !== null && - resizedCamera.width - initialCameraBox.width > 50 && - resizedCamera.height - initialCameraBox.height > 50 - ); - }) .toBe(true); - await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); await targetPage.screenshot({ - path: "test-results/camera-resized.png", + path: "test-results/recording-bar-visible.png", }); const stopResponse = (await sendServiceWorkerMessage(messengerPage, { diff --git a/apps/chrome-extension/e2e/recording-upload.spec.ts b/apps/chrome-extension/e2e/recording-upload.spec.ts index fcf73c51481..14d342cbaf3 100644 --- a/apps/chrome-extension/e2e/recording-upload.spec.ts +++ b/apps/chrome-extension/e2e/recording-upload.spec.ts @@ -1,4 +1,5 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, @@ -32,6 +33,7 @@ type MockState = { initiateBodies: unknown[]; presignBodies: unknown[]; uploadBytes: number[]; + uploadBytesBySubpath: Record; uploadHeaders: Record[]; videoId: string; }; @@ -55,7 +57,9 @@ type ChromeGlobal = typeof globalThis & { }; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); +const extensionPath = process.env.CAP_EXTENSION_E2E_DIR + ? path.resolve(process.env.CAP_EXTENSION_E2E_DIR) + : path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -157,6 +161,7 @@ const createMockCapServer = async () => { initiateBodies: [], presignBodies: [], uploadBytes: [], + uploadBytesBySubpath: {}, uploadHeaders: [], videoId: `e2e-${Date.now()}`, }; @@ -217,7 +222,7 @@ const createMockCapServer = async () => { ) { state.initiateBodies.push(await parseJsonBody(request)); sendJson(response, 200, { - uploadId: "upload-e2e", + uploadId: `upload-e2e-${state.initiateBodies.length}`, provider: "s3", }); return; @@ -237,7 +242,7 @@ const createMockCapServer = async () => { ? body.partNumber : state.presignBodies.length; sendJson(response, 200, { - presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}`, + presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}?subpath=${encodeURIComponent(String(body && typeof body === "object" && "subpath" in body ? body.subpath : "unknown"))}`, provider: "s3", }); return; @@ -246,6 +251,9 @@ const createMockCapServer = async () => { if (request.method === "PUT" && url.pathname.startsWith("/mock-s3/")) { const body = await readRequestBody(request); state.uploadBytes.push(body.byteLength); + const subpath = url.searchParams.get("subpath") ?? "unknown"; + state.uploadBytesBySubpath[subpath] = + (state.uploadBytesBySubpath[subpath] ?? 0) + body.byteLength; state.uploadHeaders.push(request.headers); response.writeHead(200, { "Access-Control-Allow-Origin": "*", @@ -260,10 +268,15 @@ const createMockCapServer = async () => { request.method === "POST" && url.pathname === "/api/upload/multipart/complete" ) { - state.completeBodies.push(await parseJsonBody(request)); + const body = await parseJsonBody(request); + state.completeBodies.push(body); sendJson(response, 200, { success: true, - processingStarted: true, + processingStarted: + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "raw-upload.webm", }); return; } @@ -333,19 +346,34 @@ const createMockCapServer = async () => { const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-extension-e2e-")); + const testExtensionPath = path.join(userDataDir, "extension"); + await cp(extensionPath, testExtensionPath, { recursive: true }); + const manifestPath = path.join(testExtensionPath, "manifest.json"); + const manifest: Record = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.commands = { + _execute_action: { + suggested_key: { + default: "Ctrl+Shift+Y", + mac: "Command+Shift+Y", + }, + }, + }; + await writeFile(manifestPath, JSON.stringify(manifest)); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, + `--disable-extensions-except=${testExtensionPath}`, + `--load-extension=${testExtensionPath}`, "--allow-http-screen-capture", "--auto-select-desktop-capture-source=Cap E2E Capture Target", "--auto-select-tab-capture-source-by-title=Cap E2E Capture Target", "--enable-usermedia-screen-capturing", "--autoplay-policy=no-user-gesture-required", "--use-fake-device-for-media-stream", - "--use-fake-ui-for-media-stream", + "--auto-accept-camera-and-microphone-capture", ], }); const cleanup = async () => { @@ -370,6 +398,48 @@ const getServiceWorker = async (context: BrowserContext) => { const getExtensionId = (worker: Awaited>) => new URL(worker.url()).host; +type DevToolsNode = { + nodeId: number; + attributes?: string[]; + children?: DevToolsNode[]; + shadowRoots?: DevToolsNode[]; +}; + +const findCameraPreview = (node: DevToolsNode): DevToolsNode | null => { + if (node.attributes?.includes("data-camera-preview")) return node; + for (const child of [...(node.children ?? []), ...(node.shadowRoots ?? [])]) { + const result = findCameraPreview(child); + if (result) return result; + } + return null; +}; + +const expectCameraPreviewOutOfCapture = async (page: Page) => { + await expect(page.locator("#cap-extension-recorder-overlay")).toHaveAttribute( + "data-cap-mounted", + "true", + ); + const session = await page.context().newCDPSession(page); + try { + await session.send("DOM.enable"); + await expect + .poll(async () => { + const document = (await session.send("DOM.getDocument", { + depth: -1, + pierce: true, + })) as { root: DevToolsNode }; + const node = findCameraPreview(document.root); + if (!node?.attributes) return true; + const styleIndex = node.attributes.indexOf("style"); + if (styleIndex < 0) return false; + return /visibility:\s*hidden/.test(node.attributes[styleIndex + 1]); + }) + .toBe(true); + } finally { + await session.detach(); + } +}; + const openExtensionMessengerPage = async ( context: BrowserContext, worker: Awaited>, @@ -434,8 +504,8 @@ const configureExtension = async ( microphone: null, }, webcam: { - enabled: false, - deviceId: null, + enabled: true, + deviceId: "__cap_default_camera__", position: "bottom-left", size: 230, shape: "round", @@ -511,20 +581,25 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { }) .toBe("completed"); - expect(state.initiateBodies).toHaveLength(1); + expect(state.initiateBodies).toHaveLength(2); + expect(state.initiateBodies[0]).toMatchObject({ subpath: "raw-upload.webm" }); + expect(state.initiateBodies[1]).toMatchObject({ + subpath: "camera-upload.webm", + }); expect(state.presignBodies.length).toBeGreaterThanOrEqual(1); expect(state.uploadBytes.length).toBeGreaterThanOrEqual(1); expect( state.uploadBytes.reduce((total, bytes) => total + bytes, 0), ).toBeGreaterThan(0); - expect(state.completeBodies).toHaveLength(1); + expect(state.completeBodies).toHaveLength(2); expect(state.progressBodies.length).toBeGreaterThanOrEqual(1); const completeBody = state.completeBodies[0]; expect(completeBody).toMatchObject({ videoId: state.videoId, - uploadId: "upload-e2e", - subpath: "raw-upload.webm", + uploadId: "upload-e2e-2", + subpath: "camera-upload.webm", + screenSubpath: "raw-upload.webm", }); expect( completeBody && @@ -534,26 +609,148 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { ? completeBody.parts.length : 0, ).toBeGreaterThanOrEqual(1); + const cameraOffsetMs = + completeBody && + typeof completeBody === "object" && + "cameraOffsetMs" in completeBody + ? completeBody.cameraOffsetMs + : null; + expect( + typeof cameraOffsetMs === "number" && Number.isFinite(cameraOffsetMs), + ).toBe(true); + expect(state.completeBodies[1]).toMatchObject({ + subpath: "raw-upload.webm", + uploadId: "upload-e2e-1", + }); + expect(state.uploadBytesBySubpath["camera-upload.webm"]).toBeGreaterThan(0); + expect(state.uploadBytesBySubpath["raw-upload.webm"]).toBeGreaterThan(0); }; +const readLiveCameraOffset = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + if (!Array.isArray(manifests)) { + throw new Error(`Live manifest storage is ${typeof manifests}`); + } + const offset = (manifests[0] as { cameraOffsetMs?: unknown } | undefined) + ?.cameraOffsetMs; + if (typeof offset !== "number") { + throw new Error( + `Live manifests: ${manifests.length}; first camera offset: ${typeof offset}`, + ); + } + return offset; + }); + +const readFailedCameraRecovery = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-failed-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const recordings = items[key]; + if (!Array.isArray(recordings)) return null; + const failed = recordings[0] as + | { + videoId?: unknown; + cameraSessionId?: unknown; + cameraSubpath?: unknown; + cameraOffsetMs?: unknown; + } + | undefined; + return failed + ? { + videoId: failed.videoId, + cameraSessionId: failed.cameraSessionId, + cameraSubpath: failed.cameraSubpath, + cameraOffsetMs: failed.cameraOffsetMs, + } + : null; + }); + +const hasLiveRecordingManifest = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + return Array.isArray(items[key]) && items[key].length > 0; + }); + const startRecording = async ( context: BrowserContext, worker: Awaited>, apiBaseUrl: string, + mode: "fullscreen" | "tab" = RECORDING_MODE, ) => { await configureExtension(worker, apiBaseUrl); const messengerPage = await openExtensionMessengerPage(context, worker); const capturePage = await context.newPage(); await capturePage.goto(`${apiBaseUrl}/capture.html`); await capturePage.bringToFront(); + await expect + .poll(async () => { + const tabs = await worker.evaluate(() => + chrome.tabs.query({ active: true, lastFocusedWindow: true }), + ); + return tabs[0]?.url; + }) + .toBe(`${apiBaseUrl}/capture.html`); + if (mode === "tab") { + await worker.evaluate(() => { + const target = globalThis as typeof globalThis & { + capE2eActionClicks?: number; + }; + target.capE2eActionClicks = 0; + chrome.action.onClicked.addListener(() => { + target.capE2eActionClicks = (target.capE2eActionClicks ?? 0) + 1; + }); + }); + } + if (mode === "tab") { + const injected = spawnSync("osascript", [ + "-e", + 'tell application "System Events" to keystroke "y" using {command down, shift down}', + ]); + if (injected.status !== 0) { + throw new Error(injected.stderr.toString()); + } + await expect + .poll(() => + worker.evaluate( + () => + (globalThis as typeof globalThis & { capE2eActionClicks?: number }) + .capE2eActionClicks ?? 0, + ), + ) + .toBeGreaterThan(0); + } const startResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "start-recording", - mode: RECORDING_MODE, + mode, }); if (!startResponse.ok) { - throw new Error(startResponse.error); + const captures = await worker.evaluate( + () => + new Promise((resolve) => + chrome.tabCapture.getCapturedTabs(resolve), + ), + ); + throw new Error( + `${startResponse.error}; captured tabs: ${JSON.stringify(captures)}`, + ); } await expect @@ -589,15 +786,18 @@ test.describe("extension recording upload", () => { await mockServer?.close(); }); - test("records the selected display surface, uploads non-empty multipart data, and completes", async () => { + test("records a separate camera sidecar and hides its page overlay during display capture", async () => { if (!extension || !mockServer) throw new Error("Test harness did not start"); const worker = await getServiceWorker(extension.context); - const { messengerPage } = await startRecording( + const { capturePage, messengerPage } = await startRecording( extension.context, worker, mockServer.origin, ); + await expectCameraPreviewOutOfCapture(capturePage); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + expect(typeof persistedCameraOffsetMs).toBe("number"); const stopResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", @@ -606,6 +806,92 @@ test.describe("extension recording upload", () => { expect(stopResponse).toMatchObject({ ok: true }); await expectSuccessfulUpload(messengerPage, mockServer.state); + expect(mockServer.state.completeBodies[0]).toMatchObject({ + cameraOffsetMs: persistedCameraOffsetMs, + }); + }); + + test("recovers both camera and screen spools after the offscreen recorder closes", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + await worker.evaluate(() => chrome.offscreen.closeDocument()); + + const refreshResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-media-devices", + }); + expect(refreshResponse).toMatchObject({ ok: true }); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ + videoId: mockServer.state.videoId, + cameraSubpath: "camera-upload.webm", + cameraOffsetMs: persistedCameraOffsetMs, + }); + + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subpath: "camera-upload.webm", + cameraOffsetMs: persistedCameraOffsetMs, + }), + expect.objectContaining({ subpath: "raw-upload.webm" }), + ]), + ); + expect( + mockServer.state.uploadBytesBySubpath["camera-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["raw-upload.webm"], + ).toBeGreaterThan(0); + expect(await readFailedCameraRecovery(worker)).toBeNull(); + expect(await hasLiveRecordingManifest(worker)).toBe(false); + }); + + test("keeps camera sidecar separate in current-tab capture", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Tab capture needs a native Chrome action gesture in headed macOS Chromium", + ); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + ); + await expectCameraPreviewOutOfCapture(capturePage); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); }); test("can complete two consecutive recording uploads without stale state", async () => { @@ -638,6 +924,7 @@ test.describe("extension recording upload", () => { mockServer.state.initiateBodies = []; mockServer.state.presignBodies = []; mockServer.state.uploadBytes = []; + mockServer.state.uploadBytesBySubpath = {}; mockServer.state.uploadHeaders = []; mockServer.state.videoId = `e2e-${Date.now()}-second`; diff --git a/apps/chrome-extension/e2e/webcam-start.spec.ts b/apps/chrome-extension/e2e/webcam-start.spec.ts index dc1344b00f4..c66ade47f3d 100644 --- a/apps/chrome-extension/e2e/webcam-start.spec.ts +++ b/apps/chrome-extension/e2e/webcam-start.spec.ts @@ -301,9 +301,45 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "bootstrap", }); logs.push(`bootstrap: ${JSON.stringify(bootstrapResponse).slice(0, 200)}`); + const openResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + logs.push(`open panel: ${JSON.stringify(openResponse)}`); + await expect + .poll(() => + targetPage + .frames() + .some((frame) => frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Give the camera preview time to go live (frames flowing over WebRTC). await targetPage.waitForTimeout(6_000); + const previewFrame = targetPage + .frames() + .find((frame) => frame.url().includes("camera-preview.html")); + if (!previewFrame) throw new Error("Camera preview frame was not mounted"); + await expect + .poll(() => + previewFrame.evaluate( + () => document.querySelector("video")?.videoWidth ?? 0, + ), + ) + .toBeGreaterThan(0); + const pictureInPictureSupported = await previewFrame.evaluate( + () => document.pictureInPictureEnabled, + ); + if (pictureInPictureSupported) { + await previewFrame.locator("[data-pip-control]").click(); + await expect + .poll(() => + previewFrame.evaluate( + () => document.pictureInPictureElement !== null, + ), + ) + .toBe(true); + } await targetPage.screenshot({ path: "test-results/repro-before-start.png", }); @@ -342,6 +378,13 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "get-recording-status", })) as { status?: { phase?: string } }; expect(finalStatus.status?.phase).toBe("recording"); + await expect + .poll(() => + targetPage + .frames() + .every((frame) => !frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Content scripts read chrome.storage.session; without the service // worker widening the access level every call fails with this error. diff --git a/apps/chrome-extension/src/background/service-worker.ts b/apps/chrome-extension/src/background/service-worker.ts index 166072e3050..6ed3411f934 100644 --- a/apps/chrome-extension/src/background/service-worker.ts +++ b/apps/chrome-extension/src/background/service-worker.ts @@ -36,6 +36,11 @@ import { saveWebcamPreviewDismissed, updateSharedUiState, } from "../shared/storage"; +import { + executeStorageBridgeRequest, + isStorageBridgeRequest, + isTrustedOffscreenStorageSender, +} from "../shared/storage-bridge"; import type { BootstrapData, CameraDevice, @@ -83,7 +88,6 @@ let activePreviewTabId: number | null = null; let pendingPreviewTabId: number | null = null; let offscreenDocumentCreation: Promise | null = null; let browserWindowFocused = true; -let externalCaptureAutoPipPending = false; let recordingStartInFlight: Promise | null = null; // Content scripts read the webcam "dismissed" flag and the cached preview @@ -314,10 +318,22 @@ const getTabStreamId = (tabId: number) => }); }); +const canUseDisplayPickerForTabCapture = (error: unknown) => + error instanceof Error && + /has not been invoked|activeTab permission/i.test(error.message); + const sendOverlayMessage = (tabId: number, message: OverlayMessage) => new Promise((resolve) => { - chrome.tabs.sendMessage(tabId, message, () => { - resolve(!chrome.runtime.lastError); + chrome.tabs.sendMessage(tabId, message, (response: unknown) => { + resolve( + !chrome.runtime.lastError && + !( + response && + typeof response === "object" && + "ok" in response && + response.ok === false + ), + ); }); }); @@ -485,21 +501,6 @@ const isBrowserCaptureSource = (source: RecordingCaptureSource) => source.displaySurface === "browser" || source.displaySurface === "tab"; -const isWindowCaptureSource = (source: RecordingCaptureSource) => - source.detectedMode === "window" || - source.displaySurface === "window" || - source.displaySurface === "application"; - -const isLikelyBrowserWindow = (source: RecordingCaptureSource) => { - if (!source.label) return false; - return /\b(google chrome|chrome|chromium|microsoft edge|edge|brave|arc|opera|vivaldi)\b/i.test( - source.label, - ); -}; - -const shouldAutoPipCaptureSource = (source: RecordingCaptureSource) => - isWindowCaptureSource(source) && !isLikelyBrowserWindow(source); - const isWebcamPreviewEnabled = (settings: ExtensionSettings) => Boolean(settings.webcam.enabled); @@ -777,6 +778,7 @@ const getPreviewTabIdForPip = async () => { }; const enterActivePreviewAutoPip = async () => { + if (isRecordingPreviewStatus(recordingStatus)) return false; const tabId = await getPreviewTabIdForPip(); if (tabId === null) return false; return sendOverlay(tabId, { type: "overlay-enter-auto-pip" }, false).catch( @@ -827,15 +829,11 @@ const broadcastOverlayCountdown = async ( if (!canInjectIntoTab(tab) || tab.id === undefined) { return undefined; } - return sendOverlay( - tab.id, - { - type: "overlay-countdown", - seconds, - durationMs, - }, - false, - ).catch(() => undefined); + return sendOverlay(tab.id, { + type: "overlay-countdown", + seconds, + durationMs, + }).catch(() => undefined); }), ); }; @@ -1182,7 +1180,6 @@ const resolveMicWarning = async ( const performRecordingStart = async (mode: RecordingMode) => { const { settings, auth, bootstrap } = await requireSignedInState(); - externalCaptureAutoPipPending = false; const recordingSettings = settings.capture.recordingMode === mode ? settings @@ -1206,7 +1203,6 @@ const performRecordingStart = async (mode: RecordingMode) => { if (micWarning) { const confirmed = await requestRecordingConfirmation(tabId, micWarning); if (!confirmed) { - externalCaptureAutoPipPending = false; return { ok: false, canceled: true, @@ -1215,20 +1211,61 @@ const performRecordingStart = async (mode: RecordingMode) => { } } - const tabStreamId = - mode === "tab" && tabId !== undefined - ? await getTabStreamId(tabId) - : undefined; - await showOverlayInTab(tab ?? null, recordingSettings, true); + if (activePreviewTabId !== null && activePreviewTabId !== tabId) { + const previousPreviewTab = await getTab(activePreviewTabId); + if (previousPreviewTab && canInjectIntoTab(previousPreviewTab)) { + const previousPrepared = await sendOverlay( + activePreviewTabId, + { + type: "overlay-settings", + settings: recordingSettings.webcam, + recording: true, + }, + false, + ); + if (!previousPrepared) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } + } + } + const previewPrepared = await showOverlayInTab( + tab ?? null, + recordingSettings, + true, + ); + if ( + isWebcamPreviewEnabled(recordingSettings) && + tab && + canInjectIntoTab(tab) && + !previewPrepared + ) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } const creatingStatus = { phase: "creating" } satisfies RecordingStatus; setRecordingStatusAndBroadcast(creatingStatus); + let tabStreamId: string | undefined; + if (mode === "tab" && tabId !== undefined) { + await ensureOffscreenDocument(); + try { + tabStreamId = await getTabStreamId(tabId); + } catch (error) { + console.warn("Tab capture stream ID unavailable", error); + if (!canUseDisplayPickerForTabCapture(error)) throw error; + } + } // The manifest injects the bootstrap content script into every page at // document_idle and onInstalled covers tabs that predate the extension, so // no blanket re-injection is needed here; sendOverlay still injects // per-tab on demand and the bootstrap lazy-loads the overlay UI. try { - return await sendOffscreen({ + const request: OffscreenRequest = { target: "offscreen", type: "start-recording", mode, @@ -1237,7 +1274,10 @@ const performRecordingStart = async (mode: RecordingMode) => { bootstrap, tabId, tabStreamId, - }); + }; + return await (tabStreamId + ? sendOffscreenRuntimeMessage(request) + : sendOffscreen(request)); } catch (error) { // The recorder panel closes as soon as the status leaves "idle", so a // silent reset would leave the user with no feedback at all. Broadcast @@ -1246,7 +1286,6 @@ const performRecordingStart = async (mode: RecordingMode) => { phase: "error", message: error instanceof Error ? error.message : String(error), }); - externalCaptureAutoPipPending = false; throw error; } }; @@ -1317,7 +1356,6 @@ const stopRecordingAndOpenDestination = async () => { if (response.ok && response.status) { setRecordingStatus(response.status); if (!isCapturingRecordingStatus(response.status)) { - externalCaptureAutoPipPending = false; await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); } @@ -1431,7 +1469,7 @@ const handlePreviewReady = async (tabId?: number) => { activePreviewTabId = tabId; pendingPreviewTabId = null; await hidePreviewTabsExcept(tabId); - if (!browserWindowFocused || externalCaptureAutoPipPending) { + if (!browserWindowFocused) { await enterActivePreviewAutoPip(); } }; @@ -1458,7 +1496,6 @@ const handlePreviewError = async ( const handleCaptureSource = async (source: RecordingCaptureSource) => { if (isBrowserCaptureSource(source)) { - externalCaptureAutoPipPending = false; const tabId = await findCapturedBrowserTabId(source); if (tabId !== null) { await focusTab(tabId); @@ -1467,14 +1504,6 @@ const handleCaptureSource = async (source: RecordingCaptureSource) => { await exitActivePreviewAutoPip(); return; } - - if (!shouldAutoPipCaptureSource(source)) { - externalCaptureAutoPipPending = false; - return; - } - - externalCaptureAutoPipPending = true; - await enterActivePreviewAutoPip(); }; const handleRequest = async ( @@ -1791,12 +1820,32 @@ const handleRequest = async ( return { ok: false, error: "Unknown request" }; }; -chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if (isStorageBridgeRequest(message)) { + if ( + !isTrustedOffscreenStorageSender( + sender, + chrome.runtime.id, + chrome.runtime.getURL(OFFSCREEN_URL), + ) + ) { + sendResponse({ ok: false, error: "Storage request denied" }); + return false; + } + executeStorageBridgeRequest(message) + .then(sendResponse) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } if (isRecordingStatusBroadcast(message)) { setRecordingStatusAndBroadcast(message.status); if (isCapturingRecordingStatus(message.status)) return false; - externalCaptureAutoPipPending = false; void (async () => { await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); @@ -1813,7 +1862,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { if (!isServiceWorkerRequest(message)) return false; - handleRequest(message, _sender) + handleRequest(message, sender) .then(sendResponse) .catch((error: unknown) => { sendResponse({ @@ -1858,7 +1907,6 @@ chrome.windows.onFocusChanged.addListener((windowId) => { return; } browserWindowFocused = true; - externalCaptureAutoPipPending = false; void exitActivePreviewAutoPip() .then(() => syncActivePreview()) .catch(() => undefined); diff --git a/apps/chrome-extension/src/content/countdown-overlay.tsx b/apps/chrome-extension/src/content/countdown-overlay.tsx index 11cd3bb5180..7c3fc711c35 100644 --- a/apps/chrome-extension/src/content/countdown-overlay.tsx +++ b/apps/chrome-extension/src/content/countdown-overlay.tsx @@ -1,6 +1,7 @@ import { type CSSProperties, useCallback, useEffect, useState } from "react"; import { isOverlayMessage } from "../shared/messages"; import { sendServiceWorkerMessage } from "../shared/runtime"; +import { replayStartupMessages } from "./startup-messages"; type ActiveCountdown = { seconds: number; @@ -69,6 +70,7 @@ export function CountdownOverlay() { }; chrome.runtime.onMessage.addListener(handleMessage); + replayStartupMessages(handleMessage); return () => chrome.runtime.onMessage.removeListener(handleMessage); }, [dismiss]); diff --git a/apps/chrome-extension/src/content/overlay.tsx b/apps/chrome-extension/src/content/overlay.tsx index f7f2946cef4..335db134cf6 100644 --- a/apps/chrome-extension/src/content/overlay.tsx +++ b/apps/chrome-extension/src/content/overlay.tsx @@ -17,6 +17,7 @@ import { loadSharedUiState, loadWebcamPreviewDismissed, OVERLAY_UI_STATE_KEY, + RECORDING_STATE_KEY, SETTINGS_KEY, SHARED_UI_STATE_KEY, saveLastWebcamPreviewFrame, @@ -30,6 +31,7 @@ import type { ExtensionSettings, OverlayPosition, RecordingStatus, + SharedRecordingState, WebcamPosition, WebcamPreviewFrame, WebcamSettings, @@ -43,6 +45,7 @@ import { ConfirmOverlay } from "./confirm-overlay"; import { CountdownOverlay } from "./countdown-overlay"; import overlayCss from "./overlay.css?inline"; import { RecordingBarOverlay } from "./recording-bar"; +import recordingErrorCss from "./recording-error-toast.css?inline"; import { replayStartupMessages, setStartupMessages } from "./startup-messages"; const ROOT_ID = "cap-extension-recorder-overlay"; @@ -157,6 +160,11 @@ type PreviewParentMessage = source: "cap-extension-overlay"; token: string; type: "stop"; + } + | { + source: "cap-extension-overlay"; + token: string; + type: "stop-for-capture"; }; const classNames = (...values: Array) => @@ -339,6 +347,7 @@ function OverlayApp() { const [previewPointerInside, setPreviewPointerInside] = useState(false); const [recordingPreviewActive, setRecordingPreviewActive] = useState(false); const [recorderPanelOpen, setRecorderPanelOpen] = useState(false); + const [recordingError, setRecordingError] = useState(null); const [previewTokenReady, setPreviewTokenReady] = useState(false); const iframeRef = useRef(null); const windowRef = useRef(null); @@ -348,6 +357,8 @@ function OverlayApp() { const pipPeerRef = useRef(null); const pipStreamRef = useRef(null); const pipSettingsRef = useRef(null); + const parentPictureInPictureRequestRef = + useRef | null>(null); const previewSessionIdRef = useRef(null); const webcamRef = useRef(null); const settingsRef = useRef(null); @@ -383,6 +394,28 @@ function OverlayApp() { }); }, []); + const requestParentPictureInPicture = useCallback( + async (video: HTMLVideoElement) => { + if (recordingPreviewActiveRef.current) return; + const request = video.requestPictureInPicture(); + parentPictureInPictureRequestRef.current = request; + try { + await request; + if ( + recordingPreviewActiveRef.current && + document.pictureInPictureElement === video + ) { + await document.exitPictureInPicture(); + } + } finally { + if (parentPictureInPictureRequestRef.current === request) { + parentPictureInPictureRequestRef.current = null; + } + } + }, + [], + ); + const persistPreviewFrame = useCallback( (frame: WebcamPreviewFrame | null, force: boolean) => { if (!frame) return; @@ -453,6 +486,67 @@ function OverlayApp() { postPreviewMessage, ]); + const stopFramePictureInPictureForCapture = useCallback( + () => + new Promise((resolve) => { + if (!iframeRef.current || !previewOpenRef.current || !iframeReady) { + resolve(true); + return; + } + + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + window.clearTimeout(timeout); + resolve(ok); + }; + const timeout = window.setTimeout(() => finish(false), 3000); + chrome.runtime.sendMessage( + { + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "stop-for-capture", + } satisfies PreviewParentMessage, + (response: unknown) => { + const responseOk = + !chrome.runtime.lastError && + response !== null && + typeof response === "object" && + "ok" in response && + response.ok === true; + finish(responseOk); + }, + ); + }), + [iframeReady], + ); + + const prepareCameraForCapture = useCallback(async () => { + recordingPreviewActiveRef.current = true; + setRecordingPreviewActive(true); + const video = pipVideoRef.current; + const pendingParentRequest = parentPictureInPictureRequestRef.current; + const [frameResult, parentResult] = await Promise.allSettled([ + stopFramePictureInPictureForCapture(), + (async () => { + if (pendingParentRequest) { + await pendingParentRequest.catch(() => undefined); + } + if (video && document.pictureInPictureElement === video) { + await document.exitPictureInPicture(); + } + })(), + ]); + stopLocalPreview(); + return ( + frameResult.status === "fulfilled" && + frameResult.value && + parentResult.status === "fulfilled" && + (!video || document.pictureInPictureElement !== video) + ); + }, [stopFramePictureInPictureForCapture, stopLocalPreview]); + const applyWebcamSettings = useCallback( (getNext: (current: WebcamSettings) => WebcamSettings) => { setExtensionSettings((current) => { @@ -603,6 +697,19 @@ function OverlayApp() { }; }, [stopLocalPreview]); + useEffect(() => { + if (!recordingPreviewActive) return; + const video = pipVideoRef.current; + if (video && document.pictureInPictureElement === video) { + void document.exitPictureInPicture().catch(() => undefined); + } + postPreviewMessage({ + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "exit-auto-pip", + }); + }, [postPreviewMessage, recordingPreviewActive]); + useEffect(() => { let disposed = false; Promise.all([ @@ -639,7 +746,19 @@ function OverlayApp() { }) .catch(() => undefined); }; + const syncRecordingError = () => { + loadSharedRecordingState() + .then((state) => { + if (!disposed) { + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } + }) + .catch(() => undefined); + }; syncSharedUi(); + syncRecordingError(); const handleStorageChange = ( changes: Record, areaName: string, @@ -647,6 +766,14 @@ function OverlayApp() { if (areaName === "session" && changes[SHARED_UI_STATE_KEY]) { syncSharedUi(); } + if (areaName === "session" && changes[RECORDING_STATE_KEY]) { + const state = changes[RECORDING_STATE_KEY].newValue as + | SharedRecordingState + | undefined; + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } }; chrome.storage.onChanged.addListener(handleStorageChange); return () => { @@ -708,12 +835,66 @@ function OverlayApp() { }, [stopLocalPreview]); useEffect(() => { + let disposed = false; + const applyPreviewSettings = (message: { + settings: WebcamSettings; + recording: boolean; + }) => { + if (disposed) return; + const webcamSettings = message.settings; + const sameLivePreview = + previewOpenRef.current && + livePreviewReadyRef.current && + isSameWebcamSettings(webcamRef.current, webcamSettings); + if (!sameLivePreview) { + setLivePreviewReady(false); + } else { + // A restarted service worker loses which tab owns the live preview, + // so re-announce readiness whenever it pushes the same settings. + void sendServiceWorkerMessage({ + target: "service-worker", + type: "webcam-preview-ready", + }).catch(() => undefined); + } + setPreviewError(null); + setShowPreviewError(false); + setPreviewOpen(true); + recordingPreviewActiveRef.current = message.recording; + setRecordingPreviewActive(message.recording); + if (settingsRef.current) { + setExtensionSettings({ + ...settingsRef.current, + webcam: webcamSettings, + }); + } else { + sendServiceWorkerMessage({ + target: "service-worker", + type: "get-overlay-settings", + }) + .then((response) => { + if (!disposed && response.ok && response.settings) { + setExtensionSettings({ + ...response.settings, + webcam: webcamSettings, + }); + } + }) + .catch(() => undefined); + } + }; + const handleMessage = ( message: unknown, _sender: chrome.runtime.MessageSender, sendResponse: (response?: unknown) => void, ) => { if (!isOverlayMessage(message)) return false; + if (message.type === "overlay-settings" && message.recording) { + void prepareCameraForCapture() + .then((ok) => sendResponse({ ok })) + .catch(() => sendResponse({ ok: false })); + return true; + } sendResponse({ ok: true }); @@ -750,6 +931,7 @@ function OverlayApp() { } if (message.type === "overlay-enter-auto-pip") { + if (recordingPreviewActiveRef.current) return false; // Only one Picture in Picture surface may drive at a time; racing // the parent fallback video against the preview iframe flips PiP // on and off and leaves the badge state inconsistent. @@ -770,7 +952,7 @@ function OverlayApp() { if (video && parentPipSupported) { void video .play() - .then(() => video.requestPictureInPicture()) + .then(() => requestParentPictureInPicture(video)) .catch(() => undefined); } return false; @@ -791,48 +973,13 @@ function OverlayApp() { if (message.type !== "overlay-settings") return false; - if (previewDismissedRef.current) return false; - - const webcamSettings = message.settings; - const sameLivePreview = - previewOpenRef.current && - livePreviewReadyRef.current && - isSameWebcamSettings(webcamRef.current, webcamSettings); - if (!sameLivePreview) { - setLivePreviewReady(false); - } else { - // A restarted service worker loses which tab owns the live preview, - // so re-announce readiness whenever it pushes the same settings. - void sendServiceWorkerMessage({ - target: "service-worker", - type: "webcam-preview-ready", - }).catch(() => undefined); - } - setPreviewError(null); - setShowPreviewError(false); - setPreviewOpen(true); - recordingPreviewActiveRef.current = message.recording; - setRecordingPreviewActive(message.recording); - if (settingsRef.current) { - setExtensionSettings({ - ...settingsRef.current, - webcam: webcamSettings, - }); - } else { - sendServiceWorkerMessage({ - target: "service-worker", - type: "get-overlay-settings", + void loadWebcamPreviewDismissed() + .then((dismissed) => { + if (disposed) return; + previewDismissedRef.current = dismissed; + if (!dismissed) applyPreviewSettings(message); }) - .then((response) => { - if (response.ok && response.settings) { - setExtensionSettings({ - ...response.settings, - webcam: webcamSettings, - }); - } - }) - .catch(() => undefined); - } + .catch(() => undefined); return false; }; @@ -841,8 +988,17 @@ function OverlayApp() { startupReplayedRef.current = true; replayStartupMessages(handleMessage); } - return () => chrome.runtime.onMessage.removeListener(handleMessage); - }, [parentPipSupported, postPreviewMessage, stopLocalPreview]); + return () => { + disposed = true; + chrome.runtime.onMessage.removeListener(handleMessage); + }; + }, [ + parentPipSupported, + postPreviewMessage, + prepareCameraForCapture, + requestParentPictureInPicture, + stopLocalPreview, + ]); const beginDrag = useCallback((clientX: number, clientY: number) => { isDraggingRef.current = true; @@ -1082,7 +1238,11 @@ function OverlayApp() { if (event.type === "ready") { setIframeReady(true); const current = webcamRef.current; - if (current?.enabled && current.deviceId) { + if ( + !recordingPreviewActiveRef.current && + current?.enabled && + current.deviceId + ) { postPreviewMessage({ source: "cap-extension-overlay", token: PREVIEW_TOKEN, @@ -1430,6 +1590,7 @@ function OverlayApp() { }, [applyWebcamSettings]); const handleTogglePictureInPicture = useCallback(() => { + if (recordingPreviewActiveRef.current) return; const video = pipVideoRef.current; if (video && document.pictureInPictureElement === video) { void document.exitPictureInPicture().catch(() => undefined); @@ -1454,10 +1615,10 @@ function OverlayApp() { void (async () => { try { await video.play().catch(() => undefined); - await video.requestPictureInPicture(); + await requestParentPictureInPicture(video); } catch {} })(); - }, [parentPipSupported, postPreviewMessage]); + }, [parentPipSupported, postPreviewMessage, requestParentPictureInPicture]); const handlePreviewLoad = useCallback(() => { const current = webcamRef.current; @@ -1511,6 +1672,7 @@ function OverlayApp() { width: `${metrics.width}px`, height: `${totalHeight}px`, borderRadius, + visibility: recordingPreviewActive ? "hidden" : "visible", }} onPointerDown={handlePointerDown} > @@ -1671,6 +1833,12 @@ function OverlayApp() { }).catch(() => undefined); }} /> + {recordingError && recorderPanelOpen ? ( +
+ Recording failed. + {recordingError} +
+ ) : null} {isDragging ? ( @@ -1726,7 +1894,7 @@ const mountOverlay = () => { root.dataset.capMounted = "true"; const shadow = root.attachShadow({ mode: "closed" }); const style = document.createElement("style"); - style.textContent = overlayCss; + style.textContent = `${overlayCss}\n${recordingErrorCss}`; const app = document.createElement("div"); shadow.append(style, app); document.documentElement.append(root); diff --git a/apps/chrome-extension/src/content/recording-error-toast.css b/apps/chrome-extension/src/content/recording-error-toast.css new file mode 100644 index 00000000000..b3b665e54a1 --- /dev/null +++ b/apps/chrome-extension/src/content/recording-error-toast.css @@ -0,0 +1,27 @@ +.cap-extension-recording-error-toast { + position: fixed; + top: 20px; + left: 50%; + z-index: 2147483647; + display: flex; + max-width: min(440px, calc(100vw - 40px)); + flex-direction: column; + gap: 5px; + padding: 12px 16px; + border: 1px solid #f1c1c4; + border-radius: 12px; + background: #fff; + box-shadow: 0 12px 40px rgba(0, 0, 0, 0.16); + color: #3f171b; + font-family: Inter, ui-sans-serif, system-ui, sans-serif; + font-size: 12px; + line-height: 1.35; + pointer-events: auto; + transform: translateX(-50%); +} + +.cap-extension-recording-error-toast strong { + color: #c9343e; + font-size: 13px; + font-weight: 600; +} diff --git a/apps/chrome-extension/src/offscreen/display-capture.test.ts b/apps/chrome-extension/src/offscreen/display-capture.test.ts index a1f860e5826..36b3654587e 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.test.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.test.ts @@ -4,6 +4,21 @@ import { captureDisplayStream } from "./display-capture"; const stream = {} as MediaStream; describe("captureDisplayStream", () => { + it("prefers the current tab when tab capture needs the display picker", async () => { + const request = vi.fn().mockResolvedValue(stream); + + await expect(captureDisplayStream("tab", false, request)).resolves.toBe( + stream, + ); + expect(request).toHaveBeenCalledTimes(1); + expect(request.mock.calls[0]?.[0]).toMatchObject({ + preferCurrentTab: true, + selfBrowserSurface: "include", + monitorTypeSurfaces: "exclude", + audio: false, + }); + }); + it("falls back when advanced options are rejected before a picker can open", async () => { const request = vi .fn() diff --git a/apps/chrome-extension/src/offscreen/display-capture.ts b/apps/chrome-extension/src/offscreen/display-capture.ts index af5ee9547da..846c62a626d 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.ts @@ -44,7 +44,7 @@ const canRetryWithoutShowingAnotherPicker = (attempt: CaptureAttempt) => shouldRetryDisplayMediaWithoutPreferences(attempt.error); export const captureDisplayStream = async ( - mode: Exclude, + mode: Exclude, includeAudio: boolean, requestDisplayMedia: DisplayMediaRequester, ) => { diff --git a/apps/chrome-extension/src/offscreen/recorder-start-sync.test.ts b/apps/chrome-extension/src/offscreen/recorder-start-sync.test.ts new file mode 100644 index 00000000000..7afa0342e6e --- /dev/null +++ b/apps/chrome-extension/src/offscreen/recorder-start-sync.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { observePairedRecorderStarts } from "./recorder-start-sync"; + +describe("observePairedRecorderStarts", () => { + it("measures from recorder start events after a long setup delay", () => { + const screenRecorder = new EventTarget() as MediaRecorder; + const cameraRecorder = new EventTarget() as MediaRecorder; + const offsets: number[] = []; + let time = 0; + observePairedRecorderStarts( + screenRecorder, + cameraRecorder, + (offset) => offsets.push(offset), + () => time, + ); + + time = 1_800; + screenRecorder.dispatchEvent(new Event("start")); + expect(offsets).toEqual([]); + time = 1_844.4; + cameraRecorder.dispatchEvent(new Event("start")); + expect(offsets).toEqual([44]); + + time = 2_000; + screenRecorder.dispatchEvent(new Event("start")); + cameraRecorder.dispatchEvent(new Event("start")); + expect(offsets).toEqual([44]); + }); +}); diff --git a/apps/chrome-extension/src/offscreen/recorder-start-sync.ts b/apps/chrome-extension/src/offscreen/recorder-start-sync.ts new file mode 100644 index 00000000000..d8ecae61c0e --- /dev/null +++ b/apps/chrome-extension/src/offscreen/recorder-start-sync.ts @@ -0,0 +1,30 @@ +export const observePairedRecorderStarts = ( + screenRecorder: MediaRecorder, + cameraRecorder: MediaRecorder, + onOffset: (offsetMs: number) => void, + now: () => number = () => performance.now(), +) => { + let screenStartedAt: number | null = null; + let cameraStartedAt: number | null = null; + const publishOffset = () => { + if (screenStartedAt !== null && cameraStartedAt !== null) { + onOffset(Math.round(cameraStartedAt - screenStartedAt)); + } + }; + screenRecorder.addEventListener( + "start", + () => { + screenStartedAt = now(); + publishOffset(); + }, + { once: true }, + ); + cameraRecorder.addEventListener( + "start", + () => { + cameraStartedAt = now(); + publishOffset(); + }, + { once: true }, + ); +}; diff --git a/apps/chrome-extension/src/offscreen/recorder.ts b/apps/chrome-extension/src/offscreen/recorder.ts index 6173df09647..e0b3637cd71 100644 --- a/apps/chrome-extension/src/offscreen/recorder.ts +++ b/apps/chrome-extension/src/offscreen/recorder.ts @@ -66,6 +66,7 @@ import { waitForIceGatheringComplete, } from "../shared/webrtc"; import { captureDisplayStream } from "./display-capture"; +import { observePairedRecorderStarts } from "./recorder-start-sync"; const RECORDING_TIMESLICE_MS = 1000; const RECORDING_TIMESLICE_GUARD_MS = RECORDING_TIMESLICE_MS * 3; @@ -93,13 +94,19 @@ type RecordingSound = "start-recording" | "stop-recording"; type ActiveRecording = { recorder: MediaRecorder; + cameraRecorder: MediaRecorder | null; + settings: ExtensionSettings; + authToken: string; stopPromise: Promise; + cameraStopPromise: Promise | null; streams: MediaStream[]; recordingStream: MediaStream; statusTimer: number | null; spool: RecordingSpool; + cameraSpool: RecordingSpool | null; uploader: InstantRecordingUploader; startedAt: number; + cameraOffsetMs: number | null; durationMs: number; lastResumedAt: number | null; videoId: VideoId; @@ -107,16 +114,24 @@ type ActiveRecording = { width: number; height: number; fps: number; + cameraWidth: number | null; + cameraHeight: number | null; + cameraFps: number | null; subpath: string; mimeType: string; + cameraSubpath: string | null; + cameraMimeType: string | null; maxDurationMs: number | null; audioContext?: AudioContext; chunkChain: Promise; + cameraChunkChain: Promise; dataRequestInterval: number | null; + cameraDataRequestInterval: number | null; chunkStartGuard: number | null; chunkingMode: ChunkingMode | null; lastChunkAt: number | null; recordedBytes: number; + cameraRecordedBytes: number; finalizePromise: Promise | null; cleanedUp: boolean; // A spool write failure (IndexedDB quota, backpressure) must not end an @@ -125,6 +140,8 @@ type ActiveRecording = { // streaming upload continues untouched. spoolFailed: boolean; memoryBackup: LocalRecordingState; + cameraSpoolFailed: boolean; + cameraMemoryBackup: LocalRecordingState; }; let activeRecording: ActiveRecording | null = null; @@ -437,12 +454,21 @@ const tabCaptureConstraints = (streamId: string, includeAudio: boolean) => const getMainStream = async (request: StartRecordingRequest) => { if (request.mode === "tab") { - if (!request.tabStreamId) throw new Error("Tab stream id is missing"); - return navigator.mediaDevices.getUserMedia( - tabCaptureConstraints( - request.tabStreamId, - request.settings.systemAudio.enabled, - ), + if (request.tabStreamId) { + return navigator.mediaDevices.getUserMedia( + tabCaptureConstraints( + request.tabStreamId, + request.settings.systemAudio.enabled, + ), + ); + } + return captureDisplayStream( + "tab", + request.settings.systemAudio.enabled, + (options) => + navigator.mediaDevices.getDisplayMedia( + options as DisplayMediaStreamOptions, + ), ); } @@ -635,6 +661,18 @@ const cleanupActiveRecording = async (recording: ActiveRecording) => { if (recording.chunkStartGuard !== null) { window.clearTimeout(recording.chunkStartGuard); } + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + } + if (recording.recorder.state !== "inactive") { + recording.recorder.stop(); + } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } for (const stream of recording.streams) { stopTracks(stream); } @@ -656,19 +694,44 @@ const sweepOrphanedRecordingSpools = async () => { loadLiveRecordingManifests(), ]); const now = Date.now(); - const knownSessions = new Set(failed.map((entry) => entry.sessionId)); + const knownSessions = new Set( + failed.flatMap((entry) => + entry.cameraSessionId + ? [entry.sessionId, entry.cameraSessionId] + : [entry.sessionId], + ), + ); + const sessionsById = new Map( + sessions.map((session) => [session.sessionId, session]), + ); const manifestsBySession = new Map( manifests.map((manifest) => [manifest.sessionId, manifest]), ); + const manifestsByCameraSession = new Map( + manifests.flatMap((manifest) => { + const sessionId = manifest.cameraSessionId; + return sessionId ? [[sessionId, manifest] as const] : []; + }), + ); const remainingSessions = new Set(); let entries = [...failed]; for (const orphan of sessions) { - if (activeRecording?.spool.sessionId === orphan.sessionId) { + if ( + activeRecording?.spool.sessionId === orphan.sessionId || + activeRecording?.cameraSpool?.sessionId === orphan.sessionId + ) { remainingSessions.add(orphan.sessionId); continue; } - if (now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS) { + const manifest = manifestsBySession.get(orphan.sessionId); + // A surviving manifest belongs to a previous offscreen document. This + // document is the extension's only spool writer, so the shared live + // window must not delay recovery after that writer has gone away. + if ( + now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS && + !manifest + ) { remainingSessions.add(orphan.sessionId); continue; } @@ -682,20 +745,47 @@ const sweepOrphanedRecordingSpools = async () => { () => undefined, ); entries = entries.filter( - (entry) => entry.sessionId !== orphan.sessionId, + (entry) => + entry.sessionId !== orphan.sessionId && + entry.cameraSessionId !== orphan.sessionId, ); continue; } remainingSessions.add(orphan.sessionId); + const cameraOwner = manifestsByCameraSession.get(orphan.sessionId); + if ( + cameraOwner && + (sessionsById.has(cameraOwner.sessionId) || + knownSessions.has(cameraOwner.sessionId)) + ) { + continue; + } if (!knownSessions.has(orphan.sessionId) && orphan.totalBytes > 0) { // A crash-stranded session whose live manifest survived keeps its // videoId/subpath so the entry stays retryable, not download-only. // The duration is a wall-clock estimate (it includes pauses); it // only feeds the completion metadata on retry. - const manifest = manifestsBySession.get(orphan.sessionId); + const cameraSession = manifest?.cameraSessionId + ? sessionsById.get(manifest.cameraSessionId) + : undefined; + const cameraMetadata = + manifest?.cameraSessionId && cameraSession?.totalBytes + ? { + cameraSessionId: manifest.cameraSessionId, + cameraMimeType: + manifest.cameraMimeType ?? cameraSession.mimeType, + cameraSubpath: manifest.cameraSubpath, + cameraOffsetMs: manifest.cameraOffsetMs ?? 0, + cameraWidth: manifest.cameraWidth, + cameraHeight: manifest.cameraHeight, + cameraFps: manifest.cameraFps, + cameraTotalBytes: cameraSession.totalBytes, + } + : {}; entries.push({ sessionId: orphan.sessionId, + ...cameraMetadata, videoId: manifest?.videoId ?? null, shareUrl: manifest?.shareUrl ?? null, mimeType: orphan.mimeType, @@ -710,6 +800,10 @@ const sweepOrphanedRecordingSpools = async () => { createdAt: orphan.updatedAt, message: "The recording was interrupted before its upload finished.", }); + if (manifest?.cameraSessionId && cameraSession?.totalBytes) { + knownSessions.add(manifest.cameraSessionId); + remainingSessions.add(manifest.cameraSessionId); + } } } @@ -724,6 +818,16 @@ const sweepOrphanedRecordingSpools = async () => { await deleteRecoveredRecordingSpool(entry.sessionId).catch( () => undefined, ); + if (entry.cameraSessionId) { + survivingSessions.delete(entry.cameraSessionId); + await deleteRecoveredRecordingSpool(entry.cameraSessionId).catch( + () => undefined, + ); + } + } + if (startInProgress && !activeRecording) return; + if (activeRecording) { + survivingSessions.add(activeRecording.spool.sessionId); } await pruneLiveRecordingManifests(survivingSessions).catch(() => undefined); } catch { @@ -736,7 +840,10 @@ function stopRecorderAfterError(recorder: MediaRecorder) { recorder.stop(); } window.setTimeout(() => { - if (activeRecording?.recorder === recorder) { + if ( + activeRecording?.recorder === recorder || + activeRecording?.cameraRecorder === recorder + ) { void stopRecording(); } }, 0); @@ -761,6 +868,20 @@ const stopManualChunking = (recording: ActiveRecording) => { } }; +const requestCameraRecorderData = (recording: ActiveRecording) => { + if (recording.cameraRecorder?.state !== "recording") return; + try { + recording.cameraRecorder.requestData(); + } catch {} +}; + +const stopCameraChunking = (recording: ActiveRecording) => { + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + recording.cameraDataRequestInterval = null; + } +}; + const beginManualChunking = (recording: ActiveRecording) => { recording.chunkingMode = "manual"; recording.lastChunkAt = null; @@ -811,13 +932,19 @@ const startRecording = async (request: StartRecordingRequest) => { const ownedStreams: MediaStream[] = []; let ownedVideoId: string | null = null; let ownedSpool: RecordingSpool | null = null; + let ownedCameraSpool: RecordingSpool | null = null; let ownedRecording: ActiveRecording | null = null; let countdownPromise: Promise | null = null; try { status = { phase: "creating" }; - const mainStream = await getMainStream(request); + const mainStream = await getMainStream(request).catch((error: unknown) => { + throw new Error( + `Unable to capture ${request.mode} source: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + }); ownedStreams.push(mainStream); throwIfStartCanceled(); const captureSource = getCaptureSource(request, mainStream); @@ -847,6 +974,34 @@ const startRecording = async (request: StartRecordingRequest) => { streams, routeFirstStreamToSpeakers: request.mode === "tab", }); + const cameraStream = + request.mode !== "camera" && request.settings.webcam.enabled + ? await getCameraPreviewStream(request.settings.webcam) + : null; + const cameraTrack = cameraStream?.getVideoTracks()[0]?.clone() ?? null; + if ( + request.settings.webcam.enabled && + request.mode !== "camera" && + !cameraTrack + ) { + throw new Error("The selected camera is not available for recording"); + } + const cameraRecordingStream = cameraTrack + ? new MediaStream([cameraTrack]) + : null; + if (cameraRecordingStream) { + ownedStreams.push(cameraRecordingStream); + streams.push(cameraRecordingStream); + } + const cameraDimensions = cameraRecordingStream + ? getStreamSize(cameraRecordingStream) + : null; + const cameraPipeline = cameraRecordingStream + ? selectRecordingPipeline(false) + : null; + if (cameraRecordingStream && !cameraPipeline) { + throw new Error("No supported camera recorder format is available"); + } const hasAudio = recordingStream.getAudioTracks().length > 0; const pipeline = selectRecordingPipeline(hasAudio); if (!pipeline) throw new Error("No supported recorder format is available"); @@ -887,6 +1042,11 @@ const startRecording = async (request: StartRecordingRequest) => { const spool = await RecordingSpool.create({ mimeType: pipeline.mimeType }); ownedSpool = spool; throwIfStartCanceled(); + const cameraSpool = cameraPipeline + ? await RecordingSpool.create({ mimeType: cameraPipeline.mimeType }) + : null; + ownedCameraSpool = cameraSpool; + throwIfStartCanceled(); const uploader = new InstantRecordingUploader({ videoId: creation.id, uploadId: uploadSession.uploadId, @@ -944,6 +1104,12 @@ const startRecording = async (request: StartRecordingRequest) => { const recorder = new MediaRecorder(recordingStream, { mimeType: pipeline.mimeType, }); + const cameraRecorder = + cameraRecordingStream && cameraPipeline + ? new MediaRecorder(cameraRecordingStream, { + mimeType: cameraPipeline.mimeType, + }) + : null; await countdownPromise; throwIfStartCanceled(); @@ -957,13 +1123,19 @@ const startRecording = async (request: StartRecordingRequest) => { const recording: ActiveRecording = { recorder, + cameraRecorder, + settings: request.settings, + authToken: request.auth.authApiKey, stopPromise: Promise.resolve(), + cameraStopPromise: null, streams, recordingStream, statusTimer: null, spool, + cameraSpool, uploader, startedAt, + cameraOffsetMs: null, durationMs: 0, lastResumedAt: startedAt, videoId: creation.id, @@ -971,20 +1143,32 @@ const startRecording = async (request: StartRecordingRequest) => { width, height, fps, + cameraWidth: cameraDimensions?.width ?? null, + cameraHeight: cameraDimensions?.height ?? null, + cameraFps: cameraDimensions?.fps ?? null, subpath, mimeType: pipeline.mimeType, + cameraSubpath: cameraPipeline + ? `camera-upload.${cameraPipeline.fileExtension}` + : null, + cameraMimeType: cameraPipeline?.mimeType ?? null, maxDurationMs, audioContext, chunkChain: Promise.resolve(), + cameraChunkChain: Promise.resolve(), dataRequestInterval: null, + cameraDataRequestInterval: null, chunkStartGuard: null, chunkingMode: null, lastChunkAt: null, recordedBytes: 0, + cameraRecordedBytes: 0, finalizePromise: null, cleanedUp: false, spoolFailed: false, memoryBackup: initialLocalRecordingState(), + cameraSpoolFailed: false, + cameraMemoryBackup: initialLocalRecordingState(), }; ownedRecording = recording; @@ -999,6 +1183,17 @@ const startRecording = async (request: StartRecordingRequest) => { // failed-recording entry (videoId, subpath) instead of download-only. await saveLiveRecordingManifest({ sessionId: spool.sessionId, + ...(cameraSpool + ? { + cameraSessionId: cameraSpool.sessionId, + cameraMimeType: cameraPipeline?.mimeType, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + } + : {}), videoId: creation.id, shareUrl: creation.shareUrl, mimeType: pipeline.mimeType, @@ -1013,6 +1208,13 @@ const startRecording = async (request: StartRecordingRequest) => { recorder.onstop = () => resolve(); recorder.onerror = () => reject(new Error("MediaRecorder failed")); }); + if (cameraRecorder) { + recording.cameraStopPromise = new Promise((resolve, reject) => { + cameraRecorder.onstop = () => resolve(); + cameraRecorder.onerror = () => + reject(new Error("Camera MediaRecorder failed")); + }); + } // A mid-recording recorder failure must stop the session right away; // without this the rejection sits unhandled while the timer keeps // ticking over a recorder that no longer produces chunks, and nothing @@ -1073,6 +1275,39 @@ const startRecording = async (request: StartRecordingRequest) => { stopRecorderAfterError(recorder); } }; + if (cameraRecorder && cameraSpool) { + cameraRecorder.ondataavailable = (event) => { + if (event.data.size === 0) return; + recording.cameraRecordedBytes += event.data.size; + recording.cameraChunkChain = recording.cameraChunkChain.then( + async () => { + if (recording.cameraSpoolFailed) { + appendCameraMemoryBackupChunk(recording, event.data); + return; + } + try { + await cameraSpool.appendChunk(event.data); + } catch (error) { + recording.cameraSpoolFailed = true; + console.warn( + "Camera recording spool failed; keeping the local backup in memory", + error, + ); + } + }, + ); + }; + recording.cameraStopPromise?.catch(() => { + if (activeRecording !== recording || recording.finalizePromise) return; + status = { + phase: "error", + message: "Camera recording stopped unexpectedly", + videoId: creation.id, + }; + broadcastStatus(); + stopRecorderAfterError(cameraRecorder); + }); + } status = { phase: "recording", @@ -1081,6 +1316,39 @@ const startRecording = async (request: StartRecordingRequest) => { durationMs: 0, updatedAt: startedAt, }; + const saveCameraManifest = () => + saveLiveRecordingManifest({ + sessionId: spool.sessionId, + cameraSessionId: cameraSpool?.sessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + videoId: creation.id, + shareUrl: creation.shareUrl, + mimeType: pipeline.mimeType, + subpath, + width, + height, + fps, + startedAt, + }); + if (cameraRecorder) { + observePairedRecorderStarts(recorder, cameraRecorder, (offsetMs) => { + if ( + activeRecording !== recording || + recording.finalizePromise || + recording.cleanedUp + ) { + return; + } + recording.cameraOffsetMs = offsetMs; + void saveCameraManifest().catch(() => undefined); + }); + } + const screenStartRequestedAt = performance.now(); try { recorder.start(RECORDING_TIMESLICE_MS); @@ -1090,6 +1358,21 @@ const startRecording = async (request: StartRecordingRequest) => { recorder.start(); beginManualChunking(recording); } + if (cameraRecorder) { + const cameraStartRequestedAt = performance.now(); + recording.cameraOffsetMs = Math.round( + cameraStartRequestedAt - screenStartRequestedAt, + ); + try { + cameraRecorder.start(RECORDING_TIMESLICE_MS); + } catch { + cameraRecorder.start(); + recording.cameraDataRequestInterval = window.setInterval(() => { + requestCameraRecorderData(recording); + }, RECORDING_TIMESLICE_MS); + } + await saveCameraManifest().catch(() => undefined); + } playRecordingSound("start-recording", request.settings); // The service worker that sent start-recording may have been killed // while the capture picker was open, which destroys the response @@ -1109,6 +1392,23 @@ const startRecording = async (request: StartRecordingRequest) => { ); countdownResolve?.(); await countdownPromise?.catch(() => undefined); + if (ownedRecording) { + if (ownedRecording.recorder.state !== "inactive") { + ownedRecording.recorder.stop(); + await ownedRecording.stopPromise.catch(() => undefined); + } + if ( + ownedRecording.cameraRecorder && + ownedRecording.cameraRecorder.state !== "inactive" + ) { + ownedRecording.cameraRecorder.stop(); + await ownedRecording.cameraStopPromise?.catch(() => undefined); + } + await Promise.all([ + ownedRecording.chunkChain, + ownedRecording.cameraChunkChain, + ]); + } // Release only what this attempt acquired. No chunk can have been // captured yet — chunks only flow once recorder.start() succeeds, after // which nothing here throws — so the spool holds no recoverable data. @@ -1134,15 +1434,22 @@ const startRecording = async (request: StartRecordingRequest) => { ); await ownedSpool.dispose().catch(() => undefined); } + await ownedCameraSpool?.dispose().catch(() => undefined); // Reset the "creating" status so later status syncs do not report a // phantom in-progress recording. - if (status.phase === "creating") { + if ( + status.phase === "creating" || + (ownedRecording && + "videoId" in status && + status.videoId === ownedRecording.videoId) + ) { status = isUserCancellationError(error) ? { phase: "idle" } : { phase: "error", message: error instanceof Error ? error.message : String(error), }; + broadcastStatus(); } throw error; } finally { @@ -1163,6 +1470,22 @@ const appendMemoryBackupChunk = (recording: ActiveRecording, chunk: Blob) => { } }; +const appendCameraMemoryBackupChunk = ( + recording: ActiveRecording, + chunk: Blob, +) => { + const previous = recording.cameraMemoryBackup; + recording.cameraMemoryBackup = appendLocalRecordingChunk(previous, chunk, { + mode: "capped", + maxBytes: MEMORY_BACKUP_MAX_BYTES, + }); + if (recording.cameraMemoryBackup.overflowed && !previous.overflowed) { + console.warn( + "In-memory camera backup exceeded its cap; dropping the local copy", + ); + } +}; + // The complete recording: the spool when it stayed healthy, otherwise the // spooled prefix plus the in-memory continuation. Null once the capped // memory backup has overflowed — a truncated local copy must not masquerade @@ -1180,6 +1503,68 @@ const recoverRecordingBlob = async (recording: ActiveRecording) => { return new Blob(parts, { type: recording.mimeType }); }; +const recoverCameraRecordingBlob = async (recording: ActiveRecording) => { + const spool = recording.cameraSpool; + if (!spool) return null; + if (!recording.cameraSpoolFailed) return spool.recoverBlob(); + if (recording.cameraMemoryBackup.overflowed) return null; + const spooledBlob = await spool.recoverBlob().catch(() => null); + const parts = spooledBlob + ? [spooledBlob, ...recording.cameraMemoryBackup.chunks] + : recording.cameraMemoryBackup.chunks; + if (parts.length === 0) return null; + return new Blob(parts, { type: recording.cameraMimeType ?? "" }); +}; + +const uploadCameraSidecar = async (options: { + videoId: VideoId; + settings: ExtensionSettings; + authToken: string; + blob: Blob; + mimeType: string; + subpath: string; + screenSubpath: string; + cameraOffsetMs: number; + durationMs: number; + width: number; + height: number; + fps: number; +}) => { + const api = { + baseUrl: options.settings.apiBaseUrl, + authToken: options.authToken, + requestTimeoutMs: DEFAULT_API_REQUEST_TIMEOUT_MS, + extraBody: { + screenSubpath: options.screenSubpath, + cameraOffsetMs: options.cameraOffsetMs, + }, + }; + const uploadSession = await initiateMultipartUpload({ + videoId: options.videoId, + contentType: options.mimeType, + subpath: options.subpath, + api, + }); + const uploader = new InstantRecordingUploader({ + videoId: options.videoId, + uploadId: uploadSession.uploadId, + provider: uploadSession.provider, + mimeType: options.mimeType, + subpath: options.subpath, + api, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + }); + await uploader.finalize({ + finalBlob: options.blob, + durationSeconds: Math.max(1, Math.round(options.durationMs / 1000)), + width: options.width, + height: options.height, + fps: options.fps, + subpath: options.subpath, + }); +}; + // Returns whether the recording bytes are persisted and retryable from the // upload page. const rememberFailedRecording = async ( @@ -1189,6 +1574,7 @@ const rememberFailedRecording = async ( if (recording.recordedBytes === 0) return false; let sessionId = recording.spool.sessionId; + let cameraSessionId = recording.cameraSpool?.sessionId; if (recording.spoolFailed) { // The original spool is missing the tail that went to memory; a retry // reading it would upload a truncated file. Persist the full recording @@ -1214,9 +1600,41 @@ const rememberFailedRecording = async ( if (!replacement) return false; sessionId = replacement.sessionId; } + if (recording.cameraSpoolFailed && recording.cameraSpool) { + const fullBlob = await recoverCameraRecordingBlob(recording).catch( + () => null, + ); + const replacement = + fullBlob && fullBlob.size >= recording.cameraRecordedBytes + ? await RecordingSpool.create({ + mimeType: recording.cameraMimeType ?? fullBlob.type, + maxPendingChunkBytes: fullBlob.size, + }) + .then(async (spool) => { + await spool.appendChunk(fullBlob); + await spool.flush(); + return spool; + }) + .catch(() => null) + : null; + await recording.cameraSpool.dispose().catch(() => undefined); + cameraSessionId = replacement?.sessionId; + } const saved = await upsertFailedRecording({ sessionId, + ...(cameraSessionId && recording.cameraRecordedBytes > 0 + ? { + cameraSessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + cameraTotalBytes: recording.cameraRecordedBytes, + } + : {}), videoId: recording.videoId, shareUrl: recording.shareUrl, mimeType: recording.mimeType, @@ -1237,6 +1655,11 @@ const rememberFailedRecording = async ( await deleteRecoveredRecordingSpool(dropped.sessionId).catch( () => undefined, ); + if (dropped.cameraSessionId) { + await deleteRecoveredRecordingSpool(dropped.cameraSessionId).catch( + () => undefined, + ); + } } return saved.kept.some((entry) => entry.sessionId === sessionId); @@ -1244,9 +1667,12 @@ const rememberFailedRecording = async ( const finalizeRecording = async (recording: ActiveRecording) => { try { - await recording.stopPromise; + await Promise.all([ + recording.stopPromise, + recording.cameraStopPromise ?? Promise.resolve(), + ]); await cleanupActiveRecording(recording); - await recording.chunkChain; + await Promise.all([recording.chunkChain, recording.cameraChunkChain]); // finalBlob is null when the capped memory backup overflowed. The // streamed parts still carry every byte; the uploader just falls back // to its recorded-bytes counter instead of the local blob's size. @@ -1254,6 +1680,35 @@ const finalizeRecording = async (recording: ActiveRecording) => { if ((!finalBlob || finalBlob.size === 0) && recording.recordedBytes === 0) { throw new Error("No recording data was captured"); } + if ( + recording.cameraSpool && + recording.cameraRecordedBytes > 0 && + (!recording.cameraSubpath || + !recording.cameraMimeType || + recording.cameraOffsetMs === null) + ) { + throw new Error("The separate camera recording metadata is incomplete"); + } + if (recording.cameraSpool && recording.cameraRecordedBytes > 0) { + const cameraBlob = await recoverCameraRecordingBlob(recording); + if (!cameraBlob || cameraBlob.size < recording.cameraRecordedBytes) { + throw new Error("The separate camera recording could not be recovered"); + } + await uploadCameraSidecar({ + videoId: recording.videoId, + settings: recording.settings, + authToken: recording.authToken, + blob: cameraBlob, + mimeType: recording.cameraMimeType ?? cameraBlob.type, + subpath: recording.cameraSubpath ?? "camera-upload.webm", + screenSubpath: recording.subpath, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + durationMs: recording.durationMs, + width: recording.cameraWidth ?? DEFAULT_WIDTH, + height: recording.cameraHeight ?? DEFAULT_HEIGHT, + fps: recording.cameraFps ?? DEFAULT_FPS, + }); + } await recording.uploader.finalize({ finalBlob: finalBlob && finalBlob.size > 0 ? finalBlob : null, durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), @@ -1263,6 +1718,7 @@ const finalizeRecording = async (recording: ActiveRecording) => { subpath: recording.subpath, }); await recording.spool.dispose(); + await recording.cameraSpool?.dispose(); await removeFailedRecording(recording.spool.sessionId).catch( () => undefined, ); @@ -1354,6 +1810,7 @@ async function stopRecording() { } stopManualChunking(recording); + stopCameraChunking(recording); const now = Date.now(); recording.durationMs = getRecordingDuration(recording, now); recording.lastResumedAt = null; @@ -1361,6 +1818,12 @@ async function stopRecording() { if (recording.recorder.state !== "inactive") { recording.recorder.stop(); } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } if (status.phase !== "error") { const snapshot = getCurrentUploadSnapshot(); @@ -1396,6 +1859,9 @@ const pauseRecording = () => { recording.durationMs = getRecordingDuration(recording, now); recording.lastResumedAt = null; recording.recorder.pause(); + if (recording.cameraRecorder?.state === "recording") { + recording.cameraRecorder.pause(); + } if (status.phase === "recording") { status = { ...status, @@ -1414,6 +1880,9 @@ const resumeRecording = () => { } const now = Date.now(); recording.recorder.resume(); + if (recording.cameraRecorder?.state === "paused") { + recording.cameraRecorder.resume(); + } recording.lastResumedAt = now; if (status.phase === "paused") { status = { @@ -1454,6 +1923,7 @@ const runFailedUploadRetry = async ( const orphan = await recoverRecordingSpoolSession(failed.sessionId); if (!orphan || orphan.blob.size === 0) { await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); throw new Error("The recorded data is no longer available."); } @@ -1497,6 +1967,35 @@ const runFailedUploadRetry = async ( }; try { + if (failed.cameraSessionId) { + const cameraOrphan = await recoverRecordingSpoolSession( + failed.cameraSessionId, + ); + if ( + !cameraOrphan || + cameraOrphan.blob.size === 0 || + !failed.cameraMimeType || + !failed.cameraSubpath || + typeof failed.cameraOffsetMs !== "number" || + !Number.isFinite(failed.cameraOffsetMs) + ) { + throw new Error("The separate camera recording is no longer available"); + } + await uploadCameraSidecar({ + videoId: typedVideoId, + settings, + authToken: auth.authApiKey, + blob: cameraOrphan.blob, + mimeType: failed.cameraMimeType, + subpath: failed.cameraSubpath, + screenSubpath: subpath, + cameraOffsetMs: failed.cameraOffsetMs, + durationMs: failed.durationMs, + width: failed.cameraWidth ?? failed.width ?? DEFAULT_WIDTH, + height: failed.cameraHeight ?? failed.height ?? DEFAULT_HEIGHT, + fps: failed.cameraFps ?? failed.fps ?? DEFAULT_FPS, + }); + } const uploadSession = await initiateMultipartUpload({ videoId: typedVideoId, contentType: failed.mimeType, @@ -1543,7 +2042,13 @@ const runFailedUploadRetry = async ( await deleteRecoveredRecordingSpool(failed.sessionId).catch( () => undefined, ); + if (failed.cameraSessionId) { + await deleteRecoveredRecordingSpool(failed.cameraSessionId).catch( + () => undefined, + ); + } await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); setRetryStatus({ phase: "completed", videoId: typedVideoId, diff --git a/apps/chrome-extension/src/preview/camera-preview.tsx b/apps/chrome-extension/src/preview/camera-preview.tsx index da7741383f1..7771a00b60d 100644 --- a/apps/chrome-extension/src/preview/camera-preview.tsx +++ b/apps/chrome-extension/src/preview/camera-preview.tsx @@ -44,6 +44,11 @@ type ParentMessage = source: "cap-extension-overlay"; token: string; type: "stop"; + } + | { + source: "cap-extension-overlay"; + token: string; + type: "stop-for-capture"; }; const token = decodeURIComponent(window.location.hash.slice(1)); @@ -247,6 +252,9 @@ function App() { const sessionCounterRef = useRef(0); const autoPictureInPictureRef = useRef(false); const autoPictureInPictureEnabledRef = useRef(false); + const captureBlockedRef = useRef(false); + const pictureInPictureRequestRef = + useRef | null>(null); const previewEnabled = Boolean(settings?.enabled); const isPictureInPictureSupported = typeof document !== "undefined" && document.pictureInPictureEnabled; @@ -275,12 +283,24 @@ function App() { enabled ? async () => { const video = videoRef.current; - if (!video || document.pictureInPictureElement) return; + if ( + !video || + captureBlockedRef.current || + document.pictureInPictureElement + ) + return; try { autoPictureInPictureRef.current = true; - await video.requestPictureInPicture(); + const request = video.requestPictureInPicture(); + pictureInPictureRequestRef.current = request; + await request; + if (captureBlockedRef.current) { + await document.exitPictureInPicture(); + } } catch { autoPictureInPictureRef.current = false; + } finally { + pictureInPictureRequestRef.current = null; } } : null, @@ -318,6 +338,7 @@ function App() { const enterPictureInPicture = useCallback( async (auto: boolean) => { const currentVideo = videoRef.current; + if (captureBlockedRef.current) return; if (auto) { setAutomaticPictureInPicture(true); } @@ -329,10 +350,18 @@ function App() { return; } if (document.pictureInPictureElement) return; - await currentVideo.requestPictureInPicture(); + const request = currentVideo.requestPictureInPicture(); + pictureInPictureRequestRef.current = request; + await request; + if (captureBlockedRef.current) { + await document.exitPictureInPicture(); + return; + } autoPictureInPictureRef.current = auto; } catch { autoPictureInPictureRef.current = false; + } finally { + pictureInPictureRequestRef.current = null; } }, [isPictureInPictureSupported, setAutomaticPictureInPicture], @@ -358,7 +387,12 @@ function App() { const togglePictureInPicture = useCallback(async () => { const currentVideo = videoRef.current; - if (!currentVideo || !isPictureInPictureSupported) return; + if ( + !currentVideo || + !isPictureInPictureSupported || + captureBlockedRef.current + ) + return; try { setAutomaticPictureInPicture(false); @@ -366,13 +400,35 @@ function App() { if (document.pictureInPictureElement === currentVideo) { await document.exitPictureInPicture(); } else { - await currentVideo.requestPictureInPicture(); + const request = currentVideo.requestPictureInPicture(); + pictureInPictureRequestRef.current = request; + await request; + if (captureBlockedRef.current) { + await document.exitPictureInPicture(); + } } } catch { autoPictureInPictureRef.current = false; + } finally { + pictureInPictureRequestRef.current = null; } }, [isPictureInPictureSupported, setAutomaticPictureInPicture]); + const stopForCapture = useCallback(async () => { + captureBlockedRef.current = true; + setAutomaticPictureInPicture(false); + const pendingRequest = pictureInPictureRequestRef.current; + if (pendingRequest) { + await pendingRequest.catch(() => undefined); + } + const video = videoRef.current; + if (video && document.pictureInPictureElement === video) { + await document.exitPictureInPicture(); + } + stopPreview(); + return !video || document.pictureInPictureElement !== video; + }, [setAutomaticPictureInPicture, stopPreview]); + useEffect(() => { postParent({ type: "ready", @@ -389,10 +445,22 @@ function App() { // other extensions) cannot speak — window messages were forgeable since // the token is readable from the iframe src in the page DOM. The token // check scopes the runtime broadcast to this tab's preview. - const handleMessage = (message: unknown) => { + const handleMessage = ( + message: unknown, + _sender: chrome.runtime.MessageSender, + sendResponse: (response?: unknown) => void, + ) => { if (!isParentMessage(message)) return false; + if (message.type === "stop-for-capture") { + void stopForCapture() + .then((ok) => sendResponse({ ok })) + .catch(() => sendResponse({ ok: false })); + return true; + } + if (message.type === "settings") { + if (captureBlockedRef.current) return false; const nextSettings = message.settings; setSettings((current) => isSameWebcamSettings(current, nextSettings) ? current : nextSettings, @@ -427,6 +495,7 @@ function App() { enterPictureInPicture, exitAutoPictureInPicture, publishPreviewFrame, + stopForCapture, stopPreview, togglePictureInPicture, ]); diff --git a/apps/chrome-extension/src/shared/storage-bridge.test.ts b/apps/chrome-extension/src/shared/storage-bridge.test.ts new file mode 100644 index 00000000000..41b0b8ec38d --- /dev/null +++ b/apps/chrome-extension/src/shared/storage-bridge.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + loadLiveRecordingManifests, + saveLiveRecordingManifest, +} from "./storage"; +import { + executeStorageBridgeRequest, + isStorageBridgeRequest, + isTrustedOffscreenStorageSender, + requestStorage, + type StorageBridgeRequest, +} from "./storage-bridge"; + +beforeEach(() => { + delete (globalThis as { chrome?: unknown }).chrome; +}); + +describe("offscreen storage bridge", () => { + it("persists recording recovery metadata when the offscreen document has runtime only", async () => { + const data = new Map(); + const requests: StorageBridgeRequest[] = []; + (globalThis as { chrome?: unknown }).chrome = { + runtime: { + sendMessage: ( + message: unknown, + callback: (response: unknown) => void, + ) => { + if (!isStorageBridgeRequest(message)) { + callback({ ok: false, error: "Invalid request" }); + return; + } + requests.push(message); + if (message.type === "get") { + callback({ + ok: true, + items: Object.fromEntries( + message.keys + .filter((key) => data.has(key)) + .map((key) => [key, data.get(key)]), + ), + }); + return; + } + if (message.type === "set") { + for (const [key, value] of Object.entries(message.items)) { + data.set(key, value); + } + } else { + for (const key of Array.isArray(message.keys) + ? message.keys + : [message.keys]) { + data.delete(key); + } + } + callback({ ok: true }); + }, + }, + }; + + await saveLiveRecordingManifest({ + sessionId: "screen-session", + cameraSessionId: "camera-session", + videoId: "video-test", + shareUrl: "https://cap.so/s/video-test", + mimeType: "video/webm", + cameraMimeType: "video/webm", + subpath: "raw-upload.webm", + cameraSubpath: "camera-upload.webm", + cameraOffsetMs: 37, + width: 1920, + height: 1080, + fps: 30, + startedAt: 1_000, + }); + + expect(await loadLiveRecordingManifests()).toMatchObject([ + { + sessionId: "screen-session", + cameraSessionId: "camera-session", + cameraOffsetMs: 37, + }, + ]); + expect(requests.map((request) => request.type)).toEqual([ + "get", + "set", + "get", + ]); + }); + + it("rejects a storage proxy request from a content script or another extension page", () => { + const extensionId = "cap-test"; + const offscreenUrl = "chrome-extension://cap-test/offscreen.html"; + expect( + isTrustedOffscreenStorageSender( + { id: extensionId, url: offscreenUrl }, + extensionId, + offscreenUrl, + ), + ).toBe(true); + expect( + isTrustedOffscreenStorageSender( + { + id: extensionId, + url: "https://example.com", + tab: { id: 1 } as chrome.tabs.Tab, + }, + extensionId, + offscreenUrl, + ), + ).toBe(false); + expect( + isTrustedOffscreenStorageSender( + { + id: extensionId, + url: "chrome-extension://cap-test/popup.html", + }, + extensionId, + offscreenUrl, + ), + ).toBe(false); + }); + + it("reports a failed bridge write to the caller", async () => { + (globalThis as { chrome?: unknown }).chrome = { + runtime: { + sendMessage: ( + _message: unknown, + callback: (response: unknown) => void, + ) => callback({ ok: false, error: "Storage full" }), + }, + }; + await expect( + requestStorage({ + target: "storage-bridge", + type: "set", + area: "local", + items: { key: "value" }, + }), + ).rejects.toThrow("Storage full"); + }); + + it("reports a failed direct storage write", async () => { + (globalThis as { chrome?: unknown }).chrome = { + runtime: { lastError: { message: "Storage full" } }, + storage: { + local: { + set: (_items: unknown, callback: () => void) => callback(), + }, + }, + }; + await expect( + executeStorageBridgeRequest({ + target: "storage-bridge", + type: "set", + area: "local", + items: { key: "value" }, + }), + ).rejects.toThrow("Storage full"); + }); +}); diff --git a/apps/chrome-extension/src/shared/storage-bridge.ts b/apps/chrome-extension/src/shared/storage-bridge.ts new file mode 100644 index 00000000000..9b6d27a4324 --- /dev/null +++ b/apps/chrome-extension/src/shared/storage-bridge.ts @@ -0,0 +1,117 @@ +export type StorageBridgeRequest = + | { + target: "storage-bridge"; + type: "get"; + area: "local" | "session"; + keys: string[]; + } + | { + target: "storage-bridge"; + type: "set"; + area: "local" | "session"; + items: Record; + } + | { + target: "storage-bridge"; + type: "remove"; + area: "local" | "session"; + keys: string[] | string; + }; + +export type StorageBridgeResponse = + | { ok: true; items?: Record } + | { ok: false; error: string }; + +const isStringArray = (value: unknown): value is string[] => + Array.isArray(value) && value.every((entry) => typeof entry === "string"); + +export const isStorageBridgeRequest = ( + message: unknown, +): message is StorageBridgeRequest => { + if (!message || typeof message !== "object") return false; + const candidate = message as Record; + if ( + candidate.target !== "storage-bridge" || + (candidate.area !== "local" && candidate.area !== "session") + ) { + return false; + } + if (candidate.type === "get") return isStringArray(candidate.keys); + if (candidate.type === "set") { + return ( + !!candidate.items && + typeof candidate.items === "object" && + !Array.isArray(candidate.items) + ); + } + if (candidate.type === "remove") { + return typeof candidate.keys === "string" || isStringArray(candidate.keys); + } + return false; +}; + +export const isTrustedOffscreenStorageSender = ( + sender: Pick, + extensionId: string, + offscreenUrl: string, +) => + sender.id === extensionId && + sender.url === offscreenUrl && + sender.tab === undefined; + +const storageError = () => + chrome.runtime?.lastError?.message ?? "Extension storage request failed"; + +export const executeStorageBridgeRequest = ( + request: StorageBridgeRequest, +): Promise> => { + const area = chrome.storage?.[request.area]; + if (!area) + return Promise.reject(new Error("Extension storage is unavailable")); + return new Promise((resolve, reject) => { + if (request.type === "get") { + area.get(request.keys, (items) => { + if (chrome.runtime?.lastError) reject(new Error(storageError())); + else resolve({ ok: true, items }); + }); + return; + } + if (request.type === "set") { + area.set(request.items, () => { + if (chrome.runtime?.lastError) reject(new Error(storageError())); + else resolve({ ok: true }); + }); + return; + } + area.remove(request.keys, () => { + if (chrome.runtime?.lastError) reject(new Error(storageError())); + else resolve({ ok: true }); + }); + }); +}; + +export const requestStorage = ( + request: StorageBridgeRequest, +): Promise> => { + if (chrome.storage?.[request.area]) { + return executeStorageBridgeRequest(request); + } + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage(request, (response: unknown) => { + if (chrome.runtime.lastError) { + reject(new Error(storageError())); + return; + } + if (!response || typeof response !== "object" || !("ok" in response)) { + reject(new Error("Extension storage bridge did not respond")); + return; + } + const result = response as StorageBridgeResponse; + if (!result.ok) { + reject(new Error(result.error)); + return; + } + resolve(result); + }); + }); +}; diff --git a/apps/chrome-extension/src/shared/storage.ts b/apps/chrome-extension/src/shared/storage.ts index 124636644cf..4c8e564b416 100644 --- a/apps/chrome-extension/src/shared/storage.ts +++ b/apps/chrome-extension/src/shared/storage.ts @@ -1,3 +1,4 @@ +import { requestStorage } from "./storage-bridge"; import { RECORDING_STATE_KEY, SHARED_UI_STATE_KEY } from "./storage-keys"; import type { BootstrapData, @@ -44,15 +45,23 @@ export type MediaAccessState = { // retried successfully or the entry is pruned. export type FailedRecording = { sessionId: string; + cameraSessionId?: string; videoId: string | null; shareUrl: string | null; mimeType: string; + cameraMimeType?: string; subpath: string | null; + cameraSubpath?: string; + cameraOffsetMs?: number; durationMs: number; width: number | null; height: number | null; fps: number | null; + cameraWidth?: number; + cameraHeight?: number; + cameraFps?: number; totalBytes: number; + cameraTotalBytes?: number; createdAt: number; message: string | null; }; @@ -118,35 +127,61 @@ const withKeyLock = (key: string, task: () => Promise): Promise => { return run; }; -const getLocal = (keys: string[]) => - new Promise>((resolve) => { - chrome.storage.local.get(keys, (items) => resolve(items)); - }); - -const setLocal = (items: Record) => - new Promise((resolve) => { - chrome.storage.local.set(items, resolve); - }); - -const removeLocal = (keys: string[] | string) => - new Promise((resolve) => { - chrome.storage.local.remove(keys, resolve); +const getLocal = async (keys: string[]) => + ( + await requestStorage({ + target: "storage-bridge", + type: "get", + area: "local", + keys, + }) + ).items ?? {}; + +const setLocal = async (items: Record) => { + await requestStorage({ + target: "storage-bridge", + type: "set", + area: "local", + items, }); +}; -const getSession = (keys: string[]) => - new Promise>((resolve) => { - chrome.storage.session.get(keys, (items) => resolve(items)); +const removeLocal = async (keys: string[] | string) => { + await requestStorage({ + target: "storage-bridge", + type: "remove", + area: "local", + keys, }); +}; -const setSession = (items: Record) => - new Promise((resolve) => { - chrome.storage.session.set(items, resolve); +const getSession = async (keys: string[]) => + ( + await requestStorage({ + target: "storage-bridge", + type: "get", + area: "session", + keys, + }) + ).items ?? {}; + +const setSession = async (items: Record) => { + await requestStorage({ + target: "storage-bridge", + type: "set", + area: "session", + items, }); +}; -const removeSession = (keys: string[] | string) => - new Promise((resolve) => { - chrome.storage.session.remove(keys, resolve); +const removeSession = async (keys: string[] | string) => { + await requestStorage({ + target: "storage-bridge", + type: "remove", + area: "session", + keys, }); +}; export const loadSettings = async () => { const result = await getLocal([SETTINGS_KEY]); @@ -281,7 +316,23 @@ const isFailedRecording = (value: unknown): value is FailedRecording => { typeof candidate.sessionId === "string" && typeof candidate.mimeType === "string" && typeof candidate.totalBytes === "number" && - typeof candidate.createdAt === "number" + typeof candidate.createdAt === "number" && + (candidate.cameraSessionId === undefined || + typeof candidate.cameraSessionId === "string") && + (candidate.cameraMimeType === undefined || + typeof candidate.cameraMimeType === "string") && + (candidate.cameraSubpath === undefined || + typeof candidate.cameraSubpath === "string") && + (candidate.cameraOffsetMs === undefined || + typeof candidate.cameraOffsetMs === "number") && + (candidate.cameraWidth === undefined || + typeof candidate.cameraWidth === "number") && + (candidate.cameraHeight === undefined || + typeof candidate.cameraHeight === "number") && + (candidate.cameraFps === undefined || + typeof candidate.cameraFps === "number") && + (candidate.cameraTotalBytes === undefined || + typeof candidate.cameraTotalBytes === "number") ); }; @@ -332,13 +383,20 @@ export const removeFailedRecording = (sessionId: string) => // subpath, dimensions) instead of a download-only one. export type LiveRecordingManifest = { sessionId: string; + cameraSessionId?: string; videoId: string; shareUrl: string; mimeType: string; + cameraMimeType?: string; subpath: string; + cameraSubpath?: string; + cameraOffsetMs?: number; width: number; height: number; fps: number; + cameraWidth?: number; + cameraHeight?: number; + cameraFps?: number; startedAt: number; }; @@ -358,7 +416,21 @@ const isLiveRecordingManifest = ( typeof candidate.width === "number" && typeof candidate.height === "number" && typeof candidate.fps === "number" && - typeof candidate.startedAt === "number" + typeof candidate.startedAt === "number" && + (candidate.cameraSessionId === undefined || + typeof candidate.cameraSessionId === "string") && + (candidate.cameraMimeType === undefined || + typeof candidate.cameraMimeType === "string") && + (candidate.cameraSubpath === undefined || + typeof candidate.cameraSubpath === "string") && + (candidate.cameraOffsetMs === undefined || + typeof candidate.cameraOffsetMs === "number") && + (candidate.cameraWidth === undefined || + typeof candidate.cameraWidth === "number") && + (candidate.cameraHeight === undefined || + typeof candidate.cameraHeight === "number") && + (candidate.cameraFps === undefined || + typeof candidate.cameraFps === "number") ); }; diff --git a/apps/desktop/src-tauri/src/clip_thumbnails.rs b/apps/desktop/src-tauri/src/clip_thumbnails.rs index 65860a5ad13..e4e0ac657d1 100644 --- a/apps/desktop/src-tauri/src/clip_thumbnails.rs +++ b/apps/desktop/src-tauri/src/clip_thumbnails.rs @@ -1,14 +1,19 @@ -use std::{path::Path, sync::LazyLock}; +use std::sync::LazyLock; use cap_project::{RecordingMetaInner, StudioRecordingMeta}; use tokio::sync::Semaphore; use crate::editor_window::WindowEditorInstance; -const THUMB_MAX_WIDTH: u32 = 240; -const THUMB_JPEG_QUALITY: u8 = 70; -const SEEK_DECODE_PACKET_LIMIT: usize = 4096; -const SEEK_DECODE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +#[path = "../../../../crates/editor/src/clip_thumbnails_shared.rs"] +mod clip_thumbnails_shared; + +use clip_thumbnails_shared::decode_clip_thumbnail; +#[cfg(test)] +use clip_thumbnails_shared::{ + SEEK_DECODE_PACKET_LIMIT, decode_clip_thumbnail_with_budget, persist_clip_thumbnail, + thumbnail_frame_reaches_target, +}; static THUMBNAIL_SEMAPHORE: LazyLock = LazyLock::new(|| Semaphore::new(4)); @@ -71,222 +76,6 @@ pub async fn get_clip_thumbnail( Ok(cache_path.to_string_lossy().into_owned()) } -fn decode_clip_thumbnail(input: &Path, time: f64, output: &Path) -> Result<(), String> { - decode_clip_thumbnail_with_budget( - input, - time, - output, - SEEK_DECODE_PACKET_LIMIT, - SEEK_DECODE_TIMEOUT, - ) -} - -fn decode_clip_thumbnail_with_budget( - input: &Path, - time: f64, - output: &Path, - packet_limit: usize, - timeout: std::time::Duration, -) -> Result<(), String> { - use ffmpeg::rescale::{Rescale, TIME_BASE}; - - let mut ictx = - ffmpeg::format::input(input).map_err(|e| format!("Failed to open video: {e}"))?; - - let stream = ictx - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("No video stream found")?; - let stream_index = stream.index(); - let stream_time_base = stream.time_base(); - let stream_start = match stream.start_time() { - ffmpeg::ffi::AV_NOPTS_VALUE => 0, - timestamp => timestamp, - }; - let target_timestamp = ((time * 1_000_000.0) as i64) - .rescale((1, 1_000_000), stream_time_base) - .saturating_add(stream_start); - - let mut decoder = ffmpeg::codec::context::Context::from_parameters(stream.parameters()) - .map_err(|e| e.to_string())? - .decoder() - .video() - .map_err(|e| e.to_string())?; - - let src_width = decoder.width(); - let src_height = decoder.height(); - if src_width == 0 || src_height == 0 { - return Err("Invalid video dimensions".to_string()); - } - - let scale = (THUMB_MAX_WIDTH as f32 / src_width as f32).min(1.0); - let target_width = ((src_width as f32 * scale).round() as u32).max(1); - let target_height = ((src_height as f32 * scale).round() as u32).max(1); - - let mut scaler = ffmpeg::software::scaling::context::Context::get( - decoder.format(), - src_width, - src_height, - ffmpeg::format::Pixel::RGB24, - target_width, - target_height, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ) - .map_err(|e| e.to_string())?; - - if time > 0.0 { - let position_us = (time * 1_000_000.0) as i64; - let seek_target = target_timestamp.rescale(stream_time_base, TIME_BASE); - decoder.flush(); - ictx.seek(seek_target, ..seek_target) - .map_err(|e| format!("Failed to seek to {position_us}us: {e}"))?; - } - - let mut frame = ffmpeg::frame::Video::empty(); - let mut decoded = ffmpeg::frame::Video::empty(); - let mut got_frame = false; - let mut reached_target = false; - let mut decoder_finished = false; - let mut packets_tried = 0usize; - let decode_started = std::time::Instant::now(); - - 'outer: for (packet_stream, packet) in ictx.packets() { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - if packet_stream.index() != stream_index { - continue; - } - - packets_tried += 1; - - if decoder.send_packet(&packet).is_err() { - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - continue; - } - - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - reached_target = true; - break 'outer; - } - } - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(ffmpeg::Error::Eof) => { - decoder_finished = true; - break 'outer; - } - Err(e) => { - if packets_tried >= packet_limit { - return Err(format!("Failed to decode frame: {e}")); - } - break; - } - } - } - - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - } - - if !reached_target && !decoder_finished { - decoder - .send_eof() - .map_err(|e| format!("Failed to flush decoder: {e}"))?; - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - break; - } - } - Err(ffmpeg::Error::Eof) => break, - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(e) => return Err(format!("Failed to flush decoder: {e}")), - } - } - } - - if !got_frame { - return Err("No decodable frames found".to_string()); - } - - let mut rgb_frame = ffmpeg::frame::Video::empty(); - scaler - .run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - let width = rgb_frame.width() as usize; - let height = rgb_frame.height() as usize; - let src_stride = rgb_frame.stride(0); - let dst_stride = width * 3; - if src_stride < dst_stride { - return Err(format!( - "Unexpected RGB stride: src_stride={src_stride}, expected >= {dst_stride}" - )); - } - let mut img_buffer = vec![0u8; height * dst_stride]; - for y in 0..height { - let src_slice = &rgb_frame.data(0)[y * src_stride..y * src_stride + dst_stride]; - img_buffer[y * dst_stride..(y + 1) * dst_stride].copy_from_slice(src_slice); - } - - use image::ImageEncoder; - - let mut jpeg_bytes = Vec::new(); - image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, THUMB_JPEG_QUALITY) - .write_image( - &img_buffer, - width as u32, - height as u32, - image::ExtendedColorType::Rgb8, - ) - .map_err(|e| format!("Failed to encode thumbnail: {e}"))?; - - persist_clip_thumbnail(output, &jpeg_bytes) -} - -fn thumbnail_frame_reaches_target(frame: &ffmpeg::frame::Video, target_timestamp: i64) -> bool { - frame - .timestamp() - .or_else(|| frame.pts()) - .is_none_or(|timestamp| timestamp >= target_timestamp) -} - -fn persist_clip_thumbnail(output: &Path, jpeg_bytes: &[u8]) -> Result<(), String> { - use std::io::Write; - - let parent = output.parent().unwrap_or_else(|| Path::new(".")); - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create thumbnail directory: {e}"))?; - - let mut staged = tempfile::NamedTempFile::new_in(parent) - .map_err(|e| format!("Failed to stage thumbnail: {e}"))?; - staged - .write_all(jpeg_bytes) - .map_err(|e| format!("Failed to write thumbnail: {e}"))?; - staged - .persist(output) - .map_err(|e| format!("Failed to persist thumbnail: {e}"))?; - - Ok(()) -} - #[cfg(test)] mod persistence_tests { use super::*; diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index cebe7d9c065..1c85c5bce2f 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -6,10 +6,9 @@ use cap_enc_ffmpeg::{ }; use cap_media_info::{AudioInfo, FFRational, Pixel, VideoInfo, ensure_even}; use cap_project::{ - AudioMeta, ClipConfiguration, CursorEvents, CursorMeta, Cursors, InstantRecordingMeta, - MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, RecordingMeta, - RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, - TimelineConfiguration, TimelineSegment, VideoMeta, XY, + AudioMeta, ClipConfiguration, Cursors, InstantRecordingMeta, MultipleSegment, MultipleSegments, + Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SingleSegment, + StudioRecordingMeta, StudioRecordingStatus, TimelineConfiguration, TimelineSegment, VideoMeta, }; use ffmpeg::{ ChannelLayout, @@ -17,11 +16,10 @@ use ffmpeg::{ format::{self as avformat}, }; use image::ImageEncoder; -use relative_path::{Component as RelativeComponent, RelativePathBuf}; +use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ - collections::HashMap, path::{Path, PathBuf}, str::FromStr, }; @@ -38,9 +36,6 @@ use crate::{ const VIDEO_IMPORT_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "wmv", "m4v", "flv"]; const IMAGE_IMPORT_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff"]; -const AUDIO_IMPORT_EXTENSIONS: &[&str] = &["ogg", "m4a", "mp3", "wav", "aac", "flac"]; -const KEYBOARD_IMPORT_EXTENSIONS: &[&str] = &["bin", "json"]; -const CURSOR_EVENTS_IMPORT_EXTENSIONS: &[&str] = &["json"]; const MAX_IMAGE_DIMENSION: u32 = 16_384; #[derive(Serialize, Deserialize, Type, Clone, Debug)] @@ -161,89 +156,6 @@ fn is_cap_project_path(path: &Path) -> bool { path.is_dir() && path.join("recording-meta.json").is_file() } -fn normalized_metadata_relative_path( - path: &RelativePathBuf, - asset_kind: &str, -) -> Result { - let normalized = path.as_str().replace('\\', "/"); - let path = RelativePathBuf::from(normalized); - let raw = path.as_str(); - if raw.is_empty() - || raw.starts_with('/') - || raw.contains(':') - || path - .components() - .any(|component| matches!(component, RelativeComponent::ParentDir)) - { - return Err(format!( - "Invalid {asset_kind} path in recording metadata: {raw}" - )); - } - - Ok(path) -} - -fn source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result, String> { - let source_relative_path = normalized_metadata_relative_path(source_relative_path, asset_kind)?; - - if !has_supported_extension(Path::new(source_relative_path.as_str()), allowed_extensions) { - return Err(format!( - "Unsupported {asset_kind} file type: {}", - source_relative_path.as_str() - )); - } - - let source_path = source_relative_path.to_path(source_project_path); - if !source_path.is_file() { - return Ok(None); - } - - let source_root = source_project_path - .canonicalize() - .map_err(|e| format!("Failed to resolve source project path: {e}"))?; - let canonical_source_path = source_path - .canonicalize() - .map_err(|e| format!("Failed to resolve {asset_kind} path: {e}"))?; - - if !canonical_source_path.starts_with(&source_root) { - return Err(format!( - "{asset_kind} path escapes source project: {}", - source_relative_path.as_str() - )); - } - - Ok(Some(canonical_source_path)) -} - -fn required_source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result { - source_asset_path( - source_project_path, - source_relative_path, - asset_kind, - allowed_extensions, - )? - .ok_or_else(|| { - format!( - "Missing {asset_kind} file: {}", - source_relative_path.to_path(source_project_path).display() - ) - }) -} - -fn legacy_cursor_relative_path(path: &str) -> Result { - normalized_metadata_relative_path(&RelativePathBuf::from(path), "cursor image") -} - fn editor_project_path_from_window(window: &Window) -> Result { let CapWindowId::Editor { id } = CapWindowId::from_str(window.label()).map_err(|e| e.to_string())? @@ -331,42 +243,6 @@ fn full_timeline_for_segments( .collect() } -fn get_source_video_duration_secs( - source_meta: &RecordingMeta, - video: &VideoMeta, -) -> Result { - let source_path = required_source_asset_path( - &source_meta.project_path, - &video.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )?; - get_video_duration_secs(&source_path) -} - -fn full_timeline_for_source_segments( - source_meta: &RecordingMeta, - segments: &[MultipleSegment], -) -> Result, String> { - segments - .iter() - .enumerate() - .map(|(index, segment)| { - let duration = get_source_video_duration_secs(source_meta, &segment.display)?; - Ok(TimelineSegment { - recording_clip: index as u32, - timescale: 1.0, - start: 0.0, - end: duration, - name: None, - speed_audio_mode: None, - hide_cursor: None, - volume: None, - }) - }) - .collect() -} - fn ensure_project_timeline<'a>( config: &'a mut ProjectConfiguration, project_path: &Path, @@ -440,600 +316,6 @@ fn unique_segment_dir( } } -fn relative_file_extension(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .extension() - .and_then(|ext| ext.to_str()) - .filter(|ext| !ext.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn relative_file_name(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .file_name() - .and_then(|name| name.to_str()) - .filter(|name| !name.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn unique_file_name(dir: &Path, preferred: &str) -> String { - let sanitized = sanitize_filename(preferred); - let sanitized = if sanitized.is_empty() { - "file".to_string() - } else { - sanitized - }; - - let path = Path::new(&sanitized); - let stem = path - .file_stem() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .unwrap_or("file") - .to_string(); - let extension = path - .extension() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .map(|value| value.to_string()); - - let mut counter = 0; - loop { - let candidate = if counter == 0 { - sanitized.clone() - } else if let Some(extension) = &extension { - format!("{stem}-{counter}.{extension}") - } else { - format!("{stem}-{counter}") - }; - - if !dir.join(&candidate).exists() { - return candidate; - } - - counter += 1; - } -} - -fn copy_file_to_relative_path( - source_path: &Path, - target_project_path: &Path, - target_relative_path: &RelativePathBuf, -) -> Result<(), String> { - let target_path = target_relative_path.to_path(target_project_path); - - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create import directory: {e}"))?; - } - - std::fs::copy(source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy {}: {e}", source_path.display())) -} - -fn copy_video_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &VideoMeta, - target_relative_dir: &str, - name: &str, - required: bool, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )? - else { - if required { - return Err(format!( - "Missing video file: {}", - source.path.to_path(source_project_path).display() - )); - } - return Ok(None); - }; - - let can_decode = probe_video_can_decode(&source_path) - .map_err(|e| format!("Cannot decode video {}: {e}", source_path.display()))?; - if !can_decode { - if required { - return Err(format!("Unsupported video file: {}", source_path.display())); - } - return Ok(None); - } - - let extension = relative_file_extension(&source.path, "mp4"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_audio_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &AudioMeta, - target_relative_dir: &str, - name: &str, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "audio", - AUDIO_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let extension = relative_file_extension(&source.path, "ogg"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_keyboard_path( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, -) -> Result, String> { - if let Some(source_relative_path) = &source_segment.keyboard { - let file_name = - relative_file_name(source_relative_path, cap_project::KEYBOARD_EVENTS_FILE_NAME); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(&file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - }; - - let Some(display_dir) = source_segment.display.path.parent() else { - return Ok(None); - }; - - for file_name in [ - cap_project::KEYBOARD_EVENTS_FILE_NAME, - cap_project::LEGACY_KEYBOARD_EVENTS_FILE_NAME, - ] { - let source_relative_path = display_dir.join(file_name); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - } - - Ok(None) -} - -fn normalize_cursors_to_correct(cursors: &mut Cursors) -> &mut HashMap { - if let Cursors::Old(old) = cursors { - let converted = old - .iter() - .map(|(id, path)| { - ( - id.clone(), - CursorMeta { - image_path: RelativePathBuf::from(path.as_str()), - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ) - }) - .collect(); - *cursors = Cursors::Correct(converted); - } - - match cursors { - Cursors::Correct(map) => map, - Cursors::Old(_) => unreachable!(), - } -} - -fn unique_cursor_id( - cursors: &HashMap, - import_token: &str, - source_id: &str, -) -> String { - let source_id = if source_id.is_empty() { - "cursor" - } else { - source_id - }; - let base = format!("{import_token}-{source_id}"); - if !cursors.contains_key(&base) { - return base; - } - - let mut counter = 1; - loop { - let candidate = format!("{base}-{counter}"); - if !cursors.contains_key(&candidate) { - return candidate; - } - counter += 1; - } -} - -fn copy_source_cursor_images( - source_meta: &RecordingMeta, - source_cursors: &Cursors, - target_project_path: &Path, - target_cursors: &mut Cursors, - import_token: &str, -) -> Result, String> { - let target_cursor_dir = target_project_path.join("content").join("cursors"); - std::fs::create_dir_all(&target_cursor_dir) - .map_err(|e| format!("Failed to create cursor directory: {e}"))?; - - let target_cursors = normalize_cursors_to_correct(target_cursors); - let mut id_map = HashMap::new(); - - match source_cursors { - Cursors::Correct(source_map) => { - for (source_id, cursor) in source_map { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &cursor.image_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&cursor.image_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: cursor.hotspot, - shape: cursor.shape, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - Cursors::Old(source_map) => { - for (source_id, source_path) in source_map { - let source_relative_path = legacy_cursor_relative_path(source_path)?; - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&source_relative_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - } - - Ok(id_map) -} - -fn copy_cursor_events_path( - source_meta: &RecordingMeta, - source_relative_path: &RelativePathBuf, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result, String> { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "cursor events", - CURSOR_EVENTS_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!("{target_relative_dir}/cursor.json")); - let target_path = target_relative_path.to_path(target_project_path); - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create cursor event directory: {e}"))?; - } - - if cursor_id_map.is_empty() { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - return Ok(Some(target_relative_path)); - } - - match CursorEvents::load_from_file(&source_path) { - Ok(mut events) => { - for event in &mut events.moves { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - for event in &mut events.clicks { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - - let file = std::fs::File::create(&target_path) - .map_err(|e| format!("Failed to create cursor event file: {e}"))?; - serde_json::to_writer_pretty(file, &events) - .map_err(|e| format!("Failed to write cursor event file: {e}"))?; - } - Err(_) => { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - } - } - - Ok(Some(target_relative_path)) -} - -fn single_segment_to_multiple(segment: &SingleSegment) -> MultipleSegment { - MultipleSegment { - display: segment.display.clone(), - camera: segment.camera.clone(), - mic: segment.audio.clone(), - system_audio: None, - cursor: segment.cursor.clone(), - keyboard: None, - display_notch: None, - } -} - -fn studio_segments_for_import(studio_meta: &StudioRecordingMeta) -> Vec { - match studio_meta { - StudioRecordingMeta::SingleSegment { segment } => { - vec![single_segment_to_multiple(segment)] - } - StudioRecordingMeta::MultipleSegments { inner } => inner.segments.clone(), - } -} - -fn source_timeline_segments_for_import( - source_meta: &RecordingMeta, - source_segments: &[MultipleSegment], -) -> Result, String> { - let source_config = ProjectConfiguration::load(&source_meta.project_path).unwrap_or_default(); - let Some(timeline) = source_config.timeline else { - return full_timeline_for_source_segments(source_meta, source_segments); - }; - - if timeline.segments.is_empty() { - return full_timeline_for_source_segments(source_meta, source_segments); - } - - let mut duration_cache = HashMap::new(); - let mut imported_segments = Vec::new(); - - for segment in timeline.segments { - let source_index = segment.recording_clip; - let Some(source_segment) = source_segments.get(source_index as usize) else { - continue; - }; - - let max_duration = if let Some(duration) = duration_cache.get(&source_index) { - *duration - } else { - let duration = get_source_video_duration_secs(source_meta, &source_segment.display)?; - duration_cache.insert(source_index, duration); - duration - }; - - if max_duration <= 0.0 { - continue; - } - - let raw_start = if segment.start.is_finite() { - segment.start - } else { - 0.0 - }; - let raw_end = if segment.end.is_finite() { - segment.end - } else { - max_duration - }; - let start = raw_start.clamp(0.0, max_duration); - let end = raw_end.clamp(start, max_duration); - if end <= start { - continue; - } - - imported_segments.push(TimelineSegment { - recording_clip: source_index, - timescale: if segment.timescale.is_finite() && segment.timescale > 0.0 { - segment.timescale - } else { - 1.0 - }, - start, - end, - name: None, - speed_audio_mode: segment.speed_audio_mode, - hide_cursor: segment.hide_cursor, - volume: segment.volume, - }); - } - - if imported_segments.is_empty() { - full_timeline_for_source_segments(source_meta, source_segments) - } else { - Ok(imported_segments) - } -} - -fn copy_source_segment( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result { - let display = copy_video_meta( - &source_meta.project_path, - target_project_path, - &source_segment.display, - target_relative_dir, - "display", - true, - )? - .ok_or_else(|| "Missing display video".to_string())?; - - let camera = source_segment - .camera - .as_ref() - .map(|camera| { - copy_video_meta( - &source_meta.project_path, - target_project_path, - camera, - target_relative_dir, - "camera", - false, - ) - }) - .transpose()? - .flatten(); - - let mic = source_segment - .mic - .as_ref() - .map(|mic| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - mic, - target_relative_dir, - "mic", - ) - }) - .transpose()? - .flatten(); - - let system_audio = source_segment - .system_audio - .as_ref() - .map(|system_audio| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - system_audio, - target_relative_dir, - "system-audio", - ) - }) - .transpose()? - .flatten(); - - let cursor = source_segment - .cursor - .as_ref() - .map(|cursor| { - copy_cursor_events_path( - source_meta, - cursor, - target_project_path, - target_relative_dir, - cursor_id_map, - ) - }) - .transpose()? - .flatten(); - - let keyboard = copy_keyboard_path( - source_meta, - source_segment, - target_project_path, - target_relative_dir, - )?; - - Ok(MultipleSegment { - display, - camera, - mic, - system_audio, - cursor, - keyboard, - display_notch: source_segment.display_notch, - }) -} - fn get_video_stream_info( input: &avformat::context::Input, ) -> Result<(usize, VideoInfo), ImportError> { @@ -1771,121 +1053,21 @@ async fn append_cap_project_to_editor_project( ) -> Result { let source_meta = RecordingMeta::load_for_project(&source_project_path) .map_err(|e| format!("Failed to load source project metadata: {e}"))?; - - let RecordingMetaInner::Studio(source_studio_meta) = &source_meta.inner else { - return match &source_meta.inner { - RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { - append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()) - .await - } - RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { - Err("Source Cap project is still recording".to_string()) - } - RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { - Err(format!("Source Cap project failed: {error}")) - } - RecordingMetaInner::Studio(_) => unreachable!(), - }; - }; - - let source_segments = studio_segments_for_import(source_studio_meta); - if source_segments.is_empty() { - return Err("Source Cap project has no recording segments".to_string()); - } - - let source_timeline = source_timeline_segments_for_import(&source_meta, &source_segments)?; - let source_cursors = match source_studio_meta.as_ref() { - StudioRecordingMeta::MultipleSegments { inner } => Some(&inner.cursors), - StudioRecordingMeta::SingleSegment { .. } => None, - }; - - let mut target_meta = RecordingMeta::load_for_project(&target_project_path) - .map_err(|e| format!("Failed to load target project metadata: {e}"))?; - let mut config = ProjectConfiguration::load(&target_project_path).unwrap_or_default(); - let existing_segments = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - inner.segments.clone() - }; - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - - let (base_index, copied_segments, source_to_target_index) = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - let base_index = inner.segments.len() as u32; - let import_token = format!("import-{}", uuid::Uuid::new_v4().simple()); - let cursor_id_map = if let Some(source_cursors) = source_cursors { - copy_source_cursor_images( - &source_meta, - source_cursors, - &target_project_path, - &mut inner.cursors, - &import_token, - )? - } else { - HashMap::new() - }; - - let mut copied_segments = Vec::new(); - let mut source_to_target_index = HashMap::new(); - - for (source_index, source_segment) in source_segments.iter().enumerate() { - let target_index = base_index + copied_segments.len() as u32; - let (_, target_relative_dir) = - unique_segment_dir(&target_project_path, target_index) - .map_err(|e| format!("Failed to create imported segment directory: {e}"))?; - let copied_segment = copy_source_segment( - &source_meta, - source_segment, - &target_project_path, - &target_relative_dir, - &cursor_id_map, - )?; - - inner.segments.push(copied_segment.clone()); - copied_segments.push(copied_segment); - source_to_target_index.insert(source_index as u32, target_index); + match &source_meta.inner { + RecordingMetaInner::Studio(_) => cap_editor::append_studio_cap_project_to_editor_project( + target_project_path, + source_project_path, + ), + RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { + append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()).await } - - (base_index, copied_segments, source_to_target_index) - }; - - if copied_segments.is_empty() { - return Err("Source Cap project has no importable recording segments".to_string()); - } - - { - let timeline = - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - for source_segment in source_timeline { - let Some(target_index) = source_to_target_index.get(&source_segment.recording_clip) - else { - continue; - }; - - timeline.segments.push(TimelineSegment { - recording_clip: *target_index, - timescale: source_segment.timescale, - start: source_segment.start, - end: source_segment.end, - name: None, - speed_audio_mode: source_segment.speed_audio_mode, - hide_cursor: source_segment.hide_cursor, - volume: source_segment.volume, - }); + RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { + Err("Source Cap project is still recording".to_string()) + } + RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { + Err(format!("Source Cap project failed: {error}")) } } - - add_clip_configs(&mut config, base_index, &copied_segments); - - target_meta - .save_for_project() - .map_err(|e| format!("Failed to save project metadata: {e:?}"))?; - config - .write(&target_project_path) - .map_err(|e| format!("Failed to save project config: {e}"))?; - - Ok(copied_segments.len()) } #[tauri::command] @@ -2142,125 +1324,4 @@ mod tests { assert_eq!((referenced.width(), referenced.height()), (12, 8)); assert_eq!(referenced.pts(), Some(93)); } - - #[test] - fn source_asset_path_allows_file_inside_source_project() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.mp4"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_allows_backslash_separators() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = - RelativePathBuf::from("content\\segments\\segment-0\\display.mp4"); - let source_path = RelativePathBuf::from("content/segments/segment-0/display.mp4") - .to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_rejects_parent_traversal() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("../secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn source_asset_path_rejects_absolute_path() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("/tmp/secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn legacy_cursor_relative_path_rejects_windows_absolute_path() { - let error = legacy_cursor_relative_path("C:\\Users\\me\\cursor.png").unwrap_err(); - - assert!(error.contains("Invalid cursor image path")); - } - - #[test] - fn source_asset_path_rejects_unsupported_extension() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.txt"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Unsupported video file type")); - } - - #[cfg(unix)] - #[test] - fn source_asset_path_rejects_symlink_escape() { - let source_project = tempfile::tempdir().unwrap(); - let external_dir = tempfile::tempdir().unwrap(); - let external_file = external_dir.path().join("cursor.png"); - std::fs::write(&external_file, b"cursor").unwrap(); - - let source_relative_path = RelativePathBuf::from("content/cursors/cursor.png"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::os::unix::fs::symlink(&external_file, &source_path).unwrap(); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("cursor image path escapes source project")); - } } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 87cae9cedb8..39f5aeb047b 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6819,15 +6819,7 @@ fn apply_screen_recording_presentation_defaults( } pub fn default_project_config() -> ProjectConfiguration { - let mut config = ProjectConfiguration::default(); - - apply_screen_recording_presentation_defaults(&mut config, None, false, None); - - if config.background.rounding <= f64::EPSILON { - config.background.rounding = DEFAULT_SCREEN_RECORDING_BACKGROUND_ROUNDING_PERCENT; - } - - config + cap_editor::default_screen_recording_project_config() } #[tauri::command] diff --git a/apps/desktop/src/routes/editor/CaptionsTab.tsx b/apps/desktop/src/routes/editor/CaptionsTab.tsx index ea3d58c0e02..219b20959ba 100644 --- a/apps/desktop/src/routes/editor/CaptionsTab.tsx +++ b/apps/desktop/src/routes/editor/CaptionsTab.tsx @@ -89,6 +89,10 @@ interface LanguageOption { } const MODEL_DOWNLOAD_STATUS_POLL_MS = 1000; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; +const webCaptionsEnabled = () => + (window as Window & { capWebEditorCaptionsEnabled?: boolean }) + .capWebEditorCaptionsEnabled === true; const MODEL_OPTIONS: ModelOption[] = [ { @@ -561,6 +565,25 @@ export function CaptionsTab(props: { }; onMount(async () => { + if (isWebEditor) { + const savedLanguage = localStorage.getItem( + "selectedTranscriptionLanguage", + ); + if ( + savedLanguage && + LANGUAGE_OPTIONS.some((option) => option.code === savedLanguage) + ) { + setSelectedLanguage(savedLanguage); + } + if (editorInstance?.recordings) { + setHasAudio( + editorInstance.recordings.segments.some( + (segment) => segment.mic !== null || segment.system_audio !== null, + ), + ); + } + return; + } try { unlistenDownloadProgress = await events.downloadProgress.listen( (event) => { @@ -777,257 +800,347 @@ export function CaptionsTab(props: { -
- - - options={availableModelOptions().map((model) => model.name)} - value={selectedModel()} - onChange={(value: string | null) => { - if (value) setSelectedModel(value); - }} - itemComponent={(props) => { - const model = availableModelOptions().find( - (option) => option.name === props.item.rawValue, - ); - - return ( - - as={KSelect.Item} - item={props.item} - > -
-
-
- - {model?.label ?? props.item.rawValue} - + +
+

+ Cap Pro captions use the same AssemblyAI transcription as your + shareable link. +

+ + + Upgrade to Cap Pro + + + + + + options={LANGUAGE_OPTIONS.map((option) => option.code)} + value={selectedLanguage()} + onChange={(value: string | null) => { + if (value) setSelectedLanguage(value); + }} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + { + LANGUAGE_OPTIONS.find( + (option) => option.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = LANGUAGE_OPTIONS.find( + (option) => option.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + +
+ + +
+ + + options={availableModelOptions().map((model) => model.name)} + value={selectedModel()} + onChange={(value: string | null) => { + if (value) setSelectedModel(value); + }} + itemComponent={(props) => { + const model = availableModelOptions().find( + (option) => option.name === props.item.rawValue, + ); + + return ( + + as={KSelect.Item} + item={props.item} + > +
+
+
+ + {model?.label ?? props.item.rawValue} + + + + + + +
- - - +
+ {model?.description} +
-
- {model?.description} -
+ + {model?.size} +
- - - {model?.size} - + + ); + }} + > + +
+
+ + {selectedModelOption()?.label || "Select a model"} + + + + +
- - ); - }} - > - -
-
- - {selectedModelOption()?.label || "Select a model"} - - - - +
+ {selectedModelOption()?.description} +
-
- {selectedModelOption()?.description} -
+ + {selectedModelOption()?.size} + -
- - - {selectedModelOption()?.size} - - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - as={KSelect.Listbox} /> - - - - + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + as={KSelect.Listbox} /> + + + + + + +

+ Parakeet caption models are unavailable on Intel Macs. Whisper + models remain available. +

+ -

- Parakeet caption models are unavailable on Intel Macs. Whisper - models remain available. + One time download to your system. All captions are stored locally.

- - -

- One time download to your system. All captions are stored locally. -

- - - - options={LANGUAGE_OPTIONS.map((l) => l.code)} - value={selectedLanguage()} - onChange={(value: string | null) => { - if (value) setSelectedLanguage(value); - }} - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - { - LANGUAGE_OPTIONS.find((l) => l.code === props.item.rawValue) - ?.label - } - - - )} - > - - class="truncate"> - {(state) => { - const language = LANGUAGE_OPTIONS.find( - (l) => l.code === state.selectedOption(), - ); - return {language?.label || "Select a language"}; - }} - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - - class="overflow-y-auto max-h-48" - as={KSelect.Listbox} - /> - - - - - - - - -
-
+ { + LANGUAGE_OPTIONS.find( + (l) => l.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = LANGUAGE_OPTIONS.find( + (l) => l.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + + + +
+ class="overflow-hidden w-full h-1.5 rounded-full bg-ed-ctl" + role="progressbar" + aria-valuemin="0" + aria-valuemax="100" + aria-valuenow={downloadPercent()} + > +
+
+

+ {downloadMessage() || + "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} +

-

- {downloadMessage() || - "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} -

-
+ +
+ } + > +
+ + -
- } - > -
- - - -
- - - - {selectedModelOption()?.label ?? "Caption"} model downloaded +
+ + + + {selectedModelOption()?.label ?? "Caption"} model downloaded + - - } - onClick={deleteModel} - disabled={ - isGenerating() || - isDownloading() || - deletingModel() === selectedModel() - } - > - {deletingModel() === selectedModel() ? "Deleting..." : "Delete"} - + } + onClick={deleteModel} + disabled={ + isGenerating() || + isDownloading() || + deletingModel() === selectedModel() + } + > + {deletingModel() === selectedModel() + ? "Deleting..." + : "Delete"} + +
-
- -
+ +
+
diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 8d1e8615992..dca28b63eb0 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -12,8 +12,9 @@ import { Tabs as KTabs } from "@kobalte/core/tabs"; import { createElementBounds } from "@solid-primitives/bounds"; import { createEventListenerMap } from "@solid-primitives/event-listener"; import { createQuery } from "@tanstack/solid-query"; -import { convertFileSrc } from "@tauri-apps/api/core"; +import { convertFileSrc, invoke } from "@tauri-apps/api/core"; import { appDataDir, resolveResource } from "@tauri-apps/api/path"; +import { open } from "@tauri-apps/plugin-dialog"; import { BaseDirectory, exists, @@ -87,6 +88,7 @@ import { AnimatedGradientEditor, copyAnimatedGradientConfig, } from "./AnimatedGradientEditor"; + import { AudioLibraryPanel } from "./AudioLibrary"; import { AUDIO_TRACK_BG_CLASS, @@ -153,6 +155,8 @@ import { import { formatTime } from "./utils"; import { ZoomModeHelper } from "./ZoomModeHelper"; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; + // Split out of the sidebar chunk: the captions tab is not visible at first // paint (Kobalte only mounts the selected tab), and its code is heavy. The // render site wraps it in a local so the chunk load stays inside @@ -302,6 +306,13 @@ type WallpaperOption = { const isCurrentDesktopBackgroundPath = (path: string | null | undefined) => { if (!path) return false; + if ( + isWebEditor && + /^content\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(png|jpg|webp|gif|bmp|tiff)$/.test( + path, + ) + ) + return true; const filename = path.split(/[\\/]/).pop(); if (!filename) return false; return ( @@ -1998,6 +2009,8 @@ function BackgroundConfig(props: { const assetsDir = `${editorInstance.path}/assets`; try { + if (isWebEditor) + return await invoke("webEditorStoredDesktopBackground"); const importedPrefix = `${CURRENT_DESKTOP_BACKGROUND_BASENAME}-`; let newest: { path: string; timestamp: number } | null = null; for (const entry of await readDir(assetsDir)) { @@ -2180,9 +2193,20 @@ function BackgroundConfig(props: { if (importingDesktopBackground()) return; setImportingDesktopBackground(true); try { - const path = await commands.importCurrentDesktopBackground( - editorInstance.path, - ); + let source = editorInstance.path; + if (isWebEditor) { + const selected = await open({ + filters: [ + { + name: "Images", + extensions: [...BACKGROUND_IMAGE_EXTENSIONS, "jpeg"], + }, + ], + }); + if (typeof selected !== "string") return; + source = selected; + } + const path = await commands.importCurrentDesktopBackground(source); const addingFromBlankBackground = isNoneBackground(); batch(() => { setCurrentDesktopBackgroundPath(path); @@ -2435,7 +2459,9 @@ function BackgroundConfig(props: {
- Use the wallpaper from your desktop + {isWebEditor + ? "Choose your desktop wallpaper" + : "Use the wallpaper from your desktop"} {importingDesktopBackground() ? "Importing..." - : "Import desktop background"} + : isWebEditor + ? "Choose desktop wallpaper" + : "Import desktop background"}
} @@ -3273,11 +3301,15 @@ function CameraConfig(props: { scrollRef: HTMLDivElement }) { - options={cameraBackgroundOptions(ostype() === "macos")} + options={cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + )} optionValue="value" optionTextValue="name" value={ - cameraBackgroundOptions(ostype() === "macos").find( + cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + ).find( (option) => option.value === (project.camera.backgroundBlur?.mode ?? "off"), diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index 68e60f5f6ac..79beccdd738 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -54,6 +54,7 @@ import { RESOLUTION_OPTIONS } from "./Header"; import { Dialog } from "./ui"; class SilentError extends Error {} +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; const EXPORT_CTA_CLASS = "flex w-full h-10 items-center justify-center gap-2 rounded-[10px] text-[13px] font-medium transition-colors outline-hidden focus-visible:ring-2 focus-visible:ring-ed-accent/40 disabled:opacity-50 disabled:cursor-not-allowed"; @@ -269,6 +270,8 @@ export function ExportPage() { if (disablesLinkExport() && _settings.exportTo === "link") ret.exportTo = "file"; + if (isWebEditor && _settings.exportTo === "clipboard") + ret.exportTo = "file"; if (shouldUseGifMode()) { if (!["720p", "1080p"].includes(_settings.resolution.value)) { @@ -842,8 +845,10 @@ export function ExportPage() { await refetchMeta(); setReuploading(!!meta().sharing); - const existingAuth = await authStore.get(); - if (!existingAuth) createSignInMutation(); + const existingAuth = isWebEditor + ? { user_id: "web" } + : await authStore.get(); + if (!existingAuth && !isWebEditor) createSignInMutation(); trackEvent("create_shareable_link_clicked", { resolution: settings.resolution, fps: settings.fps, @@ -962,13 +967,21 @@ export function ExportPage() { label: option.value === "link" && meta().sharing ? "Reupload" : option.label, icon: option.icon, - disabled: option.value === "link" && disablesLinkExport(), + disabled: + (option.value === "link" && disablesLinkExport()) || + (isWebEditor && option.value === "clipboard"), disabledReason: - option.value === "link" && disablesLinkExport() - ? cursorOnly() - ? "Cursor-only exports can only be saved to a file or clipboard" - : "Transparent exports can only be saved to a file or clipboard" - : undefined, + isWebEditor && option.value === "clipboard" + ? "Use File to download. Browser clipboard cannot paste exports as files" + : option.value === "link" && disablesLinkExport() + ? cursorOnly() + ? isWebEditor + ? "Cursor-only exports can only be saved to a file" + : "Cursor-only exports can only be saved to a file or clipboard" + : isWebEditor + ? "Transparent exports can only be saved to a file" + : "Transparent exports can only be saved to a file or clipboard" + : undefined, })); const formatOptions = () => @@ -1446,7 +1459,7 @@ export function ExportPage() {
- {settings.exportTo === "link" && !auth.data ? ( + {settings.exportTo === "link" && !auth.data && !isWebEditor ? ( + )} +
+ {phase === "idle" || (phase === "error" && !captured) ? ( +
+ + + +
+ ) : null} + {previewStream && ( +
+ + Download screen clip + + {captured.camera && downloadUrls.camera && ( + + Download camera clip + + )} +
+ )} +
+ {phase === "idle" || (phase === "error" && !captured) ? ( + + ) : null} + {phase === "recording" && ( + + )} + {phase === "paused" && ( + + )} + {(phase === "recording" || phase === "paused") && ( + + )} + {phase === "error" && captured && capturedCanImport && ( + + )} +
+
+
+ ); +} diff --git a/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx new file mode 100644 index 00000000000..8dcc32ab5f2 --- /dev/null +++ b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx @@ -0,0 +1,529 @@ +"use client"; + +import dynamic from "next/dynamic"; +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useRef, useState } from "react"; +import type { WebEditorCapImportProgress } from "@/lib/editor-cap-import-client"; +import type { EditorClipCapture } from "@/lib/editor-clip-recorder"; +import { + captureEditorLocalDraft, + clearEditorLocalDraft, + type EditorLocalDraft, + readEditorLocalDraft, +} from "@/lib/editor-local-draft"; +import type { WebEditorVideoImportProgress } from "@/lib/editor-video-import-client"; +import { EditorClipRecorder } from "./EditorClipRecorder"; +import { EditorHostBridge } from "./editor-host"; + +const UpgradeModal = dynamic( + () => + import("@/components/UpgradeModal").then((module) => module.UpgradeModal), + { ssr: false }, +); + +type Preparation = { id: string; status: "preparing" }; +type PreparationStatus = { + status: "preparing" | "ready" | "error" | "canceled" | "closed"; + sessionId?: string; +}; + +function isPreparation(value: unknown): value is Preparation { + return ( + typeof value === "object" && + value !== null && + "id" in value && + typeof value.id === "string" && + "status" in value && + value.status === "preparing" + ); +} + +function isPreparationStatus(value: unknown): value is PreparationStatus { + return ( + typeof value === "object" && + value !== null && + "status" in value && + ["preparing", "ready", "error", "canceled", "closed"].includes( + String(value.status), + ) + ); +} + +function waitForPoll(signal: AbortSignal) { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new Error("Editor preparation was canceled")); + return; + } + const timer = window.setTimeout(() => { + signal.removeEventListener("abort", canceled); + resolve(); + }, 500); + const canceled = () => { + window.clearTimeout(timer); + reject(new Error("Editor preparation was canceled")); + }; + signal.addEventListener("abort", canceled, { once: true }); + }); +} + +export function StudioEditorClient(props: { + videoId: string; + userId: string; + captionsEnabled: boolean; + savedAt: string | null; +}) { + const { videoId, userId, captionsEnabled, savedAt } = props; + const router = useRouter(); + const [sessionId, setSessionId] = useState(null); + const [error, setError] = useState(null); + const [recoveryConflict, setRecoveryConflict] = + useState(null); + const [restoringBrowserDraft, setRestoringBrowserDraft] = useState(false); + const [videoImport, setVideoImport] = useState< + WebEditorVideoImportProgress | WebEditorCapImportProgress | null + >(null); + const [recordClipOpen, setRecordClipOpen] = useState(false); + const [upgradeOpen, setUpgradeOpen] = useState(false); + const preparationRef = useRef(null); + const sessionRef = useRef(null); + const bridgeRef = useRef(null); + const iframeRef = useRef(null); + const restoreInProgressRef = useRef(false); + const savedAtRef = useRef(savedAt); + const captureDraftRef = useRef<() => boolean>(() => true); + const closedRef = useRef(false); + const restartAfterImport = useCallback(async () => { + const activeSession = sessionRef.current; + if (activeSession) { + const response = await fetch( + `/api/editor/sessions/${encodeURIComponent(activeSession)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", cache: "no-store" }, + ); + if (!response.ok && response.status !== 404) { + throw new Error( + "Imported clip was saved, but the editor could not restart", + ); + } + sessionRef.current = null; + } + window.location.reload(); + }, [videoId]); + + useEffect(() => { + const controller = new AbortController(); + savedAtRef.current = savedAt; + closedRef.current = false; + setSessionId(null); + setError(null); + setRecoveryConflict(null); + setRestoringBrowserDraft(false); + restoreInProgressRef.current = false; + setVideoImport(null); + setRecordClipOpen(false); + setUpgradeOpen(false); + let lastCapturedConfig: string | null = null; + let lastCapturedSavedAt: string | null = null; + const captureDraft = () => { + let serialized: string | null; + try { + const editorWindow = iframeRef.current?.contentWindow as + | (Window & { + capSolidEditor?: { unsavedProject?: () => string | null }; + }) + | null; + serialized = editorWindow?.capSolidEditor?.unsavedProject?.() ?? null; + } catch { + return false; + } + if (!serialized) return true; + if ( + serialized === lastCapturedConfig && + savedAtRef.current === lastCapturedSavedAt + ) + return true; + const captured = captureEditorLocalDraft( + window.localStorage, + userId, + videoId, + savedAtRef.current, + serialized, + ); + if (captured) { + lastCapturedConfig = serialized; + lastCapturedSavedAt = savedAtRef.current; + } + return captured; + }; + captureDraftRef.current = captureDraft; + const beforeUnload = (event: BeforeUnloadEvent) => { + if (captureDraft()) return; + event.preventDefault(); + event.returnValue = ""; + }; + const close = () => { + if (closedRef.current) return; + captureDraft(); + closedRef.current = true; + controller.abort(); + bridgeRef.current?.dispose(); + bridgeRef.current = null; + const activeSession = sessionRef.current; + const activePreparation = preparationRef.current; + sessionRef.current = null; + preparationRef.current = null; + if (activeSession) { + void fetch( + `/api/editor/sessions/${encodeURIComponent(activeSession)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + } else if (activePreparation) { + void fetch( + `/api/editor/preparations/${encodeURIComponent(activePreparation)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + } + }; + const prepare = async () => { + try { + const response = await fetch("/api/editor/preparations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ videoId }), + }); + if (!response.ok) throw new Error("Editor preparation could not start"); + const created: unknown = await response.json(); + if (!isPreparation(created)) { + throw new Error("Editor preparation response was invalid"); + } + preparationRef.current = created.id; + if (controller.signal.aborted) { + void fetch( + `/api/editor/preparations/${encodeURIComponent(created.id)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + preparationRef.current = null; + return; + } + const deadline = Date.now() + 5 * 60 * 1000; + while (!controller.signal.aborted && Date.now() < deadline) { + const statusResponse = await fetch( + `/api/editor/preparations/${encodeURIComponent(created.id)}?videoId=${encodeURIComponent(videoId)}`, + { signal: controller.signal }, + ); + if (!statusResponse.ok) { + throw new Error("Editor preparation status is unavailable"); + } + const status: unknown = await statusResponse.json(); + if (!isPreparationStatus(status)) { + throw new Error("Editor preparation status was invalid"); + } + if (status.status === "ready") { + if (!status.sessionId) { + throw new Error("Editor session was not returned"); + } + if (controller.signal.aborted) { + void fetch( + `/api/editor/sessions/${encodeURIComponent(status.sessionId)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + return; + } + sessionRef.current = status.sessionId; + preparationRef.current = null; + const draft = readEditorLocalDraft( + window.localStorage, + userId, + videoId, + ); + if (draft) { + const recovered = await fetch( + `/api/editor/sessions/${encodeURIComponent(status.sessionId)}/config`, + { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId, + config: draft.config, + expectedSavedAt: draft.baseSavedAt, + }), + signal: controller.signal, + }, + ); + if (recovered.status === 409) { + setRecoveryConflict(draft); + setError( + "This recording changed after the browser saved pending edits. Choose which version to open.", + ); + return; + } + if (!recovered.ok) + throw new Error( + "Pending edits are saved in this browser, but the editor could not restore them. Try again.", + ); + const result: unknown = await recovered.json(); + if ( + typeof result === "object" && + result !== null && + "savedAt" in result && + typeof result.savedAt === "string" + ) { + savedAtRef.current = result.savedAt; + } + clearEditorLocalDraft(window.localStorage, userId, videoId); + } + if (controller.signal.aborted) return; + setSessionId(status.sessionId); + return; + } + if (status.status !== "preparing") { + throw new Error("Editor preparation failed"); + } + await waitForPoll(controller.signal); + } + throw new Error("Editor preparation timed out"); + } catch (cause) { + if (!controller.signal.aborted) { + setError( + cause instanceof Error ? cause.message : "Editor is unavailable", + ); + } + } + }; + const timer = window.setTimeout(() => void prepare(), 0); + window.addEventListener("beforeunload", beforeUnload); + window.addEventListener("popstate", captureDraft); + window.addEventListener("pagehide", close); + return () => { + window.clearTimeout(timer); + window.removeEventListener("beforeunload", beforeUnload); + window.removeEventListener("popstate", captureDraft); + window.removeEventListener("pagehide", close); + close(); + captureDraftRef.current = () => true; + }; + }, [savedAt, userId, videoId]); + + const restoreBrowserDraft = useCallback(async () => { + const activeSession = sessionRef.current; + const draft = recoveryConflict; + if (!activeSession || !draft || restoreInProgressRef.current) return; + restoreInProgressRef.current = true; + setRestoringBrowserDraft(true); + try { + const configUrl = `/api/editor/sessions/${encodeURIComponent(activeSession)}/config`; + const revisionResponse = await fetch( + `${configUrl}?videoId=${encodeURIComponent(videoId)}`, + { cache: "no-store" }, + ).catch(() => null); + const revision: unknown = await revisionResponse + ?.json() + .catch(() => null); + if ( + !revisionResponse?.ok || + typeof revision !== "object" || + revision === null || + !("savedAt" in revision) || + (revision.savedAt !== null && typeof revision.savedAt !== "string") + ) { + setError("The latest saved edits could not be checked. Try again."); + return; + } + if (closedRef.current || sessionRef.current !== activeSession) return; + const response = await fetch(configUrl, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId, + config: draft.config, + expectedSavedAt: revision.savedAt, + }), + }).catch(() => null); + if (response?.status === 409) { + setError( + "This recording changed again while restoring browser edits. Choose which version to open.", + ); + return; + } + if (!response?.ok) { + setError("Browser edits could not be restored. Try again."); + return; + } + const saved: unknown = await response.json().catch(() => null); + if ( + typeof saved !== "object" || + saved === null || + !("savedAt" in saved) || + typeof saved.savedAt !== "string" + ) { + setError( + "Browser edits were saved, but the result could not be checked. Try again.", + ); + return; + } + if (closedRef.current || sessionRef.current !== activeSession) return; + savedAtRef.current = saved.savedAt; + clearEditorLocalDraft(window.localStorage, userId, videoId); + setRecoveryConflict(null); + setError(null); + setSessionId(activeSession); + } finally { + restoreInProgressRef.current = false; + setRestoringBrowserDraft(false); + } + }, [recoveryConflict, userId, videoId]); + + const onFrameLoad = useCallback( + (iframe: HTMLIFrameElement) => { + if (!sessionId || closedRef.current) return; + bridgeRef.current?.dispose(); + const bridge = new EditorHostBridge( + videoId, + sessionId, + userId, + (reason) => + router.push( + reason === "deleted" + ? "/dashboard/caps" + : `/s/${encodeURIComponent(videoId)}`, + ), + (cause) => { + captureDraftRef.current(); + setError(cause.message); + }, + (progress) => setVideoImport(progress), + () => setRecordClipOpen(true), + restartAfterImport, + captionsEnabled, + () => setUpgradeOpen(true), + (nextSavedAt) => { + savedAtRef.current = nextSavedAt; + }, + () => savedAtRef.current, + ); + bridgeRef.current = bridge; + void bridge.connect(iframe).catch((cause) => { + bridge.dispose(); + if (bridgeRef.current === bridge) { + captureDraftRef.current(); + setError( + cause instanceof Error ? cause.message : "Editor could not connect", + ); + } + }); + }, + [captionsEnabled, restartAfterImport, router, sessionId, userId, videoId], + ); + + if (error) { + return ( +
+

{error}

+ {recoveryConflict ? ( +
+ + +
+ ) : ( + + )} +
+ ); + } + if (!sessionId) { + return ( +
+ Preparing editor… +
+ ); + } + return ( +
+ + + +`; + +async function sendError( + page: Page, + state: { + message: string; + hasBrowserDraftConflict: boolean; + restoringBrowserDraft: boolean; + }, +) { + await page.evaluate((data) => { + const iframe = document.getElementById("editor") as HTMLIFrameElement; + iframe.contentWindow?.postMessage( + { kind: "cap-editor-error", version: 1, ...data }, + window.location.origin, + ); + }, state); +} + +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/test-editor-error") { + return new Response(hostHtml, { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + } + if (url.pathname === "/favicon.ico") + return new Response(null, { status: 204 }); + if (url.pathname.startsWith("/editor-solid/")) { + const staticPath = resolve( + editorPublic, + url.pathname.slice("/editor-solid/".length), + ); + if ( + staticPath.startsWith(`${editorPublic}${sep}`) && + (await Bun.file(staticPath).exists()) + ) { + const file = Bun.file(staticPath); + return new Response(file, { + headers: { "Content-Type": file.type }, + }); + } + return new Response("Missing editor asset", { status: 404 }); + } + return new Response("Not found", { status: 404 }); + }, +}); + +try { + await mkdir(screenshotRoot, { recursive: true }); + for (const [name, engine] of [ + ["chromium", chromium], + ["webkit", webkit], + ] as const) { + const browser = await engine.launch({ headless: true }); + try { + const page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + }); + const pageErrors: string[] = []; + const failedRequests: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("requestfailed", (request) => failedRequests.push(request.url())); + await page.goto(`${server.url}test-editor-error`); + const frame = page.frameLocator("#editor"); + await frame.getByRole("button", { name: "Export" }).waitFor(); + const skeletonBackground = await frame + .locator("#editor-root > div") + .evaluate((element) => getComputedStyle(element).backgroundColor); + await sendError(page, { + message: "Editor preparation could not start", + hasBrowserDraftConflict: false, + restoringBrowserDraft: false, + }); + await page.locator("body[data-ready=true]").waitFor(); + await frame + .getByRole("heading", { name: "Unable to Open Recording" }) + .waitFor(); + assert.equal( + await frame.getByRole("alert").textContent(), + "Editor preparation could not start", + ); + assert.equal( + await frame.getByRole("button", { name: "Try again" }).count(), + 1, + ); + const background = await frame + .locator("#editor-root > div") + .evaluate((element) => getComputedStyle(element).backgroundColor); + assert.equal(background, skeletonBackground); + assert.notEqual(background, "rgb(255, 255, 255)"); + await page.screenshot({ + path: join(screenshotRoot, `${name}-preparation-error.png`), + }); + await frame.getByRole("button", { name: "Try again" }).click(); + await page.locator("body[data-action=retry]").waitFor(); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: false, + }); + await frame + .getByRole("button", { name: "Restore browser edits" }) + .waitFor(); + assert.equal( + await frame.getByRole("button", { name: "Try again" }).count(), + 0, + ); + await page.screenshot({ + path: join(screenshotRoot, `${name}-browser-draft-conflict.png`), + }); + await frame + .getByRole("button", { name: "Restore browser edits" }) + .click(); + await page.locator("body[data-action=restore-browser]").waitFor(); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: true, + }); + assert.equal( + await frame + .getByRole("button", { name: "Restoring browser edits…" }) + .isDisabled(), + true, + ); + assert.equal( + await frame + .getByRole("button", { name: "Open latest saved" }) + .isDisabled(), + true, + ); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: false, + }); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await frame.getByRole("button", { name: "Open latest saved" }).click(); + await page.locator("body[data-action=open-latest]").waitFor(); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await frame.getByRole("button", { name: "Back to recording" }).click(); + await page.locator("body[data-action=back-to-recording]").waitFor(); + assert.deepEqual(pageErrors, []); + assert.deepEqual(failedRequests, []); + process.stdout.write( + `${JSON.stringify({ browser: name, background, skeletonBackground, pageErrors, failedRequests })}\n`, + ); + } finally { + await browser.close(); + } + } +} finally { + server.stop(true); +} diff --git a/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx index df939f84f64..fea7466e290 100644 --- a/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx +++ b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx @@ -88,6 +88,7 @@ export function StudioEditorClient(props: { const router = useRouter(); const [sessionId, setSessionId] = useState(null); const [error, setError] = useState(null); + const [errorFrameReady, setErrorFrameReady] = useState(false); const [recoveryConflict, setRecoveryConflict] = useState(null); const [restoringBrowserDraft, setRestoringBrowserDraft] = useState(false); @@ -100,6 +101,7 @@ export function StudioEditorClient(props: { const sessionRef = useRef(null); const bridgeRef = useRef(null); const iframeRef = useRef(null); + const errorFrameRef = useRef(null); const frameConnectRef = useRef<{ document: Document; sessionId: string; @@ -388,6 +390,92 @@ export function StudioEditorClient(props: { } }, [recoveryConflict, userId, videoId]); + const sendErrorToFrame = useCallback( + (iframe: HTMLIFrameElement) => { + const document = iframe.contentDocument; + if ( + !error || + !document || + document.readyState !== "complete" || + new URL(document.URL).pathname !== "/editor-solid/index.html" + ) + return; + iframe.contentWindow?.postMessage( + { + kind: "cap-editor-error", + version: 1, + message: error, + hasBrowserDraftConflict: recoveryConflict !== null, + restoringBrowserDraft, + }, + window.location.origin, + ); + }, + [error, recoveryConflict, restoringBrowserDraft], + ); + + useEffect(() => { + if (!error) { + setErrorFrameReady(false); + return; + } + bridgeRef.current?.dispose(); + bridgeRef.current = null; + frameConnectRef.current = null; + }, [error]); + + useEffect(() => { + if (errorFrameRef.current) sendErrorToFrame(errorFrameRef.current); + }, [sendErrorToFrame]); + + useEffect(() => { + if (!error) return; + const onErrorFrameMessage = (event: MessageEvent) => { + if ( + event.origin !== window.location.origin || + event.source !== errorFrameRef.current?.contentWindow || + typeof event.data !== "object" || + event.data === null + ) + return; + const message = event.data as Record; + if (message.version !== 1) return; + if (message.kind === "cap-editor-error-ready") { + setErrorFrameReady(true); + return; + } + if (message.kind !== "cap-editor-error-action") return; + if (message.action === "retry") { + window.location.reload(); + } else if (message.action === "back-to-recording") { + router.push(`/s/${encodeURIComponent(videoId)}`); + } else if ( + message.action === "restore-browser" && + recoveryConflict && + !restoringBrowserDraft + ) { + void restoreBrowserDraft(); + } else if ( + message.action === "open-latest" && + recoveryConflict && + !restoringBrowserDraft + ) { + clearEditorLocalDraft(window.localStorage, userId, videoId); + window.location.reload(); + } + }; + window.addEventListener("message", onErrorFrameMessage); + return () => window.removeEventListener("message", onErrorFrameMessage); + }, [ + error, + recoveryConflict, + restoringBrowserDraft, + restoreBrowserDraft, + router, + userId, + videoId, + ]); + const onFrameLoad = useCallback( (iframe: HTMLIFrameElement) => { const document = iframe.contentDocument; @@ -472,40 +560,55 @@ export function StudioEditorClient(props: { if (error) { return ( -
-

{error}

- {recoveryConflict ? ( -
- - +
+