diff --git a/.github/workflows/docker-build-editor-worker.yml b/.github/workflows/docker-build-editor-worker.yml new file mode 100644 index 00000000000..a56f11476a3 --- /dev/null +++ b/.github/workflows/docker-build-editor-worker.yml @@ -0,0 +1,106 @@ +name: Docker Build Editor Worker + +on: + pull_request: + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + push: + branches: + - main + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + +permissions: {} + +jobs: + build: + name: Build and test editor worker (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: amd64 + runner: ubuntu-24.04 + - platform: arm64 + runner: ubuntu-24.04-arm + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Set up Docker Buildx v3.12.0 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - name: Build editor worker image with Docker v6.19.2 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + file: apps/media-server/Dockerfile.editor + platforms: linux/${{ matrix.platform }} + load: true + tags: cap-editor-worker:verification + cache-from: type=gha,scope=editor-worker-${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=editor-worker-${{ matrix.platform }} + - name: Check native worker readiness + run: | + docker run --rm --network none \ + -e CAP_WEB_EDITOR_PUBLIC_ORIGIN=http://127.0.0.1:3457 \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification \ + -e 'import app from "./src/editor-worker-app.ts"; const response = await app.request("/health"); if (!response.ok) process.exit(1)' + - name: Exercise paired native editor routes + run: | + docker run --rm --network none --cpus 4 --memory 4g \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification test \ + src/__tests__/editor-worker-app.test.ts \ + src/__tests__/lib/editor-assets.test.ts \ + src/__tests__/lib/editor-clip-thumbnails.test.ts \ + src/__tests__/lib/editor-config-paths.test.ts \ + src/__tests__/lib/editor-image-assets.test.ts \ + src/__tests__/lib/editor-security.test.ts \ + src/__tests__/lib/editor-wallpapers.test.ts \ + src/__tests__/routes/editor-clips.test.ts \ + src/__tests__/routes/editor.test.ts + - name: Verify editor subprocess credential isolation + run: | + docker run --rm --network none \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + -e AWS_SECRET_ACCESS_KEY=test \ + --entrypoint bun cap-editor-worker:verification \ + src/__tests__/lib/editor-process-env-replay.ts + - name: Verify cursor-only MOV export + run: | + docker run --rm --network none --cpus 4 --memory 4g \ + --entrypoint bun cap-editor-worker:verification \ + src/__tests__/routes/editor-cursor-mov-replay.ts diff --git a/.github/workflows/recording-reliability.yml b/.github/workflows/recording-reliability.yml index 74838a71b43..f0e479e6726 100644 --- a/.github/workflows/recording-reliability.yml +++ b/.github/workflows/recording-reliability.yml @@ -6,6 +6,9 @@ on: paths: - "apps/web/**" - "apps/media-server/**" + - "packages/editor-solid-web/**" + - "packages/editor-cap-bundle/**" + - "packages/recorder-core/**" - "packages/database/**" - "packages/web-*/**" - "crates/recording/**" @@ -16,6 +19,9 @@ on: paths: - "apps/web/**" - "apps/media-server/**" + - "packages/editor-solid-web/**" + - "packages/editor-cap-bundle/**" + - "packages/recorder-core/**" - "packages/database/**" - "packages/web-*/**" - "crates/recording/**" @@ -30,17 +36,59 @@ concurrency: cancel-in-progress: true jobs: + browser-cap-import: + name: Web editor recording folder import (Chromium/WebKit) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Playwright browsers + run: bunx playwright install --with-deps chromium webkit + - name: Replay browser recording import + run: bun run --filter @cap/media-server test:browser-cap-import + + browser-paired-recording: + name: Web recorder paired capture (${{ matrix.browser }}) + strategy: + fail-fast: false + matrix: + include: + - browser: chromium + runner: ubuntu-24.04 + headed: false + - browser: webkit + runner: macos-15-intel + headed: true + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium + if: matrix.browser == 'chromium' + run: bunx playwright install --with-deps chromium + - name: Install WebKit + if: matrix.browser == 'webkit' + run: bunx playwright install webkit + - name: Replay paired web recorder capture + working-directory: apps/web + run: bun __tests__/web-recorder-paired-browser-replay.mjs ${{ matrix.browser }} + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + recording-contract: name: Recording ownership, preservation, and recovery runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - uses: ./.github/actions/setup-js - name: Verify recording contracts working-directory: apps/web run: | bun run vitest run \ + __tests__/editor-image-file.test.ts \ __tests__/unit/audio-level-publication.test.ts \ __tests__/unit/recording-output.test.ts \ __tests__/unit/recording-storage-lifecycle.test.ts \ diff --git a/Cargo.lock b/Cargo.lock index 4d08eb8ddc6..db5e3d86b58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1657,6 +1657,7 @@ dependencies = [ "futures", "image 0.25.8", "lru", + "relative-path", "ringbuf", "sentry", "serde", @@ -1667,6 +1668,7 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "uuid", "windows 0.60.0", "workspace-hack", ] diff --git a/Cargo.toml b/Cargo.toml index 8a9be0ffb64..e8d0146f07a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,18 @@ opt-level = 2 [profile.dev.package.cap-audio] opt-level = 2 +[profile.dev.package.cap-editor] +opt-level = 3 + +[profile.dev.package.image] +opt-level = 3 + +[profile.dev.package.png] +opt-level = 3 + +[profile.dev.package.flate2] +opt-level = 3 + # Optimize for smaller binary size [profile.release] panic = "unwind" diff --git a/apps/chrome-extension/e2e/overlay-ui.spec.ts b/apps/chrome-extension/e2e/overlay-ui.spec.ts index 307609c637a..c0e55a5846b 100644 --- a/apps/chrome-extension/e2e/overlay-ui.spec.ts +++ b/apps/chrome-extension/e2e/overlay-ui.spec.ts @@ -202,7 +202,7 @@ const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-close-e2e-")); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ `--disable-extensions-except=${extensionPath}`, `--load-extension=${extensionPath}`, @@ -379,9 +379,9 @@ const getClosedShadowNodeId = async ( for (let index = 0; index < attributes.length; index += 2) { if (attributes[index] !== attribute) continue; if (!classToken) return true; - return ( - attributes[index + 1]?.split(/\s+/).includes(classToken) === true - ); + return attribute === "class" + ? attributes[index + 1]?.split(/\s+/).includes(classToken) === true + : attributes[index + 1] === classToken; } return false; })?.nodeId ?? null @@ -484,7 +484,7 @@ test("idle bootstrap and tab activation never open the camera preview", async () } }); -test("dismissing or closing the panel also closes the idle camera preview", async () => { +test("closing extension UI removes the ready bar and camera preview", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer(); const extension = await launchExtensionContext(); @@ -502,30 +502,63 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn await targetPage.goto(`${mockServer.origin}/capture.html`); await targetPage.bringToFront(); - await sendServiceWorkerMessage(messengerPage, { + const openResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "open-recorder-panel", }); - + expect(openResponse).toMatchObject({ ok: true }); + const activeTabUrl = await worker.evaluate(async () => { + const tabs = await chrome.tabs.query({ + active: true, + currentWindow: true, + }); + return tabs[0]?.url; + }); + expect(activeTabUrl).toBe(targetPage.url()); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) + .poll(() => + worker.evaluate(async () => { + const items = await chrome.storage.session.get( + "cap-extension-shared-ui-state", + ); + const state = items["cap-extension-shared-ui-state"] as + | { panelOpen?: boolean } + | undefined; + return state?.panelOpen ?? false; + }), + ) .toBe(true); + + const devtools = await extension.context.newCDPSession(targetPage); + await expect + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - await targetPage.screenshot({ path: "test-results/close-ui-before.png" }); - await targetPage.mouse.click(400, 300); - + const closeResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); + expect(closeResponse).toMatchObject({ ok: true }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, @@ -537,36 +570,52 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn type: "open-recorder-panel", }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await panelFrame - .locator('button[aria-label="Close Cap and hide all recorder UI"]') - .click(); + const hideButton = await getClosedShadowElementBox( + devtools, + "aria-label", + "Hide recording bar", + ); + if (!hideButton) throw new Error("Hide recording bar button missing"); + await targetPage.mouse.click( + hideButton.x + hideButton.width / 2, + hideButton.y + hideButton.height / 2, + ); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); + expect(frameWithUrl(targetPage, "camera-preview.html")).not.toBeNull(); + await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - await targetPage.waitForTimeout(2_500); expect(frameWithUrl(targetPage, "camera-preview.html")).toBeNull(); - await targetPage.screenshot({ path: "test-results/close-ui-after.png" }); } finally { await extension.cleanup(); await mockServer.close(); @@ -594,13 +643,17 @@ test("an abandoned recording start clears without leaving controls behind", asyn target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); - const devtools = await extension.context.newCDPSession(targetPage); await writeStaleCreatingState(worker); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("creating"); await targetPage.waitForTimeout(500); @@ -608,7 +661,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); @@ -623,15 +676,19 @@ test("an abandoned recording start clears without leaving controls behind", asyn expect(statusResponse.status?.phase).toBe("idle"); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("idle"); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); expect( await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); } finally { @@ -640,7 +697,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn } }); -test("a failed recording start reopens the panel with the error", async () => { +test("a failed recording start shows the error on the recorded tab", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer({ failInstantRecordings: true }); const extension = await launchExtensionContext(); @@ -676,22 +733,32 @@ test("a failed recording start reopens the panel with the error", async () => { const statusResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "get-recording-status", - })) as { status?: { phase?: string } }; + })) as { status?: { phase?: string; message?: string } }; expect(statusResponse.status?.phase).toBe("error"); - // The panel should reopen in the page to show the failure. + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 10_000, - }) - .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await expect(panelFrame.getByText("Recording failed.")).toBeVisible({ - timeout: 10_000, - }); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-error-toast", + ), + ) + .not.toBeNull(); + const toastNodeId = await getClosedShadowNodeId( + devtools, + "class", + "cap-extension-recording-error-toast", + ); + if (!toastNodeId) throw new Error("recording error toast missing"); + const { outerHTML } = (await devtools.send("DOM.getOuterHTML", { + nodeId: toastNodeId, + })) as { outerHTML: string }; + expect(outerHTML).toContain("Recording failed."); + expect(outerHTML).toContain(statusResponse.status?.message); await targetPage.screenshot({ - path: "test-results/start-error-panel.png", + path: "test-results/start-error-toast.png", }); } finally { await extension.cleanup(); @@ -724,9 +791,7 @@ test("the countdown appears while recording setup is still pending", async () => type: "start-recording", mode: "fullscreen", }); - await expect - .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) - .toBe(true); + void startPromise.catch(() => undefined); const devtools = await extension.context.newCDPSession(targetPage); await expect @@ -737,9 +802,12 @@ test("the countdown appears while recording setup is still pending", async () => "class", "cap-extension-countdown", )) !== null, - { timeout: 1_000 }, + { timeout: 5_000 }, ) .toBe(true); + await expect + .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) + .toBe(true); await targetPage.screenshot({ path: "test-results/countdown-during-setup.png", }); @@ -869,11 +937,77 @@ test("recording controls stay stable and the camera resizes directly", async () target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + )) !== null, + { timeout: 15_000 }, + ) + .toBe(true); + await expect + .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + timeout: 10_000, }) .toBe(true); + const initialCameraBox = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + if (!initialCameraBox) throw new Error("camera preview missing"); + await targetPage.mouse.move( + initialCameraBox.x + initialCameraBox.width / 2, + initialCameraBox.y + initialCameraBox.height / 2, + ); + await expect + .poll(() => + getClosedShadowComputedStyle( + devtools, + "data-camera-resize-ne", + "opacity", + ), + ) + .toBe("1"); + const cameraResizeHandle = await getClosedShadowElementBox( + devtools, + "data-camera-resize-ne", + ); + if (!cameraResizeHandle) throw new Error("camera resize handle missing"); + await targetPage.screenshot({ + path: "test-results/camera-resize-handles.png", + }); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2, + cameraResizeHandle.y + cameraResizeHandle.height / 2, + ); + await targetPage.mouse.down(); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, + cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, + { steps: 8 }, + ); + await targetPage.mouse.up(); + await expect + .poll(async () => { + const resizedCamera = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + return ( + resizedCamera !== null && + resizedCamera.width - initialCameraBox.width > 50 && + resizedCamera.height - initialCameraBox.height > 50 + ); + }) + .toBe(true); + await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); + await targetPage.screenshot({ + path: "test-results/camera-resized.png", + }); const startResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", @@ -882,104 +1016,86 @@ test("recording controls stay stable and the camera resizes directly", async () })) as { ok: boolean }; expect(startResponse.ok).toBe(true); await expect - .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - const devtools = await extension.context.newCDPSession(targetPage); await expect .poll( async () => (await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", )) !== null, { timeout: 10_000 }, ) .toBe(true); + await expect + .poll(async () => { + const box = await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-active-recording-container", + ); + return box?.height ?? 0; + }) + .toBeGreaterThanOrEqual(47); const initialBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); if (!initialBox) throw new Error("recording bar missing"); expect(initialBox.x).toBeLessThanOrEqual(20); - expect(initialBox.width).toBeGreaterThan(100); - expect(initialBox.width).toBeLessThanOrEqual(128); - expect(initialBox.height).toBeLessThanOrEqual(44); - const handleBox = await getClosedShadowElementBox( - devtools, - "data-drag-handle", - ); - if (!handleBox) throw new Error("recording bar drag handle missing"); + expect(initialBox.width).toBeGreaterThan(70); + expect(initialBox.height).toBeGreaterThanOrEqual(47); const timeBox = await getClosedShadowElementBox( devtools, - "data-recording-time", + "class", + "cap-extension-recording-badge-time", ); - if (!timeBox || timeBox.width < 30) { + if (!timeBox || timeBox.width < 20) { throw new Error("recording time is not visible at rest"); } - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("0"); - await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) !== null, ) - .toBe("1"); - const actionsBox = await getClosedShadowElementBox( - devtools, - "data-recording-actions", - ); - if (!actionsBox) throw new Error("recording action capsule missing"); - const actionGap = actionsBox.x - (initialBox.x + initialBox.width); - expect(actionGap).toBeGreaterThanOrEqual(4); - expect(actionGap).toBeLessThanOrEqual(8); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "border-top-left-radius", - ), - ).toBe("14px"); + .toBe(true); const pauseBox = await getClosedShadowElementBox( devtools, - "data-recording-pause", + "aria-label", + "Pause recording", ); - if (!pauseBox) throw new Error("pause action missing on hover"); + if (!pauseBox) throw new Error("pause control missing on hover"); await targetPage.mouse.move( pauseBox.x + pauseBox.width / 2, pauseBox.y + pauseBox.height / 2, ); - await targetPage.waitForTimeout(300); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); await targetPage.screenshot({ path: "test-results/recording-bar-expanded.png", }); + const pauseClickBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Pause recording", + ); + if (!pauseClickBox) + throw new Error("pause control disappeared before click"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + pauseClickBox.x + pauseClickBox.width / 2, + pauseClickBox.y + pauseClickBox.height / 2, ); await expect .poll(async () => { @@ -990,16 +1106,25 @@ test("recording controls stay stable and the camera resizes directly", async () return response.status?.phase; }) .toBe("paused"); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); + await expect + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + )) !== null, + ) + .toBe(true); + const resumeBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + ); + if (!resumeBox) throw new Error("resume control missing"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + resumeBox.x + resumeBox.width / 2, + resumeBox.y + resumeBox.height / 2, ); await expect .poll(async () => { @@ -1011,13 +1136,13 @@ test("recording controls stay stable and the camera resizes directly", async () }) .toBe("recording"); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await targetPage.mouse.down(); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2 + 120, - handleBox.y + handleBox.height / 2 - 80, + initialBox.x + initialBox.width / 2 + 120, + initialBox.y + initialBox.height / 2 - 80, { steps: 8 }, ); await targetPage.mouse.up(); @@ -1027,7 +1152,7 @@ test("recording controls stay stable and the camera resizes directly", async () const movedBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); return ( movedBox !== null && @@ -1066,72 +1191,17 @@ test("recording controls stay stable and the camera resizes directly", async () await targetPage.mouse.move(760, 40); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ) - .toBe("0"); - await targetPage.waitForTimeout(500); - await targetPage.screenshot({ - path: "test-results/recording-bar-visible.png", - }); - - const initialCameraBox = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - if (!initialCameraBox) throw new Error("camera preview missing"); - await targetPage.mouse.move( - initialCameraBox.x + initialCameraBox.width / 2, - initialCameraBox.y + initialCameraBox.height / 2, - ); - await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-camera-resize-ne", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) === null, ) - .toBe("1"); - const cameraResizeHandle = await getClosedShadowElementBox( - devtools, - "data-camera-resize-ne", - ); - if (!cameraResizeHandle) throw new Error("camera resize handle missing"); - await targetPage.screenshot({ - path: "test-results/camera-resize-handles.png", - }); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2, - cameraResizeHandle.y + cameraResizeHandle.height / 2, - ); - await targetPage.mouse.down(); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, - cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, - { steps: 8 }, - ); - await targetPage.mouse.up(); - await expect - .poll(async () => { - const resizedCamera = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - return ( - resizedCamera !== null && - resizedCamera.width - initialCameraBox.width > 50 && - resizedCamera.height - initialCameraBox.height > 50 - ); - }) .toBe(true); - await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); await targetPage.screenshot({ - path: "test-results/camera-resized.png", + path: "test-results/recording-bar-visible.png", }); const stopResponse = (await sendServiceWorkerMessage(messengerPage, { diff --git a/apps/chrome-extension/e2e/recording-upload.spec.ts b/apps/chrome-extension/e2e/recording-upload.spec.ts index fcf73c51481..51f6901c0d3 100644 --- a/apps/chrome-extension/e2e/recording-upload.spec.ts +++ b/apps/chrome-extension/e2e/recording-upload.spec.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, @@ -27,13 +27,23 @@ type ChromeRuntimeResponse = }; type MockState = { + abortBodies: unknown[]; completeBodies: unknown[]; progressBodies: unknown[]; initiateBodies: unknown[]; presignBodies: unknown[]; uploadBytes: number[]; + uploadBytesBySubpath: Record; + inputParts: Record; + completedPartsBySubpath: Record; uploadHeaders: Record[]; videoId: string; + simulateSlowCameraUpload: boolean; + failCameraCompletion: boolean; + failAudioCompletion: boolean; + failInputCompletion: boolean; + failScreenCompletion: boolean; + cameraInitiateDelayMs: number; }; type ChromeGlobal = typeof globalThis & { @@ -55,7 +65,9 @@ type ChromeGlobal = typeof globalThis & { }; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); +const extensionPath = process.env.CAP_EXTENSION_E2E_DIR + ? path.resolve(process.env.CAP_EXTENSION_E2E_DIR) + : path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -103,62 +115,38 @@ const sendHtml = (response: ServerResponse, html: string) => { const animatedCapturePage = () => ` - - Cap E2E Capture Target - - + Cap E2E Capture Target - - +
+
+
+ +
Custom input
+
PIN widget
+ + `; const createMockCapServer = async () => { const state: MockState = { + abortBodies: [], completeBodies: [], progressBodies: [], initiateBodies: [], presignBodies: [], uploadBytes: [], + uploadBytesBySubpath: {}, + inputParts: {}, + completedPartsBySubpath: {}, uploadHeaders: [], videoId: `e2e-${Date.now()}`, + simulateSlowCameraUpload: false, + failCameraCompletion: false, + failAudioCompletion: false, + failInputCompletion: false, + failScreenCompletion: false, + cameraInitiateDelayMs: 0, }; const server = createServer(async (request, response) => { @@ -215,14 +203,35 @@ const createMockCapServer = async () => { request.method === "POST" && url.pathname === "/api/upload/multipart/initiate" ) { - state.initiateBodies.push(await parseJsonBody(request)); + const body = await parseJsonBody(request); + state.initiateBodies.push(body); + if ( + state.cameraInitiateDelayMs > 0 && + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "camera-upload.webm" + ) { + await new Promise((resolve) => + setTimeout(resolve, state.cameraInitiateDelayMs), + ); + } sendJson(response, 200, { - uploadId: "upload-e2e", + uploadId: `upload-e2e-${state.initiateBodies.length}`, provider: "s3", }); return; } + if ( + request.method === "POST" && + url.pathname === "/api/upload/multipart/abort" + ) { + state.abortBodies.push(await parseJsonBody(request)); + sendJson(response, 200, { success: true }); + return; + } + if ( request.method === "POST" && url.pathname === "/api/upload/multipart/presign-part" @@ -237,7 +246,7 @@ const createMockCapServer = async () => { ? body.partNumber : state.presignBodies.length; sendJson(response, 200, { - presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}`, + presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}?subpath=${encodeURIComponent(String(body && typeof body === "object" && "subpath" in body ? body.subpath : "unknown"))}`, provider: "s3", }); return; @@ -246,13 +255,33 @@ const createMockCapServer = async () => { if (request.method === "PUT" && url.pathname.startsWith("/mock-s3/")) { const body = await readRequestBody(request); state.uploadBytes.push(body.byteLength); + const subpath = url.searchParams.get("subpath") ?? "unknown"; + const partNumber = Number(url.pathname.slice("/mock-s3/part-".length)); + if ( + subpath === "input-events-upload.ndjson" && + Number.isSafeInteger(partNumber) + ) { + state.inputParts[partNumber] = body; + } + state.uploadBytesBySubpath[subpath] = + (state.uploadBytesBySubpath[subpath] ?? 0) + body.byteLength; state.uploadHeaders.push(request.headers); + if ( + state.simulateSlowCameraUpload && + subpath === "camera-upload.webm" + ) { + await new Promise((resolve) => + setTimeout(resolve, Math.ceil(body.byteLength / 1_250)), + ); + } response.writeHead(200, { "Access-Control-Allow-Origin": "*", "Access-Control-Expose-Headers": "ETag", ETag: `"etag-${state.uploadBytes.length}"`, }); response.end(); + state.completedPartsBySubpath[subpath] = + (state.completedPartsBySubpath[subpath] ?? 0) + 1; return; } @@ -260,10 +289,57 @@ const createMockCapServer = async () => { request.method === "POST" && url.pathname === "/api/upload/multipart/complete" ) { - state.completeBodies.push(await parseJsonBody(request)); + const body = await parseJsonBody(request); + state.completeBodies.push(body); + if ( + state.failCameraCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "camera-upload.webm" + ) { + sendJson(response, 400, { error: "Camera completion rejected" }); + return; + } + if ( + state.failAudioCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm" + ) { + sendJson(response, 400, { error: "Microphone completion rejected" }); + return; + } + if ( + state.failInputCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson" + ) { + sendJson(response, 400, { + error: "Input events completion rejected", + }); + return; + } + if ( + state.failScreenCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "raw-upload.webm" + ) { + sendJson(response, 400, { error: "Screen completion rejected" }); + return; + } sendJson(response, 200, { success: true, - processingStarted: true, + processingStarted: + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "raw-upload.webm", }); return; } @@ -333,19 +409,34 @@ const createMockCapServer = async () => { const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-extension-e2e-")); + const testExtensionPath = path.join(userDataDir, "extension"); + await cp(extensionPath, testExtensionPath, { recursive: true }); + const manifestPath = path.join(testExtensionPath, "manifest.json"); + const manifest: Record = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.commands = { + _execute_action: { + suggested_key: { + default: "Ctrl+Shift+Y", + mac: "Command+Shift+Y", + }, + }, + }; + await writeFile(manifestPath, JSON.stringify(manifest)); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, + `--disable-extensions-except=${testExtensionPath}`, + `--load-extension=${testExtensionPath}`, "--allow-http-screen-capture", "--auto-select-desktop-capture-source=Cap E2E Capture Target", "--auto-select-tab-capture-source-by-title=Cap E2E Capture Target", "--enable-usermedia-screen-capturing", "--autoplay-policy=no-user-gesture-required", "--use-fake-device-for-media-stream", - "--use-fake-ui-for-media-stream", + "--auto-accept-camera-and-microphone-capture", ], }); const cleanup = async () => { @@ -370,6 +461,48 @@ const getServiceWorker = async (context: BrowserContext) => { const getExtensionId = (worker: Awaited>) => new URL(worker.url()).host; +type DevToolsNode = { + nodeId: number; + attributes?: string[]; + children?: DevToolsNode[]; + shadowRoots?: DevToolsNode[]; +}; + +const findCameraPreview = (node: DevToolsNode): DevToolsNode | null => { + if (node.attributes?.includes("data-camera-preview")) return node; + for (const child of [...(node.children ?? []), ...(node.shadowRoots ?? [])]) { + const result = findCameraPreview(child); + if (result) return result; + } + return null; +}; + +const expectCameraPreviewOutOfCapture = async (page: Page) => { + await expect(page.locator("#cap-extension-recorder-overlay")).toHaveAttribute( + "data-cap-mounted", + "true", + ); + const session = await page.context().newCDPSession(page); + try { + await session.send("DOM.enable"); + await expect + .poll(async () => { + const document = (await session.send("DOM.getDocument", { + depth: -1, + pierce: true, + })) as { root: DevToolsNode }; + const node = findCameraPreview(document.root); + if (!node?.attributes) return true; + const styleIndex = node.attributes.indexOf("style"); + if (styleIndex < 0) return false; + return /visibility:\s*hidden/.test(node.attributes[styleIndex + 1]); + }) + .toBe(true); + } finally { + await session.detach(); + } +}; + const openExtensionMessengerPage = async ( context: BrowserContext, worker: Awaited>, @@ -382,6 +515,7 @@ const openExtensionMessengerPage = async ( const configureExtension = async ( worker: Awaited>, apiBaseUrl: string, + microphoneEnabled = false, ) => { await worker.evaluate( async ({ @@ -390,6 +524,7 @@ const configureExtension = async ( recordingMode, settingsKey, apiBaseUrl, + microphoneEnabled, }) => { const chromeApi = (globalThis as ChromeGlobal).chrome; await new Promise((resolve, reject) => { @@ -434,15 +569,15 @@ const configureExtension = async ( microphone: null, }, webcam: { - enabled: false, - deviceId: null, + enabled: true, + deviceId: "__cap_default_camera__", position: "bottom-left", size: 230, shape: "round", mirror: false, }, microphone: { - enabled: false, + enabled: microphoneEnabled, deviceId: null, }, systemAudio: { @@ -477,6 +612,7 @@ const configureExtension = async ( bootstrapKey: BOOTSTRAP_CACHE_KEY, recordingMode: RECORDING_MODE, settingsKey: SETTINGS_KEY, + microphoneEnabled, }, ); }; @@ -499,7 +635,11 @@ const sendServiceWorkerMessage = async ( }); }, message); -const expectSuccessfulUpload = async (page: Page, state: MockState) => { +const expectSuccessfulUpload = async ( + page: Page, + state: MockState, + inputSidecar = false, +) => { await expect .poll(async () => { const response = await sendServiceWorkerMessage(page, { @@ -511,20 +651,30 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { }) .toBe("completed"); - expect(state.initiateBodies).toHaveLength(1); + expect(state.initiateBodies).toHaveLength(inputSidecar ? 3 : 2); + expect(state.initiateBodies[0]).toMatchObject({ subpath: "raw-upload.webm" }); + expect(state.initiateBodies[1]).toMatchObject({ + subpath: "camera-upload.webm", + }); + if (inputSidecar) { + expect(state.initiateBodies[2]).toMatchObject({ + subpath: "input-events-upload.ndjson", + }); + } expect(state.presignBodies.length).toBeGreaterThanOrEqual(1); expect(state.uploadBytes.length).toBeGreaterThanOrEqual(1); expect( state.uploadBytes.reduce((total, bytes) => total + bytes, 0), ).toBeGreaterThan(0); - expect(state.completeBodies).toHaveLength(1); + expect(state.completeBodies).toHaveLength(inputSidecar ? 3 : 2); expect(state.progressBodies.length).toBeGreaterThanOrEqual(1); const completeBody = state.completeBodies[0]; expect(completeBody).toMatchObject({ videoId: state.videoId, - uploadId: "upload-e2e", - subpath: "raw-upload.webm", + uploadId: "upload-e2e-2", + subpath: "camera-upload.webm", + screenSubpath: "raw-upload.webm", }); expect( completeBody && @@ -534,26 +684,237 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { ? completeBody.parts.length : 0, ).toBeGreaterThanOrEqual(1); + const cameraOffsetMs = + completeBody && + typeof completeBody === "object" && + "cameraOffsetMs" in completeBody + ? completeBody.cameraOffsetMs + : null; + expect( + typeof cameraOffsetMs === "number" && Number.isFinite(cameraOffsetMs), + ).toBe(true); + if (inputSidecar) { + expect(state.completeBodies[1]).toMatchObject({ + subpath: "input-events-upload.ndjson", + screenSubpath: "raw-upload.webm", + }); + expect( + state.uploadBytesBySubpath["input-events-upload.ndjson"], + ).toBeGreaterThan(0); + } + expect(state.completeBodies[inputSidecar ? 2 : 1]).toMatchObject({ + subpath: "raw-upload.webm", + uploadId: "upload-e2e-1", + }); + expect(state.uploadBytesBySubpath["camera-upload.webm"]).toBeGreaterThan(0); + expect(state.uploadBytesBySubpath["raw-upload.webm"]).toBeGreaterThan(0); }; +const readLiveCameraOffset = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + if (!Array.isArray(manifests)) { + throw new Error(`Live manifest storage is ${typeof manifests}`); + } + const offset = (manifests[0] as { cameraOffsetMs?: unknown } | undefined) + ?.cameraOffsetMs; + if (typeof offset !== "number") { + throw new Error( + `Live manifests: ${manifests.length}; first camera offset: ${typeof offset}`, + ); + } + return offset; + }); + +const readLiveMicrophoneOffset = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + const sources = Array.isArray(manifests) + ? ( + manifests[0] as { + audioSources?: Array<{ kind: string; offsetMs: number }>; + } + )?.audioSources + : undefined; + const offset = sources?.find((source) => source.kind === "mic")?.offsetMs; + if (typeof offset !== "number") { + throw new Error("The live microphone offset is unavailable"); + } + return offset; + }); + +const readLiveCameraSessionId = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + const sessionId = Array.isArray(manifests) + ? (manifests[0] as { cameraSessionId?: unknown } | undefined) + ?.cameraSessionId + : null; + if (typeof sessionId !== "string") { + throw new Error("The live camera spool session is unavailable"); + } + return sessionId; + }); + +const deleteCameraSpoolSession = async ( + worker: Awaited>, + sessionId: string, +) => + worker.evaluate(async (sessionId) => { + const request = indexedDB.open("cap-recording-spool", 1); + const database = await new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + const read = database.transaction("chunks", "readonly"); + const keysRequest = read + .objectStore("chunks") + .index("by-session") + .getAllKeys(IDBKeyRange.only(sessionId)); + const keys = await new Promise((resolve, reject) => { + keysRequest.onsuccess = () => resolve(keysRequest.result); + keysRequest.onerror = () => reject(keysRequest.error); + }); + const write = database.transaction(["sessions", "chunks"], "readwrite"); + const finished = new Promise((resolve, reject) => { + write.oncomplete = () => resolve(); + write.onerror = () => reject(write.error); + write.onabort = () => reject(write.error); + }); + write.objectStore("sessions").delete(sessionId); + const chunks = write.objectStore("chunks"); + for (const key of keys) chunks.delete(key); + await finished; + database.close(); + }, sessionId); + +const readFailedCameraRecovery = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-failed-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const recordings = items[key]; + if (!Array.isArray(recordings)) return null; + const failed = recordings[0] as + | { + videoId?: unknown; + cameraSessionId?: unknown; + cameraSubpath?: unknown; + cameraOffsetMs?: unknown; + cameraRetryUnavailable?: unknown; + } + | undefined; + return failed + ? { + videoId: failed.videoId, + cameraSessionId: failed.cameraSessionId, + cameraSubpath: failed.cameraSubpath, + cameraOffsetMs: failed.cameraOffsetMs, + cameraRetryUnavailable: failed.cameraRetryUnavailable, + } + : null; + }); + +const hasLiveRecordingManifest = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + return Array.isArray(items[key]) && items[key].length > 0; + }); + const startRecording = async ( context: BrowserContext, worker: Awaited>, apiBaseUrl: string, + mode: "fullscreen" | "tab" = RECORDING_MODE, + recordingMs = RECORDING_MS, + microphoneEnabled = false, ) => { - await configureExtension(worker, apiBaseUrl); + await configureExtension(worker, apiBaseUrl, microphoneEnabled); const messengerPage = await openExtensionMessengerPage(context, worker); const capturePage = await context.newPage(); await capturePage.goto(`${apiBaseUrl}/capture.html`); await capturePage.bringToFront(); + await expect + .poll(async () => { + const tabs = await worker.evaluate(() => + chrome.tabs.query({ active: true, lastFocusedWindow: true }), + ); + return tabs[0]?.url; + }) + .toBe(`${apiBaseUrl}/capture.html`); + if (mode === "tab") { + await worker.evaluate(() => { + const target = globalThis as typeof globalThis & { + capE2eActionClicks?: number; + }; + target.capE2eActionClicks = 0; + chrome.action.onClicked.addListener(() => { + target.capE2eActionClicks = (target.capE2eActionClicks ?? 0) + 1; + }); + }); + } + if (mode === "tab") { + await expect + .poll( + () => + worker.evaluate( + () => + ( + globalThis as typeof globalThis & { + capE2eActionClicks?: number; + } + ).capE2eActionClicks ?? 0, + ), + { + message: + "Click the Cap extension action icon in the test Chromium window", + timeout: 60_000, + }, + ) + .toBeGreaterThan(0); + } const startResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "start-recording", - mode: RECORDING_MODE, + mode, }); if (!startResponse.ok) { - throw new Error(startResponse.error); + const captures = await worker.evaluate( + () => + new Promise((resolve) => + chrome.tabCapture.getCapturedTabs(resolve), + ), + ); + throw new Error( + `${startResponse.error}; captured tabs: ${JSON.stringify(captures)}`, + ); } await expect @@ -567,7 +928,7 @@ const startRecording = async ( }) .toBe("recording"); - await capturePage.waitForTimeout(RECORDING_MS); + await capturePage.waitForTimeout(recordingMs); return { capturePage, messengerPage, @@ -589,23 +950,874 @@ test.describe("extension recording upload", () => { await mockServer?.close(); }); - test("records the selected display surface, uploads non-empty multipart data, and completes", async () => { + test("records a separate camera sidecar and hides its page overlay during display capture", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + await expectCameraPreviewOutOfCapture(capturePage); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + expect(typeof persistedCameraOffsetMs).toBe("number"); + + expect(mockServer.state.initiateBodies).toHaveLength(2); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + + await expectSuccessfulUpload(messengerPage, mockServer.state); + expect(mockServer.state.completeBodies[0]).toMatchObject({ + cameraOffsetMs: persistedCameraOffsetMs, + }); + }); + + test("uploads microphone audio independently of screen and camera in Chromium", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const persistedAudioOffsetMs = await readLiveMicrophoneOffset(worker); + expect(mockServer.state.initiateBodies).toHaveLength(3); + expect(mockServer.state.initiateBodies).toContainEqual( + expect.objectContaining({ + subpath: "mic-upload.webm", + contentType: "audio/webm", + }), + ); + + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toHaveLength(3); + const audioCompletion = mockServer.state.completeBodies.find( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ); + expect(audioCompletion).toMatchObject({ + videoId: mockServer.state.videoId, + subpath: "mic-upload.webm", + screenSubpath: "raw-upload.webm", + audioOffsetMs: persistedAudioOffsetMs, + }); + expect( + mockServer.state.uploadBytesBySubpath["mic-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["camera-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["raw-upload.webm"], + ).toBeGreaterThan(0); + }); + + test("keeps the microphone spool downloadable and retries all sources after audio upload failure", async () => { + test.setTimeout(120_000); if (!extension || !mockServer) throw new Error("Test harness did not start"); + mockServer.state.failAudioCompletion = true; const worker = await getServiceWorker(extension.context); const { messengerPage } = await startRecording( extension.context, worker, mockServer.origin, + "fullscreen", + RECORDING_MS, + true, ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + await expect( + uploadPage.getByRole("button", { name: "Download microphone" }), + ).toBeVisible(); + const micDownload = uploadPage.waitForEvent("download"); + await uploadPage + .getByRole("button", { name: "Download microphone" }) + .click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + const screenDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + "-screen.webm", + ); + + mockServer.state.failAudioCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ), + ).toHaveLength(2); + }); + + test("keeps uploaded microphone audio downloadable when screen completion fails", async () => { + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.failScreenCompletion = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); const stopResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "stop-recording", }); expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + expect(mockServer.state.completeBodies).toContainEqual( + expect.objectContaining({ subpath: "mic-upload.webm" }), + ); + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + const downloadButton = uploadPage.getByRole("button", { + name: "Download microphone", + }); + await expect(downloadButton).toBeVisible(); + const micDownload = uploadPage.waitForEvent("download"); + await downloadButton.click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ), + ).toHaveLength(2); + }); + + test("uploads a camera multipart part during a long recording on a constrained network", async () => { + test.setTimeout(180_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.simulateSlowCameraUpload = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + 0, + ); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + expect(mockServer.state.initiateBodies).toHaveLength(2); + await expect + .poll( + () => + mockServer?.state.completedPartsBySubpath["camera-upload.webm"] ?? 0, + { timeout: 120_000 }, + ) + .toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["camera-upload.webm"], + ).toBeGreaterThanOrEqual(5 * 1024 * 1024); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); await expectSuccessfulUpload(messengerPage, mockServer.state); + expect(mockServer.state.completeBodies[0]).toMatchObject({ + cameraOffsetMs: persistedCameraOffsetMs, + }); + }); + + test("offers separate clip downloads after a camera upload failure", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.failCameraCompletion = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ videoId: mockServer.state.videoId }); + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + await expect( + uploadPage.getByRole("button", { name: "Try again" }), + ).toBeVisible(); + const screenDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + "-screen.webm", + ); + const cameraDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download camera" }).click(); + expect((await cameraDownload).suggestedFilename()).toContain( + "-camera.webm", + ); + await worker.evaluate(async () => { + const key = "cap-extension-failed-recordings"; + const data = await new Promise>((resolve) => + chrome.storage.local.get([key], (items) => resolve(items)), + ); + const recordings = data[key]; + if (!Array.isArray(recordings) || !recordings[0]) { + throw new Error("The failed paired recording is unavailable"); + } + const failed = { ...(recordings[0] as Record) }; + delete failed.cameraSessionId; + failed.cameraRetryUnavailable = true; + await new Promise((resolve) => + chrome.storage.local.set({ [key]: [failed] }, () => resolve()), + ); + }); + await uploadPage.reload(); + await expect( + uploadPage.getByRole("button", { name: "Try again" }), + ).toHaveCount(0); + await expect( + uploadPage.getByRole("button", { name: "Download camera" }), + ).toHaveCount(0); + await expect( + uploadPage.getByRole("button", { name: "Download screen" }), + ).toBeVisible(); + expect(mockServer.state.completeBodies).toEqual([ + expect.objectContaining({ subpath: "camera-upload.webm" }), + ]); + }); + + test("recovers camera, microphone and screen spools after the offscreen recorder closes", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + const persistedAudioOffsetMs = await readLiveMicrophoneOffset(worker); + await worker.evaluate(() => chrome.offscreen.closeDocument()); + + const refreshResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-media-devices", + }); + expect(refreshResponse).toMatchObject({ ok: true }); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ + videoId: mockServer.state.videoId, + cameraSubpath: "camera-upload.webm", + cameraOffsetMs: persistedCameraOffsetMs, + }); + const optionsPage = await extension.context.newPage(); + await optionsPage.addInitScript(() => { + const target = globalThis as typeof globalThis & { + capE2eChunkReads?: number; + }; + target.capE2eChunkReads = 0; + const originalGetAll = IDBIndex.prototype.getAll; + IDBIndex.prototype.getAll = function ( + ...args: Parameters + ) { + if (this.name === "by-session") { + target.capE2eChunkReads = (target.capE2eChunkReads ?? 0) + 1; + } + return originalGetAll.apply(this, args); + }; + }); + await optionsPage.goto( + `chrome-extension://${getExtensionId(worker)}/options.html`, + ); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(1); + expect( + await optionsPage.evaluate( + () => + (globalThis as typeof globalThis & { capE2eChunkReads?: number }) + .capE2eChunkReads ?? 0, + ), + ).toBe(0); + const screenDownload = optionsPage.waitForEvent("download"); + await optionsPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + "-screen.webm", + ); + const cameraDownload = optionsPage.waitForEvent("download"); + await optionsPage.getByRole("button", { name: "Download camera" }).click(); + expect((await cameraDownload).suggestedFilename()).toContain( + "-camera.webm", + ); + const micDownload = optionsPage.waitForEvent("download"); + await optionsPage + .getByRole("button", { name: "Download microphone" }) + .click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + expect( + await optionsPage.evaluate( + () => + (globalThis as typeof globalThis & { capE2eChunkReads?: number }) + .capE2eChunkReads ?? 0, + ), + ).toBeGreaterThanOrEqual(3); + + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subpath: "camera-upload.webm", + cameraOffsetMs: persistedCameraOffsetMs, + }), + expect.objectContaining({ + subpath: "mic-upload.webm", + audioOffsetMs: persistedAudioOffsetMs, + }), + expect.objectContaining({ subpath: "raw-upload.webm" }), + ]), + ); + expect( + mockServer.state.uploadBytesBySubpath["camera-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["mic-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["raw-upload.webm"], + ).toBeGreaterThan(0); + expect(await readFailedCameraRecovery(worker)).toBeNull(); + expect(await hasLiveRecordingManifest(worker)).toBe(false); + }); + + test("never retries a screen-only upload after a crash loses the camera spool", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + const cameraSessionId = await readLiveCameraSessionId(worker); + await worker.evaluate(() => chrome.offscreen.closeDocument()); + await deleteCameraSpoolSession(worker, cameraSessionId); + const refreshResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-media-devices", + }); + expect(refreshResponse).toMatchObject({ ok: true }); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ + videoId: mockServer.state.videoId, + cameraRetryUnavailable: true, + }); + const optionsPage = await extension.context.newPage(); + await optionsPage.goto( + `chrome-extension://${getExtensionId(worker)}/options.html`, + ); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(1); + await expect( + optionsPage.getByRole("button", { name: "Retry upload" }), + ).toHaveCount(0); + await expect( + optionsPage.getByRole("button", { name: "Download screen" }), + ).toBeVisible(); + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ + ok: false, + error: expect.stringContaining( + "camera recording is unavailable to retry", + ), + }); + expect(mockServer.state.completeBodies).toHaveLength(0); + }); + + test("aborts both multipart sessions when Stop cancels camera upload setup", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.cameraInitiateDelayMs = 1_500; + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin); + const startPage = await openExtensionMessengerPage( + extension.context, + worker, + ); + const stopPage = await openExtensionMessengerPage( + extension.context, + worker, + ); + const capturePage = await extension.context.newPage(); + await capturePage.goto(`${mockServer.origin}/capture.html`); + await capturePage.bringToFront(); + const startPromise = sendServiceWorkerMessage(startPage, { + target: "service-worker", + type: "start-recording", + mode: "fullscreen", + }); + await expect.poll(() => mockServer?.state.initiateBodies.length).toBe(2); + const stopResponse = await sendServiceWorkerMessage(stopPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + const startResponse = await startPromise; + expect(startResponse).toMatchObject({ ok: false }); + expect(mockServer.state.abortBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ subpath: "raw-upload.webm" }), + expect.objectContaining({ subpath: "camera-upload.webm" }), + ]), + ); + expect(mockServer.state.completeBodies).toHaveLength(0); + }); + + test("keeps camera sidecar separate in current-tab capture", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 20_000, + ); + await expectCameraPreviewOutOfCapture(capturePage); + await capturePage.locator('input[type="password"]').click(); + await capturePage.keyboard.type("secret"); + await capturePage + .getByRole("textbox", { name: "Custom private field" }) + .click(); + await capturePage.keyboard.type("private"); + await capturePage.locator('[aria-label="Custom PIN widget"]').click(); + await capturePage.keyboard.type("1234"); + await capturePage.getByRole("button", { name: "Secret button" }).click(); + await capturePage.keyboard.type("secret"); + await capturePage.locator('[aria-label="Canvas secret widget"]').click(); + await capturePage.keyboard.type("pin"); + await capturePage.mouse.click(500, 400); + await capturePage.keyboard.press("Escape"); + await capturePage.goto(`${mockServer.origin}/capture.html?step=2`); + await capturePage.bringToFront(); + await capturePage.mouse.click(600, 500); + await capturePage.keyboard.press("ArrowRight"); + await capturePage.waitForTimeout(500); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state, true); + const inputText = Buffer.concat( + Object.entries(mockServer.state.inputParts) + .sort(([a], [b]) => Number(a) - Number(b)) + .map(([, bytes]) => bytes), + ).toString("utf8"); + const inputLines = inputText + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(inputLines[0]).toMatchObject({ version: 1 }); + const recordedKeys = inputLines + .filter((line) => line.kind === "keyDown") + .map((line) => line.key); + expect( + recordedKeys.every((key) => typeof key === "string" && key.length > 1), + ).toBe(true); + expect( + inputLines + .filter((line) => line.kind === "keyDown") + .every((line) => + typeof line.code === "string" + ? !/^(Key[A-Z]|Digit[0-9]|Numpad[0-9])$/.test(line.code) + : false, + ), + ).toBe(true); + expect(inputLines).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "down" }), + expect.objectContaining({ kind: "keyDown", key: "Escape" }), + expect.objectContaining({ kind: "keyDown", key: "ArrowRight" }), + ]), + ); + }); + + test("completes screen and camera when the tab input sidecar fails", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ subpath: "camera-upload.webm" }), + expect.objectContaining({ subpath: "input-events-upload.ndjson" }), + expect.objectContaining({ subpath: "raw-upload.webm" }), + ]), + ); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: "raw-upload.webm", + }); + const failed = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + return items["cap-extension-failed-recordings"]; + }); + expect(Array.isArray(failed) ? failed : []).toEqual([]); + const tabInputSession = await worker.evaluate(async () => { + const key = "cap-extension-tab-input-capture"; + const saved = await chrome.storage.session.get([key]); + return saved[key] ?? null; + }); + expect(tabInputSession).toBeNull(); + }); + + test("recovers a tab input sidecar before retrying the screen upload", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + mockServer.state.failScreenCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + const recovery = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + const failed = items["cap-extension-failed-recordings"]; + return Array.isArray(failed) ? failed[0] : null; + }); + if (!recovery || typeof recovery !== "object") + throw new Error("The input sidecar recovery record is missing"); + const failed = recovery as Record; + expect(failed.videoId).toBe(mockServer.state.videoId); + expect(failed.inputEventsRetryUnavailable).not.toBe(true); + expect(failed.inputEventsSessionId).toEqual(expect.any(String)); + expect(failed.inputEventsTotalBytes).toBeGreaterThan(0); + mockServer.state.failInputCompletion = false; + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length, + ).toBeGreaterThanOrEqual(2); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: "raw-upload.webm", + }); + }); + + test("retries intact screen media when tab input metadata is unavailable", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + mockServer.state.failScreenCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + const recovery = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + const failed = items["cap-extension-failed-recordings"]; + return Array.isArray(failed) ? failed[0] : null; + }); + if (!recovery || typeof recovery !== "object") + throw new Error("The failed screen recording is missing"); + expect(recovery).toMatchObject({ + videoId: mockServer.state.videoId, + inputEventsSessionId: expect.any(String), + }); + expect( + await worker.evaluate(async (videoId) => { + const key = "cap-extension-failed-recordings"; + const items = await chrome.storage.local.get([key]); + const failed: unknown = items[key]; + if (!Array.isArray(failed)) return false; + const updated = failed.map((entry: unknown) => { + if ( + !entry || + typeof entry !== "object" || + !("videoId" in entry) || + entry.videoId !== videoId + ) + return entry; + return { ...entry, inputEventsRetryUnavailable: true }; + }); + await chrome.storage.local.set({ [key]: updated }); + return true; + }, mockServer.state.videoId), + ).toBe(true); + const inputAttempts = mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length; + mockServer.state.failInputCompletion = false; + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: "raw-upload.webm", + }); + expect( + mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length, + ).toBe(inputAttempts); }); test("can complete two consecutive recording uploads without stale state", async () => { @@ -638,6 +1850,8 @@ test.describe("extension recording upload", () => { mockServer.state.initiateBodies = []; mockServer.state.presignBodies = []; mockServer.state.uploadBytes = []; + mockServer.state.uploadBytesBySubpath = {}; + mockServer.state.completedPartsBySubpath = {}; mockServer.state.uploadHeaders = []; mockServer.state.videoId = `e2e-${Date.now()}-second`; diff --git a/apps/chrome-extension/e2e/webcam-start.spec.ts b/apps/chrome-extension/e2e/webcam-start.spec.ts index dc1344b00f4..c66ade47f3d 100644 --- a/apps/chrome-extension/e2e/webcam-start.spec.ts +++ b/apps/chrome-extension/e2e/webcam-start.spec.ts @@ -301,9 +301,45 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "bootstrap", }); logs.push(`bootstrap: ${JSON.stringify(bootstrapResponse).slice(0, 200)}`); + const openResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + logs.push(`open panel: ${JSON.stringify(openResponse)}`); + await expect + .poll(() => + targetPage + .frames() + .some((frame) => frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Give the camera preview time to go live (frames flowing over WebRTC). await targetPage.waitForTimeout(6_000); + const previewFrame = targetPage + .frames() + .find((frame) => frame.url().includes("camera-preview.html")); + if (!previewFrame) throw new Error("Camera preview frame was not mounted"); + await expect + .poll(() => + previewFrame.evaluate( + () => document.querySelector("video")?.videoWidth ?? 0, + ), + ) + .toBeGreaterThan(0); + const pictureInPictureSupported = await previewFrame.evaluate( + () => document.pictureInPictureEnabled, + ); + if (pictureInPictureSupported) { + await previewFrame.locator("[data-pip-control]").click(); + await expect + .poll(() => + previewFrame.evaluate( + () => document.pictureInPictureElement !== null, + ), + ) + .toBe(true); + } await targetPage.screenshot({ path: "test-results/repro-before-start.png", }); @@ -342,6 +378,13 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "get-recording-status", })) as { status?: { phase?: string } }; expect(finalStatus.status?.phase).toBe("recording"); + await expect + .poll(() => + targetPage + .frames() + .every((frame) => !frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Content scripts read chrome.storage.session; without the service // worker widening the access level every call fails with this error. diff --git a/apps/chrome-extension/src/background/service-worker.ts b/apps/chrome-extension/src/background/service-worker.ts index 166072e3050..3ddfdbf211a 100644 --- a/apps/chrome-extension/src/background/service-worker.ts +++ b/apps/chrome-extension/src/background/service-worker.ts @@ -23,6 +23,7 @@ import { loadSettings, loadSharedRecordingState, loadSharedUiState, + loadTabInputCaptureSession, loadUploadProgressTabId, loadWebcamPreviewDismissed, registerOverlayToken, @@ -32,10 +33,16 @@ import { savePendingAuth, saveSettings, saveSharedRecordingState, + saveTabInputCaptureSession, saveUploadProgressTabId, saveWebcamPreviewDismissed, updateSharedUiState, } from "../shared/storage"; +import { + executeStorageBridgeRequest, + isStorageBridgeRequest, + isTrustedOffscreenStorageSender, +} from "../shared/storage-bridge"; import type { BootstrapData, CameraDevice, @@ -83,7 +90,6 @@ let activePreviewTabId: number | null = null; let pendingPreviewTabId: number | null = null; let offscreenDocumentCreation: Promise | null = null; let browserWindowFocused = true; -let externalCaptureAutoPipPending = false; let recordingStartInFlight: Promise | null = null; // Content scripts read the webcam "dismissed" flag and the cached preview @@ -314,10 +320,22 @@ const getTabStreamId = (tabId: number) => }); }); +const canUseDisplayPickerForTabCapture = (error: unknown) => + error instanceof Error && + /has not been invoked|activeTab permission/i.test(error.message); + const sendOverlayMessage = (tabId: number, message: OverlayMessage) => new Promise((resolve) => { - chrome.tabs.sendMessage(tabId, message, () => { - resolve(!chrome.runtime.lastError); + chrome.tabs.sendMessage(tabId, message, (response: unknown) => { + resolve( + !chrome.runtime.lastError && + !( + response && + typeof response === "object" && + "ok" in response && + response.ok === false + ), + ); }); }); @@ -485,21 +503,6 @@ const isBrowserCaptureSource = (source: RecordingCaptureSource) => source.displaySurface === "browser" || source.displaySurface === "tab"; -const isWindowCaptureSource = (source: RecordingCaptureSource) => - source.detectedMode === "window" || - source.displaySurface === "window" || - source.displaySurface === "application"; - -const isLikelyBrowserWindow = (source: RecordingCaptureSource) => { - if (!source.label) return false; - return /\b(google chrome|chrome|chromium|microsoft edge|edge|brave|arc|opera|vivaldi)\b/i.test( - source.label, - ); -}; - -const shouldAutoPipCaptureSource = (source: RecordingCaptureSource) => - isWindowCaptureSource(source) && !isLikelyBrowserWindow(source); - const isWebcamPreviewEnabled = (settings: ExtensionSettings) => Boolean(settings.webcam.enabled); @@ -777,6 +780,7 @@ const getPreviewTabIdForPip = async () => { }; const enterActivePreviewAutoPip = async () => { + if (isRecordingPreviewStatus(recordingStatus)) return false; const tabId = await getPreviewTabIdForPip(); if (tabId === null) return false; return sendOverlay(tabId, { type: "overlay-enter-auto-pip" }, false).catch( @@ -827,15 +831,11 @@ const broadcastOverlayCountdown = async ( if (!canInjectIntoTab(tab) || tab.id === undefined) { return undefined; } - return sendOverlay( - tab.id, - { - type: "overlay-countdown", - seconds, - durationMs, - }, - false, - ).catch(() => undefined); + return sendOverlay(tab.id, { + type: "overlay-countdown", + seconds, + durationMs, + }).catch(() => undefined); }), ); }; @@ -1182,7 +1182,6 @@ const resolveMicWarning = async ( const performRecordingStart = async (mode: RecordingMode) => { const { settings, auth, bootstrap } = await requireSignedInState(); - externalCaptureAutoPipPending = false; const recordingSettings = settings.capture.recordingMode === mode ? settings @@ -1206,7 +1205,6 @@ const performRecordingStart = async (mode: RecordingMode) => { if (micWarning) { const confirmed = await requestRecordingConfirmation(tabId, micWarning); if (!confirmed) { - externalCaptureAutoPipPending = false; return { ok: false, canceled: true, @@ -1215,20 +1213,61 @@ const performRecordingStart = async (mode: RecordingMode) => { } } - const tabStreamId = - mode === "tab" && tabId !== undefined - ? await getTabStreamId(tabId) - : undefined; - await showOverlayInTab(tab ?? null, recordingSettings, true); + if (activePreviewTabId !== null && activePreviewTabId !== tabId) { + const previousPreviewTab = await getTab(activePreviewTabId); + if (previousPreviewTab && canInjectIntoTab(previousPreviewTab)) { + const previousPrepared = await sendOverlay( + activePreviewTabId, + { + type: "overlay-settings", + settings: recordingSettings.webcam, + recording: true, + }, + false, + ); + if (!previousPrepared) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } + } + } + const previewPrepared = await showOverlayInTab( + tab ?? null, + recordingSettings, + true, + ); + if ( + isWebcamPreviewEnabled(recordingSettings) && + tab && + canInjectIntoTab(tab) && + !previewPrepared + ) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } const creatingStatus = { phase: "creating" } satisfies RecordingStatus; setRecordingStatusAndBroadcast(creatingStatus); + let tabStreamId: string | undefined; + if (mode === "tab" && tabId !== undefined) { + await ensureOffscreenDocument(); + try { + tabStreamId = await getTabStreamId(tabId); + } catch (error) { + console.warn("Tab capture stream ID unavailable", error); + if (!canUseDisplayPickerForTabCapture(error)) throw error; + } + } // The manifest injects the bootstrap content script into every page at // document_idle and onInstalled covers tabs that predate the extension, so // no blanket re-injection is needed here; sendOverlay still injects // per-tab on demand and the bootstrap lazy-loads the overlay UI. try { - return await sendOffscreen({ + const request: OffscreenRequest = { target: "offscreen", type: "start-recording", mode, @@ -1237,7 +1276,31 @@ const performRecordingStart = async (mode: RecordingMode) => { bootstrap, tabId, tabStreamId, - }); + }; + const response = await (tabStreamId + ? sendOffscreenRuntimeMessage(request) + : sendOffscreen(request)); + if ( + tabStreamId && + tabId !== undefined && + response.ok && + response.status?.phase === "recording" && + response.status.videoId + ) { + await saveTabInputCaptureSession({ + tabId, + recordingId: response.status.videoId, + }); + await chrome.tabs + .sendMessage(tabId, { + type: "input-capture-start", + recordingId: response.status.videoId, + }) + .catch((error: unknown) => { + console.warn("Tab input capture could not start", error); + }); + } + return response; } catch (error) { // The recorder panel closes as soon as the status leaves "idle", so a // silent reset would leave the user with no feedback at all. Broadcast @@ -1246,7 +1309,6 @@ const performRecordingStart = async (mode: RecordingMode) => { phase: "error", message: error instanceof Error ? error.message : String(error), }); - externalCaptureAutoPipPending = false; throw error; } }; @@ -1317,7 +1379,6 @@ const stopRecordingAndOpenDestination = async () => { if (response.ok && response.status) { setRecordingStatus(response.status); if (!isCapturingRecordingStatus(response.status)) { - externalCaptureAutoPipPending = false; await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); } @@ -1431,7 +1492,7 @@ const handlePreviewReady = async (tabId?: number) => { activePreviewTabId = tabId; pendingPreviewTabId = null; await hidePreviewTabsExcept(tabId); - if (!browserWindowFocused || externalCaptureAutoPipPending) { + if (!browserWindowFocused) { await enterActivePreviewAutoPip(); } }; @@ -1458,7 +1519,6 @@ const handlePreviewError = async ( const handleCaptureSource = async (source: RecordingCaptureSource) => { if (isBrowserCaptureSource(source)) { - externalCaptureAutoPipPending = false; const tabId = await findCapturedBrowserTabId(source); if (tabId !== null) { await focusTab(tabId); @@ -1467,14 +1527,6 @@ const handleCaptureSource = async (source: RecordingCaptureSource) => { await exitActivePreviewAutoPip(); return; } - - if (!shouldAutoPipCaptureSource(source)) { - externalCaptureAutoPipPending = false; - return; - } - - externalCaptureAutoPipPending = true; - await enterActivePreviewAutoPip(); }; const handleRequest = async ( @@ -1791,12 +1843,76 @@ const handleRequest = async ( return { ok: false, error: "Unknown request" }; }; -chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if ( + message?.target === "service-worker" && + message.type === "input-capture-stop" + ) { + if ( + !isTrustedOffscreenStorageSender( + sender, + chrome.runtime.id, + chrome.runtime.getURL(OFFSCREEN_URL), + ) || + !Number.isSafeInteger(message.tabId) || + typeof message.recordingId !== "string" + ) { + sendResponse({ ok: false, error: "Tab input stop denied" }); + return false; + } + void Promise.all([syncRecordingStatus(), loadTabInputCaptureSession()]) + .then(async ([currentStatus, session]) => { + if ( + !session || + session.tabId !== message.tabId || + session.recordingId !== message.recordingId || + !("videoId" in currentStatus) || + currentStatus.videoId !== message.recordingId + ) { + return { ok: false, error: "Tab input stop denied" }; + } + try { + return await chrome.tabs.sendMessage(message.tabId, { + type: "input-capture-stop", + }); + } finally { + await saveTabInputCaptureSession(null); + } + }) + .then(sendResponse) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } + if (isStorageBridgeRequest(message)) { + if ( + !isTrustedOffscreenStorageSender( + sender, + chrome.runtime.id, + chrome.runtime.getURL(OFFSCREEN_URL), + ) + ) { + sendResponse({ ok: false, error: "Storage request denied" }); + return false; + } + executeStorageBridgeRequest(message) + .then(sendResponse) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } if (isRecordingStatusBroadcast(message)) { setRecordingStatusAndBroadcast(message.status); if (isCapturingRecordingStatus(message.status)) return false; - externalCaptureAutoPipPending = false; void (async () => { await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); @@ -1813,7 +1929,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { if (!isServiceWorkerRequest(message)) return false; - handleRequest(message, _sender) + handleRequest(message, sender) .then(sendResponse) .catch((error: unknown) => { sendResponse({ @@ -1858,13 +1974,37 @@ chrome.windows.onFocusChanged.addListener((windowId) => { return; } browserWindowFocused = true; - externalCaptureAutoPipPending = false; void exitActivePreviewAutoPip() .then(() => syncActivePreview()) .catch(() => undefined); }); chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === "complete") { + void loadTabInputCaptureSession() + .then(async (session) => { + if (!session || session.tabId !== tabId) return; + const currentStatus = await syncRecordingStatus(); + if ( + !isCapturingRecordingStatus(currentStatus) || + !("videoId" in currentStatus) || + currentStatus.videoId !== session.recordingId + ) { + if (!isActiveRecordingStatus(currentStatus)) { + await saveTabInputCaptureSession(null); + } + return; + } + if (!canInjectIntoTab(tab)) return; + await chrome.tabs.sendMessage(tabId, { + type: "input-capture-start", + recordingId: session.recordingId, + }); + }) + .catch((error: unknown) => { + console.warn("Tab input capture could not resume", error); + }); + } if (changeInfo.status === "complete" && tab.active) { void syncActivePreview(tabId).catch(() => undefined); } @@ -1884,6 +2024,11 @@ chrome.tabs.onRemoved.addListener((tabId) => { if (activePreviewTabId === tabId) { activePreviewTabId = null; } + void loadTabInputCaptureSession() + .then((session) => + session?.tabId === tabId ? saveTabInputCaptureSession(null) : undefined, + ) + .catch(() => undefined); if (pendingPreviewTabId === tabId) { pendingPreviewTabId = null; } diff --git a/apps/chrome-extension/src/content/bootstrap.ts b/apps/chrome-extension/src/content/bootstrap.ts index 60cf019e170..6183046cf8b 100644 --- a/apps/chrome-extension/src/content/bootstrap.ts +++ b/apps/chrome-extension/src/content/bootstrap.ts @@ -6,6 +6,7 @@ import { RECORDING_STATE_KEY, SHARED_UI_STATE_KEY, } from "../shared/storage-keys"; +import { initTabInputCapture } from "./input-events"; // The manifest injects only this bootstrap into every page: a few KB of // vanilla code that decides whether the page actually needs the recorder UI. @@ -186,5 +187,6 @@ const bootstrap = () => { const globalScope = globalThis as Record; if (globalScope[BOOTSTRAP_FLAG] !== true) { globalScope[BOOTSTRAP_FLAG] = true; + initTabInputCapture(); bootstrap(); } diff --git a/apps/chrome-extension/src/content/countdown-overlay.tsx b/apps/chrome-extension/src/content/countdown-overlay.tsx index 11cd3bb5180..7c3fc711c35 100644 --- a/apps/chrome-extension/src/content/countdown-overlay.tsx +++ b/apps/chrome-extension/src/content/countdown-overlay.tsx @@ -1,6 +1,7 @@ import { type CSSProperties, useCallback, useEffect, useState } from "react"; import { isOverlayMessage } from "../shared/messages"; import { sendServiceWorkerMessage } from "../shared/runtime"; +import { replayStartupMessages } from "./startup-messages"; type ActiveCountdown = { seconds: number; @@ -69,6 +70,7 @@ export function CountdownOverlay() { }; chrome.runtime.onMessage.addListener(handleMessage); + replayStartupMessages(handleMessage); return () => chrome.runtime.onMessage.removeListener(handleMessage); }, [dismiss]); diff --git a/apps/chrome-extension/src/content/input-events.ts b/apps/chrome-extension/src/content/input-events.ts new file mode 100644 index 00000000000..02157b210f1 --- /dev/null +++ b/apps/chrome-extension/src/content/input-events.ts @@ -0,0 +1,285 @@ +import { + type CapturedTabInputEvent, + isSafeTabKeyboardEvent, +} from "../shared/input-events"; +import { RECORDING_STATE_KEY } from "../shared/storage-keys"; + +const MOVE_INTERVAL_MS = 16; +const BATCH_INTERVAL_MS = 250; +const MAX_BATCH_EVENTS = 128; +const MAX_PENDING_BATCHES = 8; +const OVERLAY_ROOT_ID = "cap-extension-recorder-overlay"; +const EDITABLE_ROLES = new Set(["textbox", "searchbox", "combobox"]); +const ALLOWED_CURSORS = new Set([ + "auto", + "default", + "pointer", + "text", + "crosshair", + "grab", + "grabbing", + "not-allowed", + "ew-resize", + "ns-resize", +]); + +const modifiersOf = (event: MouseEvent | KeyboardEvent) => { + const modifiers: string[] = []; + if (event.metaKey) modifiers.push("Meta"); + if (event.ctrlKey) modifiers.push("LControl"); + if (event.altKey) modifiers.push("LAlt"); + if (event.shiftKey) modifiers.push("LShift"); + return modifiers; +}; + +const isEditableTarget = (event: KeyboardEvent) => + event.composedPath().some((target) => { + if (!(target instanceof HTMLElement)) return false; + return ( + target.isContentEditable || + target instanceof HTMLInputElement || + target instanceof HTMLTextAreaElement || + target instanceof HTMLSelectElement || + EDITABLE_ROLES.has(target.getAttribute("role") ?? "") || + target.hasAttribute("aria-multiline") + ); + }); + +const isExtensionUiEvent = (event: Event) => + event + .composedPath() + .some( + (target) => target instanceof Element && target.id === OVERLAY_ROOT_ID, + ); + +export function initTabInputCapture() { + let recordingId: string | null = null; + let batch: CapturedTabInputEvent[] = []; + let pendingBatches = 0; + let sendChain: Promise = Promise.resolve(); + let sendFailed: Error | null = null; + let sequence = 0; + let collectorId = ""; + let timer: number | null = null; + let lastMoveAt = -Infinity; + let lastTarget: EventTarget | null = null; + let lastCursor = "default"; + let viewportWidth = window.innerWidth; + let viewportHeight = window.innerHeight; + + const stopListeners = () => { + recordingId = null; + if (timer !== null) window.clearInterval(timer); + timer = null; + document.removeEventListener("pointermove", onMove, true); + document.removeEventListener("pointerdown", onDown, true); + document.removeEventListener("pointerup", onUp, true); + document.removeEventListener("keydown", onKeyDown, true); + document.removeEventListener("keyup", onKeyUp, true); + }; + + const stop = async () => { + flush(); + stopListeners(); + await sendChain; + return sendFailed ? { ok: false, error: sendFailed.message } : { ok: true }; + }; + + const flush = () => { + if (!recordingId || batch.length === 0 || sendFailed) return; + if (pendingBatches >= MAX_PENDING_BATCHES) { + sendFailed = new Error("Tab input event queue exceeded its limit"); + stopListeners(); + return; + } + const events = batch; + batch = []; + pendingBatches += 1; + const message = { + target: "offscreen", + type: "input-events-batch", + recordingId, + collectorId, + sequence: sequence++, + platform: navigator.platform, + viewportWidth, + viewportHeight, + events, + }; + const pending = sendChain.then(() => { + if (sendFailed) throw sendFailed; + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage(message, (response: unknown) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message ?? "Tab input upload failed")); + return; + } + if ( + !response || + typeof response !== "object" || + !("ok" in response) || + response.ok !== true + ) { + reject(new Error("Tab input batch was not accepted")); + return; + } + resolve(); + }); + }); + }); + sendChain = pending.then( + () => { + pendingBatches -= 1; + }, + (error: unknown) => { + pendingBatches -= 1; + sendFailed = error instanceof Error ? error : new Error(String(error)); + stopListeners(); + }, + ); + }; + + const push = (event: CapturedTabInputEvent) => { + if (!recordingId) return; + if ( + window.innerWidth !== viewportWidth || + window.innerHeight !== viewportHeight + ) { + flush(); + viewportWidth = window.innerWidth; + viewportHeight = window.innerHeight; + } + batch.push(event); + if (batch.length >= MAX_BATCH_EVENTS) flush(); + }; + + const pointer = ( + kind: "move" | "down" | "up", + event: PointerEvent, + forceMove = false, + ) => { + if ( + !recordingId || + !event.isTrusted || + isExtensionUiEvent(event) || + window.innerWidth <= 0 || + window.innerHeight <= 0 + ) + return; + const epochMs = performance.timeOrigin + event.timeStamp; + if ( + kind === "move" && + !forceMove && + epochMs - lastMoveAt < MOVE_INTERVAL_MS + ) + return; + if (kind === "move") lastMoveAt = epochMs; + if (event.target !== lastTarget) { + lastTarget = event.target; + const cursor = + event.target instanceof Element + ? getComputedStyle(event.target).cursor + : "default"; + lastCursor = ALLOWED_CURSORS.has(cursor) ? cursor : "default"; + } + push({ + kind, + epochMs, + x: event.clientX / window.innerWidth, + y: event.clientY / window.innerHeight, + cursor: lastCursor, + button: event.button < 0 ? 0 : event.button, + modifiers: modifiersOf(event), + }); + }; + const onMove = (event: PointerEvent) => pointer("move", event); + const onDown = (event: PointerEvent) => { + pointer("move", event, true); + pointer("down", event); + }; + const onUp = (event: PointerEvent) => pointer("up", event); + const key = (kind: "keyDown" | "keyUp", event: KeyboardEvent) => { + if ( + !recordingId || + !event.isTrusted || + isExtensionUiEvent(event) || + isEditableTarget(event) || + !isSafeTabKeyboardEvent(event.key, event.code) + ) + return; + push({ + kind, + epochMs: performance.timeOrigin + event.timeStamp, + key: event.key, + code: event.code, + modifiers: modifiersOf(event), + }); + }; + const onKeyDown = (event: KeyboardEvent) => key("keyDown", event); + const onKeyUp = (event: KeyboardEvent) => key("keyUp", event); + + const start = async (nextRecordingId: string) => { + if (recordingId === nextRecordingId) return { ok: true }; + if (recordingId) await stop(); + recordingId = nextRecordingId; + collectorId = crypto.randomUUID(); + batch = []; + pendingBatches = 0; + sendChain = Promise.resolve(); + sendFailed = null; + sequence = 0; + lastMoveAt = -Infinity; + lastTarget = null; + viewportWidth = window.innerWidth; + viewportHeight = window.innerHeight; + document.addEventListener("pointermove", onMove, { + capture: true, + passive: true, + }); + document.addEventListener("pointerdown", onDown, { + capture: true, + passive: true, + }); + document.addEventListener("pointerup", onUp, { + capture: true, + passive: true, + }); + document.addEventListener("keydown", onKeyDown, { + capture: true, + passive: true, + }); + document.addEventListener("keyup", onKeyUp, { + capture: true, + passive: true, + }); + timer = window.setInterval(flush, BATCH_INTERVAL_MS); + return { ok: true }; + }; + + chrome.runtime.onMessage.addListener((message: unknown, _sender, respond) => { + if (!message || typeof message !== "object" || !("type" in message)) + return false; + if ( + message.type === "input-capture-start" && + "recordingId" in message && + typeof message.recordingId === "string" + ) { + void start(message.recordingId).then(respond); + } else if (message.type === "input-capture-stop") { + void stop().then(respond); + } else { + return false; + } + return true; + }); + chrome.storage.onChanged.addListener((changes, area) => { + if (area !== "session" || !recordingId) return; + const state = changes[RECORDING_STATE_KEY]?.newValue; + if (!state || typeof state !== "object" || !("status" in state)) return; + const status = state.status; + if (!status || typeof status !== "object" || !("phase" in status)) return; + if (status.phase !== "recording" && status.phase !== "paused") void stop(); + }); + window.addEventListener("pagehide", () => void stop()); +} diff --git a/apps/chrome-extension/src/content/overlay.tsx b/apps/chrome-extension/src/content/overlay.tsx index f7f2946cef4..335db134cf6 100644 --- a/apps/chrome-extension/src/content/overlay.tsx +++ b/apps/chrome-extension/src/content/overlay.tsx @@ -17,6 +17,7 @@ import { loadSharedUiState, loadWebcamPreviewDismissed, OVERLAY_UI_STATE_KEY, + RECORDING_STATE_KEY, SETTINGS_KEY, SHARED_UI_STATE_KEY, saveLastWebcamPreviewFrame, @@ -30,6 +31,7 @@ import type { ExtensionSettings, OverlayPosition, RecordingStatus, + SharedRecordingState, WebcamPosition, WebcamPreviewFrame, WebcamSettings, @@ -43,6 +45,7 @@ import { ConfirmOverlay } from "./confirm-overlay"; import { CountdownOverlay } from "./countdown-overlay"; import overlayCss from "./overlay.css?inline"; import { RecordingBarOverlay } from "./recording-bar"; +import recordingErrorCss from "./recording-error-toast.css?inline"; import { replayStartupMessages, setStartupMessages } from "./startup-messages"; const ROOT_ID = "cap-extension-recorder-overlay"; @@ -157,6 +160,11 @@ type PreviewParentMessage = source: "cap-extension-overlay"; token: string; type: "stop"; + } + | { + source: "cap-extension-overlay"; + token: string; + type: "stop-for-capture"; }; const classNames = (...values: Array) => @@ -339,6 +347,7 @@ function OverlayApp() { const [previewPointerInside, setPreviewPointerInside] = useState(false); const [recordingPreviewActive, setRecordingPreviewActive] = useState(false); const [recorderPanelOpen, setRecorderPanelOpen] = useState(false); + const [recordingError, setRecordingError] = useState(null); const [previewTokenReady, setPreviewTokenReady] = useState(false); const iframeRef = useRef(null); const windowRef = useRef(null); @@ -348,6 +357,8 @@ function OverlayApp() { const pipPeerRef = useRef(null); const pipStreamRef = useRef(null); const pipSettingsRef = useRef(null); + const parentPictureInPictureRequestRef = + useRef | null>(null); const previewSessionIdRef = useRef(null); const webcamRef = useRef(null); const settingsRef = useRef(null); @@ -383,6 +394,28 @@ function OverlayApp() { }); }, []); + const requestParentPictureInPicture = useCallback( + async (video: HTMLVideoElement) => { + if (recordingPreviewActiveRef.current) return; + const request = video.requestPictureInPicture(); + parentPictureInPictureRequestRef.current = request; + try { + await request; + if ( + recordingPreviewActiveRef.current && + document.pictureInPictureElement === video + ) { + await document.exitPictureInPicture(); + } + } finally { + if (parentPictureInPictureRequestRef.current === request) { + parentPictureInPictureRequestRef.current = null; + } + } + }, + [], + ); + const persistPreviewFrame = useCallback( (frame: WebcamPreviewFrame | null, force: boolean) => { if (!frame) return; @@ -453,6 +486,67 @@ function OverlayApp() { postPreviewMessage, ]); + const stopFramePictureInPictureForCapture = useCallback( + () => + new Promise((resolve) => { + if (!iframeRef.current || !previewOpenRef.current || !iframeReady) { + resolve(true); + return; + } + + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + window.clearTimeout(timeout); + resolve(ok); + }; + const timeout = window.setTimeout(() => finish(false), 3000); + chrome.runtime.sendMessage( + { + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "stop-for-capture", + } satisfies PreviewParentMessage, + (response: unknown) => { + const responseOk = + !chrome.runtime.lastError && + response !== null && + typeof response === "object" && + "ok" in response && + response.ok === true; + finish(responseOk); + }, + ); + }), + [iframeReady], + ); + + const prepareCameraForCapture = useCallback(async () => { + recordingPreviewActiveRef.current = true; + setRecordingPreviewActive(true); + const video = pipVideoRef.current; + const pendingParentRequest = parentPictureInPictureRequestRef.current; + const [frameResult, parentResult] = await Promise.allSettled([ + stopFramePictureInPictureForCapture(), + (async () => { + if (pendingParentRequest) { + await pendingParentRequest.catch(() => undefined); + } + if (video && document.pictureInPictureElement === video) { + await document.exitPictureInPicture(); + } + })(), + ]); + stopLocalPreview(); + return ( + frameResult.status === "fulfilled" && + frameResult.value && + parentResult.status === "fulfilled" && + (!video || document.pictureInPictureElement !== video) + ); + }, [stopFramePictureInPictureForCapture, stopLocalPreview]); + const applyWebcamSettings = useCallback( (getNext: (current: WebcamSettings) => WebcamSettings) => { setExtensionSettings((current) => { @@ -603,6 +697,19 @@ function OverlayApp() { }; }, [stopLocalPreview]); + useEffect(() => { + if (!recordingPreviewActive) return; + const video = pipVideoRef.current; + if (video && document.pictureInPictureElement === video) { + void document.exitPictureInPicture().catch(() => undefined); + } + postPreviewMessage({ + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "exit-auto-pip", + }); + }, [postPreviewMessage, recordingPreviewActive]); + useEffect(() => { let disposed = false; Promise.all([ @@ -639,7 +746,19 @@ function OverlayApp() { }) .catch(() => undefined); }; + const syncRecordingError = () => { + loadSharedRecordingState() + .then((state) => { + if (!disposed) { + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } + }) + .catch(() => undefined); + }; syncSharedUi(); + syncRecordingError(); const handleStorageChange = ( changes: Record, areaName: string, @@ -647,6 +766,14 @@ function OverlayApp() { if (areaName === "session" && changes[SHARED_UI_STATE_KEY]) { syncSharedUi(); } + if (areaName === "session" && changes[RECORDING_STATE_KEY]) { + const state = changes[RECORDING_STATE_KEY].newValue as + | SharedRecordingState + | undefined; + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } }; chrome.storage.onChanged.addListener(handleStorageChange); return () => { @@ -708,12 +835,66 @@ function OverlayApp() { }, [stopLocalPreview]); useEffect(() => { + let disposed = false; + const applyPreviewSettings = (message: { + settings: WebcamSettings; + recording: boolean; + }) => { + if (disposed) return; + const webcamSettings = message.settings; + const sameLivePreview = + previewOpenRef.current && + livePreviewReadyRef.current && + isSameWebcamSettings(webcamRef.current, webcamSettings); + if (!sameLivePreview) { + setLivePreviewReady(false); + } else { + // A restarted service worker loses which tab owns the live preview, + // so re-announce readiness whenever it pushes the same settings. + void sendServiceWorkerMessage({ + target: "service-worker", + type: "webcam-preview-ready", + }).catch(() => undefined); + } + setPreviewError(null); + setShowPreviewError(false); + setPreviewOpen(true); + recordingPreviewActiveRef.current = message.recording; + setRecordingPreviewActive(message.recording); + if (settingsRef.current) { + setExtensionSettings({ + ...settingsRef.current, + webcam: webcamSettings, + }); + } else { + sendServiceWorkerMessage({ + target: "service-worker", + type: "get-overlay-settings", + }) + .then((response) => { + if (!disposed && response.ok && response.settings) { + setExtensionSettings({ + ...response.settings, + webcam: webcamSettings, + }); + } + }) + .catch(() => undefined); + } + }; + const handleMessage = ( message: unknown, _sender: chrome.runtime.MessageSender, sendResponse: (response?: unknown) => void, ) => { if (!isOverlayMessage(message)) return false; + if (message.type === "overlay-settings" && message.recording) { + void prepareCameraForCapture() + .then((ok) => sendResponse({ ok })) + .catch(() => sendResponse({ ok: false })); + return true; + } sendResponse({ ok: true }); @@ -750,6 +931,7 @@ function OverlayApp() { } if (message.type === "overlay-enter-auto-pip") { + if (recordingPreviewActiveRef.current) return false; // Only one Picture in Picture surface may drive at a time; racing // the parent fallback video against the preview iframe flips PiP // on and off and leaves the badge state inconsistent. @@ -770,7 +952,7 @@ function OverlayApp() { if (video && parentPipSupported) { void video .play() - .then(() => video.requestPictureInPicture()) + .then(() => requestParentPictureInPicture(video)) .catch(() => undefined); } return false; @@ -791,48 +973,13 @@ function OverlayApp() { if (message.type !== "overlay-settings") return false; - if (previewDismissedRef.current) return false; - - const webcamSettings = message.settings; - const sameLivePreview = - previewOpenRef.current && - livePreviewReadyRef.current && - isSameWebcamSettings(webcamRef.current, webcamSettings); - if (!sameLivePreview) { - setLivePreviewReady(false); - } else { - // A restarted service worker loses which tab owns the live preview, - // so re-announce readiness whenever it pushes the same settings. - void sendServiceWorkerMessage({ - target: "service-worker", - type: "webcam-preview-ready", - }).catch(() => undefined); - } - setPreviewError(null); - setShowPreviewError(false); - setPreviewOpen(true); - recordingPreviewActiveRef.current = message.recording; - setRecordingPreviewActive(message.recording); - if (settingsRef.current) { - setExtensionSettings({ - ...settingsRef.current, - webcam: webcamSettings, - }); - } else { - sendServiceWorkerMessage({ - target: "service-worker", - type: "get-overlay-settings", + void loadWebcamPreviewDismissed() + .then((dismissed) => { + if (disposed) return; + previewDismissedRef.current = dismissed; + if (!dismissed) applyPreviewSettings(message); }) - .then((response) => { - if (response.ok && response.settings) { - setExtensionSettings({ - ...response.settings, - webcam: webcamSettings, - }); - } - }) - .catch(() => undefined); - } + .catch(() => undefined); return false; }; @@ -841,8 +988,17 @@ function OverlayApp() { startupReplayedRef.current = true; replayStartupMessages(handleMessage); } - return () => chrome.runtime.onMessage.removeListener(handleMessage); - }, [parentPipSupported, postPreviewMessage, stopLocalPreview]); + return () => { + disposed = true; + chrome.runtime.onMessage.removeListener(handleMessage); + }; + }, [ + parentPipSupported, + postPreviewMessage, + prepareCameraForCapture, + requestParentPictureInPicture, + stopLocalPreview, + ]); const beginDrag = useCallback((clientX: number, clientY: number) => { isDraggingRef.current = true; @@ -1082,7 +1238,11 @@ function OverlayApp() { if (event.type === "ready") { setIframeReady(true); const current = webcamRef.current; - if (current?.enabled && current.deviceId) { + if ( + !recordingPreviewActiveRef.current && + current?.enabled && + current.deviceId + ) { postPreviewMessage({ source: "cap-extension-overlay", token: PREVIEW_TOKEN, @@ -1430,6 +1590,7 @@ function OverlayApp() { }, [applyWebcamSettings]); const handleTogglePictureInPicture = useCallback(() => { + if (recordingPreviewActiveRef.current) return; const video = pipVideoRef.current; if (video && document.pictureInPictureElement === video) { void document.exitPictureInPicture().catch(() => undefined); @@ -1454,10 +1615,10 @@ function OverlayApp() { void (async () => { try { await video.play().catch(() => undefined); - await video.requestPictureInPicture(); + await requestParentPictureInPicture(video); } catch {} })(); - }, [parentPipSupported, postPreviewMessage]); + }, [parentPipSupported, postPreviewMessage, requestParentPictureInPicture]); const handlePreviewLoad = useCallback(() => { const current = webcamRef.current; @@ -1511,6 +1672,7 @@ function OverlayApp() { width: `${metrics.width}px`, height: `${totalHeight}px`, borderRadius, + visibility: recordingPreviewActive ? "hidden" : "visible", }} onPointerDown={handlePointerDown} > @@ -1671,6 +1833,12 @@ function OverlayApp() { }).catch(() => undefined); }} /> + {recordingError && recorderPanelOpen ? ( +
+ Recording failed. + {recordingError} +
+ ) : null} {isDragging ? ( @@ -1726,7 +1894,7 @@ const mountOverlay = () => { root.dataset.capMounted = "true"; const shadow = root.attachShadow({ mode: "closed" }); const style = document.createElement("style"); - style.textContent = overlayCss; + style.textContent = `${overlayCss}\n${recordingErrorCss}`; const app = document.createElement("div"); shadow.append(style, app); document.documentElement.append(root); diff --git a/apps/chrome-extension/src/content/recording-error-toast.css b/apps/chrome-extension/src/content/recording-error-toast.css new file mode 100644 index 00000000000..b3b665e54a1 --- /dev/null +++ b/apps/chrome-extension/src/content/recording-error-toast.css @@ -0,0 +1,27 @@ +.cap-extension-recording-error-toast { + position: fixed; + top: 20px; + left: 50%; + z-index: 2147483647; + display: flex; + max-width: min(440px, calc(100vw - 40px)); + flex-direction: column; + gap: 5px; + padding: 12px 16px; + border: 1px solid #f1c1c4; + border-radius: 12px; + background: #fff; + box-shadow: 0 12px 40px rgba(0, 0, 0, 0.16); + color: #3f171b; + font-family: Inter, ui-sans-serif, system-ui, sans-serif; + font-size: 12px; + line-height: 1.35; + pointer-events: auto; + transform: translateX(-50%); +} + +.cap-extension-recording-error-toast strong { + color: #c9343e; + font-size: 13px; + font-weight: 600; +} diff --git a/apps/chrome-extension/src/offscreen/display-capture.test.ts b/apps/chrome-extension/src/offscreen/display-capture.test.ts index a1f860e5826..36b3654587e 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.test.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.test.ts @@ -4,6 +4,21 @@ import { captureDisplayStream } from "./display-capture"; const stream = {} as MediaStream; describe("captureDisplayStream", () => { + it("prefers the current tab when tab capture needs the display picker", async () => { + const request = vi.fn().mockResolvedValue(stream); + + await expect(captureDisplayStream("tab", false, request)).resolves.toBe( + stream, + ); + expect(request).toHaveBeenCalledTimes(1); + expect(request.mock.calls[0]?.[0]).toMatchObject({ + preferCurrentTab: true, + selfBrowserSurface: "include", + monitorTypeSurfaces: "exclude", + audio: false, + }); + }); + it("falls back when advanced options are rejected before a picker can open", async () => { const request = vi .fn() diff --git a/apps/chrome-extension/src/offscreen/display-capture.ts b/apps/chrome-extension/src/offscreen/display-capture.ts index af5ee9547da..846c62a626d 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.ts @@ -44,7 +44,7 @@ const canRetryWithoutShowingAnotherPicker = (attempt: CaptureAttempt) => shouldRetryDisplayMediaWithoutPreferences(attempt.error); export const captureDisplayStream = async ( - mode: Exclude, + mode: Exclude, includeAudio: boolean, requestDisplayMedia: DisplayMediaRequester, ) => { diff --git a/apps/chrome-extension/src/offscreen/input-coordinates.test.ts b/apps/chrome-extension/src/offscreen/input-coordinates.test.ts new file mode 100644 index 00000000000..bcb8450ecbd --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-coordinates.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { mapTabPointerToVideo } from "./input-coordinates"; + +describe("tab input coordinates", () => { + it("places viewport corners inside Chrome tab-capture padding", () => { + const topLeft = mapTabPointerToVideo(0, 0, 1273, 716, 3024, 1964); + const center = mapTabPointerToVideo(0.5, 0.5, 1273, 716, 3024, 1964); + const bottomRight = mapTabPointerToVideo(1, 1, 1273, 716, 3024, 1964); + expect(topLeft?.x).toBe(0); + expect(topLeft?.y).toBeCloseTo(131 / 1964, 2); + expect(center).toEqual({ x: 0.5, y: 0.5 }); + expect(bottomRight?.x).toBe(1); + expect(bottomRight?.y).toBeCloseTo(1832 / 1964, 2); + }); + + it("maps portrait tabs with side padding", () => { + const pointer = mapTabPointerToVideo(0, 0, 716, 1273, 1920, 1080); + expect(pointer?.x).toBeGreaterThan(0); + expect(pointer?.y).toBe(0); + }); + + it("rejects impossible source dimensions", () => { + expect(mapTabPointerToVideo(0.5, 0.5, 0, 100, 1920, 1080)).toBeNull(); + expect( + mapTabPointerToVideo(Number.NaN, 0.5, 100, 100, 1920, 1080), + ).toBeNull(); + }); +}); diff --git a/apps/chrome-extension/src/offscreen/input-coordinates.ts b/apps/chrome-extension/src/offscreen/input-coordinates.ts new file mode 100644 index 00000000000..60e6499fcbb --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-coordinates.ts @@ -0,0 +1,34 @@ +export function mapTabPointerToVideo( + x: number, + y: number, + viewportWidth: number, + viewportHeight: number, + videoWidth: number, + videoHeight: number, +) { + if ( + ![x, y, viewportWidth, viewportHeight, videoWidth, videoHeight].every( + Number.isFinite, + ) || + viewportWidth <= 0 || + viewportHeight <= 0 || + videoWidth <= 0 || + videoHeight <= 0 + ) { + return null; + } + const scale = Math.min( + videoWidth / viewportWidth, + videoHeight / viewportHeight, + ); + const contentWidth = viewportWidth * scale; + const contentHeight = viewportHeight * scale; + return { + x: + (videoWidth - contentWidth) / (2 * videoWidth) + + (x * contentWidth) / videoWidth, + y: + (videoHeight - contentHeight) / (2 * videoHeight) + + (y * contentHeight) / videoHeight, + }; +} diff --git a/apps/chrome-extension/src/offscreen/input-event-sidecar.ts b/apps/chrome-extension/src/offscreen/input-event-sidecar.ts new file mode 100644 index 00000000000..71d36942727 --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-event-sidecar.ts @@ -0,0 +1,244 @@ +import { + InstantRecordingUploader, + initiateMultipartUpload, + type RecorderApiOptions, + RecordingSpool, + type VideoId, +} from "@cap/recorder-core"; +import { + type CapturedTabInputEvent, + parseCapturedTabInputBatch, +} from "../shared/input-events"; +import { mapTabPointerToVideo } from "./input-coordinates"; +import { InputTimeline } from "./input-timeline"; + +const INPUT_SUBPATH = "input-events-upload.ndjson"; +const INPUT_MIME_TYPE = "application/x-ndjson"; +const MAX_INPUT_BYTES = 64 * 1024 * 1024; +const MAX_COLLECTORS = 1_000; + +export class InputEventSidecar { + private timeline: InputTimeline | null = null; + private readonly nextSequenceByCollector = new Map(); + private appendChain = Promise.resolve(); + private platform: string | null = null; + private totalBytes = 0; + private failed: Error | null = null; + private uploadCompleted = false; + + private constructor( + readonly spool: RecordingSpool, + readonly recordingId: string, + readonly tabId: number, + readonly videoWidth: number, + readonly videoHeight: number, + ) {} + + static async create(input: { + recordingId: string; + tabId: number; + videoWidth: number; + videoHeight: number; + }) { + const spool = await RecordingSpool.create({ mimeType: INPUT_MIME_TYPE }); + return new InputEventSidecar( + spool, + input.recordingId, + input.tabId, + input.videoWidth, + input.videoHeight, + ); + } + + get recordedBytes() { + return this.totalBytes; + } + + get isUploadCompleted() { + return this.uploadCompleted; + } + + start(epochMs: number) { + this.timeline = new InputTimeline(epochMs); + } + + pause(epochMs: number) { + this.timeline?.pause(epochMs); + } + + resume(epochMs: number) { + this.timeline?.resume(epochMs); + } + + stop(epochMs: number) { + this.timeline?.stop(epochMs); + } + + accept(raw: unknown, senderTabId: number | undefined) { + const batch = parseCapturedTabInputBatch(raw); + if ( + !batch || + senderTabId !== this.tabId || + batch.recordingId !== this.recordingId + ) { + return Promise.reject(new Error("Tab input batch is unauthorized")); + } + const pending = this.appendChain.then(async () => { + if (this.failed) throw this.failed; + const nextSequence = + this.nextSequenceByCollector.get(batch.collectorId) ?? 0; + if (batch.sequence < nextSequence) return; + if (batch.sequence !== nextSequence) { + throw new Error("Tab input batch sequence is incomplete"); + } + if ( + nextSequence === 0 && + this.nextSequenceByCollector.size >= MAX_COLLECTORS + ) { + throw new Error("Too many tab input capture sessions"); + } + if (this.platform && this.platform !== batch.platform) { + throw new Error("Tab input platform changed during recording"); + } + const mapped = batch.events.flatMap((event) => { + const timeMs = this.timeline?.timeFor(event.epochMs) ?? null; + if (timeMs === null) return []; + return [ + this.mapEvent( + event, + timeMs, + batch.viewportWidth, + batch.viewportHeight, + ), + ]; + }); + const lines = [ + ...(!this.platform + ? [JSON.stringify({ version: 1, platform: batch.platform })] + : []), + ...mapped.map((event) => JSON.stringify(event)), + ]; + if (lines.length > 0) { + const blob = new Blob([`${lines.join("\n")}\n`], { + type: INPUT_MIME_TYPE, + }); + if (this.totalBytes + blob.size > MAX_INPUT_BYTES) { + throw new Error("Tab input events exceeded the supported size"); + } + await this.spool.appendChunk(blob); + this.totalBytes += blob.size; + } + this.platform ??= batch.platform; + this.nextSequenceByCollector.set(batch.collectorId, nextSequence + 1); + }); + this.appendChain = pending.then( + () => undefined, + (error: unknown) => { + this.failed = error instanceof Error ? error : new Error(String(error)); + }, + ); + return pending; + } + + private mapEvent( + event: CapturedTabInputEvent, + timeMs: number, + viewportWidth: number, + viewportHeight: number, + ) { + if (!("x" in event)) { + return { + kind: event.kind, + timeMs, + key: event.key, + code: event.code, + modifiers: event.modifiers, + }; + } + const position = mapTabPointerToVideo( + event.x, + event.y, + viewportWidth, + viewportHeight, + this.videoWidth, + this.videoHeight, + ); + if (!position) throw new Error("Tab input viewport is invalid"); + return { + kind: event.kind, + timeMs, + x: position.x, + y: position.y, + cursor: event.cursor, + button: event.button, + modifiers: event.modifiers, + }; + } + + async recoverBlob() { + await this.appendChain; + if (this.failed) throw this.failed; + return this.spool.recoverBlob(); + } + + async upload(input: { + videoId: VideoId; + screenSubpath: string; + durationSeconds: number; + api: RecorderApiOptions; + }) { + const blob = await this.recoverBlob(); + if (!blob || blob.size === 0) return false; + await uploadInputEventBlob({ ...input, blob }); + this.uploadCompleted = true; + return true; + } + + async dispose() { + await this.spool.dispose(); + } +} + +export async function uploadInputEventBlob(input: { + videoId: VideoId; + screenSubpath: string; + durationSeconds: number; + api: RecorderApiOptions; + blob: Blob; +}) { + if ( + input.blob.size < 1 || + input.blob.size > MAX_INPUT_BYTES || + input.blob.type !== INPUT_MIME_TYPE + ) { + throw new Error("Invalid tab input event sidecar"); + } + const api: RecorderApiOptions = { + ...input.api, + extraBody: { + ...input.api.extraBody, + screenSubpath: input.screenSubpath, + }, + }; + const session = await initiateMultipartUpload({ + videoId: input.videoId, + contentType: INPUT_MIME_TYPE, + subpath: INPUT_SUBPATH, + api, + }); + const uploader = new InstantRecordingUploader({ + videoId: input.videoId, + uploadId: session.uploadId, + provider: session.provider, + mimeType: INPUT_MIME_TYPE, + subpath: INPUT_SUBPATH, + api, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + }); + await uploader.finalize({ + finalBlob: input.blob, + durationSeconds: input.durationSeconds, + subpath: INPUT_SUBPATH, + }); +} diff --git a/apps/chrome-extension/src/offscreen/input-timeline.test.ts b/apps/chrome-extension/src/offscreen/input-timeline.test.ts new file mode 100644 index 00000000000..eb82cab6dfc --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-timeline.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { InputTimeline } from "./input-timeline"; + +describe("tab input timeline", () => { + it("maps delayed batches across pause and resume", () => { + const timeline = new InputTimeline(1_000); + expect(timeline.timeFor(1_100)).toBe(100); + timeline.pause(1_500); + timeline.resume(3_000); + expect(timeline.timeFor(1_250)).toBe(250); + expect(timeline.timeFor(2_000)).toBeNull(); + expect(timeline.timeFor(3_050)).toBe(550); + timeline.stop(3_500); + expect(timeline.timeFor(3_500)).toBeNull(); + }); + + it("keeps consecutive active spans continuous", () => { + const timeline = new InputTimeline(1_000); + timeline.pause(1_500); + timeline.resume(3_000); + timeline.pause(3_500); + timeline.resume(4_000); + expect(timeline.timeFor(4_050)).toBe(1_050); + expect(timeline.timeFor(500)).toBeNull(); + }); +}); diff --git a/apps/chrome-extension/src/offscreen/input-timeline.ts b/apps/chrome-extension/src/offscreen/input-timeline.ts new file mode 100644 index 00000000000..214500e311b --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-timeline.ts @@ -0,0 +1,57 @@ +type ActiveSpan = { + startEpochMs: number; + startClipMs: number; + endEpochMs: number | null; +}; + +export class InputTimeline { + private spans: ActiveSpan[]; + private clipDurationMs = 0; + + constructor(startEpochMs: number) { + this.spans = [{ startEpochMs, startClipMs: 0, endEpochMs: null }]; + } + + pause(epochMs: number) { + const span = this.spans.at(-1); + if (!span || span.endEpochMs !== null) return; + span.endEpochMs = Math.max(span.startEpochMs, epochMs); + this.clipDurationMs = + span.startClipMs + span.endEpochMs - span.startEpochMs; + } + + resume(epochMs: number) { + const span = this.spans.at(-1); + if (!span || span.endEpochMs === null) return; + this.spans.push({ + startEpochMs: Math.max(span.endEpochMs, epochMs), + startClipMs: this.clipDurationMs, + endEpochMs: null, + }); + } + + stop(epochMs: number) { + this.pause(epochMs); + } + + timeFor(epochMs: number) { + if (!Number.isFinite(epochMs)) return null; + let low = 0; + let high = this.spans.length - 1; + let candidate = -1; + while (low <= high) { + const middle = (low + high) >>> 1; + const span = this.spans[middle]; + if (epochMs < span.startEpochMs) { + high = middle - 1; + } else { + candidate = middle; + low = middle + 1; + } + } + if (candidate < 0) return null; + const span = this.spans[candidate]; + if (span.endEpochMs !== null && epochMs >= span.endEpochMs) return null; + return Math.max(0, span.startClipMs + epochMs - span.startEpochMs); + } +} diff --git a/apps/chrome-extension/src/offscreen/recorder.ts b/apps/chrome-extension/src/offscreen/recorder.ts index 6173df09647..8f1c9f581a4 100644 --- a/apps/chrome-extension/src/offscreen/recorder.ts +++ b/apps/chrome-extension/src/offscreen/recorder.ts @@ -1,4 +1,5 @@ import { + AudioRecordingSidecar, appendLocalRecordingChunk, type ChunkUploadState, DEFAULT_API_REQUEST_TIMEOUT_MS, @@ -14,9 +15,11 @@ import { listRecordingSpoolSessions, MultipartCompletionUncertainError, RECORDING_SPOOL_LIVE_MIN_IDLE_MS, + type RecorderApiOptions, RecordingSpool, recoverRecordingSpoolSession, selectRecordingPipeline, + uploadRecoveredAudioSidecar, type VideoId, } from "@cap/recorder-core"; @@ -26,6 +29,7 @@ import { updateUploadProgress, } from "../shared/api"; import { toCameraDevices, toMicrophoneDevices } from "../shared/devices"; +import { parseCapturedTabInputBatch } from "../shared/input-events"; import { isOffscreenRequest } from "../shared/messages"; import { loadAuth, @@ -66,6 +70,7 @@ import { waitForIceGatheringComplete, } from "../shared/webrtc"; import { captureDisplayStream } from "./display-capture"; +import { InputEventSidecar, uploadInputEventBlob } from "./input-event-sidecar"; const RECORDING_TIMESLICE_MS = 1000; const RECORDING_TIMESLICE_GUARD_MS = RECORDING_TIMESLICE_MS * 3; @@ -93,13 +98,25 @@ type RecordingSound = "start-recording" | "stop-recording"; type ActiveRecording = { recorder: MediaRecorder; + cameraRecorder: MediaRecorder | null; + audioSidecars: AudioRecordingSidecar[]; + inputSidecar: InputEventSidecar | null; + inputCaptureTabId: number | null; + inputUploadApi: RecorderApiOptions; + inputStopPromise: Promise | null; + inputCaptureIncomplete: boolean; stopPromise: Promise; + cameraStopPromise: Promise | null; streams: MediaStream[]; + microphoneStream: MediaStream | null; recordingStream: MediaStream; statusTimer: number | null; spool: RecordingSpool; + cameraSpool: RecordingSpool | null; uploader: InstantRecordingUploader; + cameraUploader: InstantRecordingUploader | null; startedAt: number; + cameraOffsetMs: number | null; durationMs: number; lastResumedAt: number | null; videoId: VideoId; @@ -107,16 +124,25 @@ type ActiveRecording = { width: number; height: number; fps: number; + cameraWidth: number | null; + cameraHeight: number | null; + cameraFps: number | null; subpath: string; mimeType: string; + cameraSubpath: string | null; + cameraMimeType: string | null; maxDurationMs: number | null; audioContext?: AudioContext; chunkChain: Promise; + cameraChunkChain: Promise; dataRequestInterval: number | null; + cameraDataRequestInterval: number | null; chunkStartGuard: number | null; chunkingMode: ChunkingMode | null; lastChunkAt: number | null; recordedBytes: number; + cameraRecordedBytes: number; + cameraUploadCompleted: boolean; finalizePromise: Promise | null; cleanedUp: boolean; // A spool write failure (IndexedDB quota, backpressure) must not end an @@ -125,6 +151,8 @@ type ActiveRecording = { // streaming upload continues untouched. spoolFailed: boolean; memoryBackup: LocalRecordingState; + cameraSpoolFailed: boolean; + cameraMemoryBackup: LocalRecordingState; }; let activeRecording: ActiveRecording | null = null; @@ -437,12 +465,21 @@ const tabCaptureConstraints = (streamId: string, includeAudio: boolean) => const getMainStream = async (request: StartRecordingRequest) => { if (request.mode === "tab") { - if (!request.tabStreamId) throw new Error("Tab stream id is missing"); - return navigator.mediaDevices.getUserMedia( - tabCaptureConstraints( - request.tabStreamId, - request.settings.systemAudio.enabled, - ), + if (request.tabStreamId) { + return navigator.mediaDevices.getUserMedia( + tabCaptureConstraints( + request.tabStreamId, + request.settings.systemAudio.enabled, + ), + ); + } + return captureDisplayStream( + "tab", + request.settings.systemAudio.enabled, + (options) => + navigator.mediaDevices.getDisplayMedia( + options as DisplayMediaStreamOptions, + ), ); } @@ -635,6 +672,23 @@ const cleanupActiveRecording = async (recording: ActiveRecording) => { if (recording.chunkStartGuard !== null) { window.clearTimeout(recording.chunkStartGuard); } + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + } + if (recording.recorder.state !== "inactive") { + recording.recorder.stop(); + } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } + await Promise.all( + recording.audioSidecars.map((sidecar) => + sidecar.stop().catch(() => undefined), + ), + ); for (const stream of recording.streams) { stopTracks(stream); } @@ -656,19 +710,62 @@ const sweepOrphanedRecordingSpools = async () => { loadLiveRecordingManifests(), ]); const now = Date.now(); - const knownSessions = new Set(failed.map((entry) => entry.sessionId)); + const knownSessions = new Set( + failed.flatMap((entry) => [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.inputEventsSessionId ? [entry.inputEventsSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ]), + ); + const sessionsById = new Map( + sessions.map((session) => [session.sessionId, session]), + ); const manifestsBySession = new Map( manifests.map((manifest) => [manifest.sessionId, manifest]), ); + const manifestsByCameraSession = new Map( + manifests.flatMap((manifest) => { + const sessionId = manifest.cameraSessionId; + return sessionId ? [[sessionId, manifest] as const] : []; + }), + ); + const manifestsByAudioSession = new Map( + manifests.flatMap((manifest) => + (manifest.audioSources ?? []).map( + (source) => [source.sessionId, manifest] as const, + ), + ), + ); + const manifestsByInputSession = new Map( + manifests.flatMap((manifest) => { + const sessionId = manifest.inputEventsSessionId; + return sessionId ? [[sessionId, manifest] as const] : []; + }), + ); const remainingSessions = new Set(); let entries = [...failed]; for (const orphan of sessions) { - if (activeRecording?.spool.sessionId === orphan.sessionId) { + if ( + activeRecording?.spool.sessionId === orphan.sessionId || + activeRecording?.cameraSpool?.sessionId === orphan.sessionId || + activeRecording?.inputSidecar?.spool.sessionId === orphan.sessionId || + activeRecording?.audioSidecars.some( + (sidecar) => sidecar.metadata.sessionId === orphan.sessionId, + ) + ) { remainingSessions.add(orphan.sessionId); continue; } - if (now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS) { + const manifest = manifestsBySession.get(orphan.sessionId); + // A surviving manifest belongs to a previous offscreen document. This + // document is the extension's only spool writer, so the shared live + // window must not delay recovery after that writer has gone away. + if ( + now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS && + !manifest + ) { remainingSessions.add(orphan.sessionId); continue; } @@ -682,20 +779,95 @@ const sweepOrphanedRecordingSpools = async () => { () => undefined, ); entries = entries.filter( - (entry) => entry.sessionId !== orphan.sessionId, + (entry) => + entry.sessionId !== orphan.sessionId && + entry.cameraSessionId !== orphan.sessionId, + ); + entries = entries.map((entry) => { + const audioSources = (entry.audioSources ?? []).filter( + (source) => source.sessionId !== orphan.sessionId, + ); + return audioSources.length === (entry.audioSources ?? []).length + ? entry + : { ...entry, audioSources, audioRetryUnavailable: true }; + }); + entries = entries.map((entry) => + entry.inputEventsSessionId === orphan.sessionId + ? { + ...entry, + inputEventsSessionId: undefined, + inputEventsTotalBytes: undefined, + inputEventsRetryUnavailable: true, + } + : entry, ); continue; } remainingSessions.add(orphan.sessionId); + const cameraOwner = manifestsByCameraSession.get(orphan.sessionId); + const audioOwner = manifestsByAudioSession.get(orphan.sessionId); + const inputOwner = manifestsByInputSession.get(orphan.sessionId); + const sourceOwner = cameraOwner ?? audioOwner ?? inputOwner; + if ( + sourceOwner && + (sessionsById.has(sourceOwner.sessionId) || + knownSessions.has(sourceOwner.sessionId)) + ) { + continue; + } if (!knownSessions.has(orphan.sessionId) && orphan.totalBytes > 0) { // A crash-stranded session whose live manifest survived keeps its // videoId/subpath so the entry stays retryable, not download-only. // The duration is a wall-clock estimate (it includes pauses); it // only feeds the completion metadata on retry. - const manifest = manifestsBySession.get(orphan.sessionId); + const cameraSession = manifest?.cameraSessionId + ? sessionsById.get(manifest.cameraSessionId) + : undefined; + const cameraMetadata = + manifest?.cameraSessionId && cameraSession?.totalBytes + ? { + cameraSessionId: manifest.cameraSessionId, + cameraMimeType: + manifest.cameraMimeType ?? cameraSession.mimeType, + cameraSubpath: manifest.cameraSubpath, + cameraOffsetMs: manifest.cameraOffsetMs ?? 0, + cameraWidth: manifest.cameraWidth, + cameraHeight: manifest.cameraHeight, + cameraFps: manifest.cameraFps, + cameraTotalBytes: cameraSession.totalBytes, + } + : {}; + const audioSources = (manifest?.audioSources ?? []).flatMap( + (source) => { + const session = sessionsById.get(source.sessionId); + return session?.totalBytes + ? [{ ...source, recordedBytes: session.totalBytes }] + : []; + }, + ); + const inputSession = manifest?.inputEventsSessionId + ? sessionsById.get(manifest.inputEventsSessionId) + : undefined; entries.push({ sessionId: orphan.sessionId, + ...cameraMetadata, + ...(manifest?.inputEventsSessionId && inputSession?.totalBytes + ? { + inputEventsSessionId: manifest.inputEventsSessionId, + inputEventsTotalBytes: inputSession.totalBytes, + } + : {}), + ...(manifest?.inputEventsSessionId && !inputSession + ? { inputEventsRetryUnavailable: true } + : {}), + ...(audioSources.length > 0 ? { audioSources } : {}), + ...(audioSources.length !== (manifest?.audioSources ?? []).length + ? { audioRetryUnavailable: true } + : {}), + ...(manifest?.cameraSessionId && !cameraSession?.totalBytes + ? { cameraRetryUnavailable: true } + : {}), videoId: manifest?.videoId ?? null, shareUrl: manifest?.shareUrl ?? null, mimeType: orphan.mimeType, @@ -710,6 +882,18 @@ const sweepOrphanedRecordingSpools = async () => { createdAt: orphan.updatedAt, message: "The recording was interrupted before its upload finished.", }); + if (manifest?.cameraSessionId && cameraSession?.totalBytes) { + knownSessions.add(manifest.cameraSessionId); + remainingSessions.add(manifest.cameraSessionId); + } + if (manifest?.inputEventsSessionId && inputSession?.totalBytes) { + knownSessions.add(manifest.inputEventsSessionId); + remainingSessions.add(manifest.inputEventsSessionId); + } + for (const source of audioSources) { + knownSessions.add(source.sessionId); + remainingSessions.add(source.sessionId); + } } } @@ -724,6 +908,37 @@ const sweepOrphanedRecordingSpools = async () => { await deleteRecoveredRecordingSpool(entry.sessionId).catch( () => undefined, ); + if (entry.cameraSessionId) { + survivingSessions.delete(entry.cameraSessionId); + await deleteRecoveredRecordingSpool(entry.cameraSessionId).catch( + () => undefined, + ); + } + if (entry.inputEventsSessionId) { + survivingSessions.delete(entry.inputEventsSessionId); + await deleteRecoveredRecordingSpool(entry.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of entry.audioSources ?? []) { + survivingSessions.delete(source.sessionId); + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } + } + if (startInProgress && !activeRecording) return; + if (activeRecording) { + survivingSessions.add(activeRecording.spool.sessionId); + if (activeRecording.cameraSpool) { + survivingSessions.add(activeRecording.cameraSpool.sessionId); + } + if (activeRecording.inputSidecar) { + survivingSessions.add(activeRecording.inputSidecar.spool.sessionId); + } + for (const sidecar of activeRecording.audioSidecars) { + survivingSessions.add(sidecar.metadata.sessionId); + } } await pruneLiveRecordingManifests(survivingSessions).catch(() => undefined); } catch { @@ -736,7 +951,10 @@ function stopRecorderAfterError(recorder: MediaRecorder) { recorder.stop(); } window.setTimeout(() => { - if (activeRecording?.recorder === recorder) { + if ( + activeRecording?.recorder === recorder || + activeRecording?.cameraRecorder === recorder + ) { void stopRecording(); } }, 0); @@ -761,6 +979,20 @@ const stopManualChunking = (recording: ActiveRecording) => { } }; +const requestCameraRecorderData = (recording: ActiveRecording) => { + if (recording.cameraRecorder?.state !== "recording") return; + try { + recording.cameraRecorder.requestData(); + } catch {} +}; + +const stopCameraChunking = (recording: ActiveRecording) => { + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + recording.cameraDataRequestInterval = null; + } +}; + const beginManualChunking = (recording: ActiveRecording) => { recording.chunkingMode = "manual"; recording.lastChunkAt = null; @@ -811,13 +1043,23 @@ const startRecording = async (request: StartRecordingRequest) => { const ownedStreams: MediaStream[] = []; let ownedVideoId: string | null = null; let ownedSpool: RecordingSpool | null = null; + let ownedCameraSpool: RecordingSpool | null = null; + let ownedCameraUploader: InstantRecordingUploader | null = null; + let ownedInputSidecar: InputEventSidecar | null = null; + const ownedAudioSidecars: AudioRecordingSidecar[] = []; + let ownedUploader: InstantRecordingUploader | null = null; let ownedRecording: ActiveRecording | null = null; let countdownPromise: Promise | null = null; try { status = { phase: "creating" }; - const mainStream = await getMainStream(request); + const mainStream = await getMainStream(request).catch((error: unknown) => { + throw new Error( + `Unable to capture ${request.mode} source: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + }); ownedStreams.push(mainStream); throwIfStartCanceled(); const captureSource = getCaptureSource(request, mainStream); @@ -847,6 +1089,34 @@ const startRecording = async (request: StartRecordingRequest) => { streams, routeFirstStreamToSpeakers: request.mode === "tab", }); + const cameraStream = + request.mode !== "camera" && request.settings.webcam.enabled + ? await getCameraPreviewStream(request.settings.webcam) + : null; + const cameraTrack = cameraStream?.getVideoTracks()[0]?.clone() ?? null; + if ( + request.settings.webcam.enabled && + request.mode !== "camera" && + !cameraTrack + ) { + throw new Error("The selected camera is not available for recording"); + } + const cameraRecordingStream = cameraTrack + ? new MediaStream([cameraTrack]) + : null; + if (cameraRecordingStream) { + ownedStreams.push(cameraRecordingStream); + streams.push(cameraRecordingStream); + } + const cameraDimensions = cameraRecordingStream + ? getStreamSize(cameraRecordingStream) + : null; + const cameraPipeline = cameraRecordingStream + ? selectRecordingPipeline(false) + : null; + if (cameraRecordingStream && !cameraPipeline) { + throw new Error("No supported camera recorder format is available"); + } const hasAudio = recordingStream.getAudioTracks().length > 0; const pipeline = selectRecordingPipeline(hasAudio); if (!pipeline) throw new Error("No supported recorder format is available"); @@ -883,10 +1153,6 @@ const startRecording = async (request: StartRecordingRequest) => { subpath, api, }); - throwIfStartCanceled(); - const spool = await RecordingSpool.create({ mimeType: pipeline.mimeType }); - ownedSpool = spool; - throwIfStartCanceled(); const uploader = new InstantRecordingUploader({ videoId: creation.id, uploadId: uploadSession.uploadId, @@ -940,15 +1206,133 @@ const startRecording = async (request: StartRecordingRequest) => { } }, }); + ownedUploader = uploader; + throwIfStartCanceled(); + const spool = await RecordingSpool.create({ mimeType: pipeline.mimeType }); + ownedSpool = spool; + throwIfStartCanceled(); + const cameraSpool = cameraPipeline + ? await RecordingSpool.create({ mimeType: cameraPipeline.mimeType }) + : null; + ownedCameraSpool = cameraSpool; + throwIfStartCanceled(); + const cameraSubpath = cameraPipeline + ? `camera-upload.${cameraPipeline.fileExtension}` + : null; + const cameraUploadApi = cameraSubpath + ? { + ...api, + extraBody: { screenSubpath: subpath, cameraOffsetMs: 0 }, + } + : null; + const cameraUploadSession = + cameraPipeline && cameraSubpath && cameraUploadApi + ? await initiateMultipartUpload({ + videoId: creation.id, + contentType: cameraPipeline.mimeType, + subpath: cameraSubpath, + api: cameraUploadApi, + }) + : null; + const cameraUploader = + cameraPipeline && cameraSubpath && cameraUploadApi && cameraUploadSession + ? new InstantRecordingUploader({ + videoId: creation.id, + uploadId: cameraUploadSession.uploadId, + provider: cameraUploadSession.provider, + mimeType: cameraPipeline.mimeType, + subpath: cameraSubpath, + api: cameraUploadApi, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + }) + : null; + ownedCameraUploader = cameraUploader; + throwIfStartCanceled(); + if (microphoneStream?.getAudioTracks().length) { + ownedAudioSidecars.push( + await AudioRecordingSidecar.create({ + kind: "mic", + stream: new MediaStream(microphoneStream.getAudioTracks()), + videoId: creation.id, + screenSubpath: subpath, + api, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + onBackupFallback: (error) => { + console.warn("Microphone backup moved to memory", error); + }, + }), + ); + } + if (mainStream.getAudioTracks().length > 0) { + ownedAudioSidecars.push( + await AudioRecordingSidecar.create({ + kind: "systemAudio", + stream: new MediaStream(mainStream.getAudioTracks()), + videoId: creation.id, + screenSubpath: subpath, + api, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + onBackupFallback: (error) => { + console.warn("System audio backup moved to memory", error); + }, + }), + ); + } + throwIfStartCanceled(); const recorder = new MediaRecorder(recordingStream, { mimeType: pipeline.mimeType, }); + const cameraRecorder = + cameraRecordingStream && cameraPipeline + ? new MediaRecorder(cameraRecordingStream, { + mimeType: cameraPipeline.mimeType, + }) + : null; await countdownPromise; throwIfStartCanceled(); const startedAt = Date.now(); + const inputSidecar = + request.tabStreamId && request.tabId !== undefined + ? await InputEventSidecar.create({ + recordingId: creation.id, + tabId: request.tabId, + videoWidth: width, + videoHeight: height, + }).catch((error: unknown) => { + console.warn("Tab input capture could not start", error); + return null; + }) + : null; + ownedInputSidecar = inputSidecar; const plan = request.bootstrap.plan; const maxDurationMs = !plan.isPro && plan.maxRecordingSeconds !== null @@ -957,13 +1341,28 @@ const startRecording = async (request: StartRecordingRequest) => { const recording: ActiveRecording = { recorder, + cameraRecorder, + audioSidecars: ownedAudioSidecars, + inputSidecar, + inputCaptureTabId: + request.tabStreamId && request.tabId !== undefined + ? request.tabId + : null, + inputUploadApi: api, + inputStopPromise: null, + inputCaptureIncomplete: false, stopPromise: Promise.resolve(), + cameraStopPromise: null, streams, + microphoneStream, recordingStream, statusTimer: null, spool, + cameraSpool, uploader, + cameraUploader, startedAt, + cameraOffsetMs: null, durationMs: 0, lastResumedAt: startedAt, videoId: creation.id, @@ -971,20 +1370,31 @@ const startRecording = async (request: StartRecordingRequest) => { width, height, fps, + cameraWidth: cameraDimensions?.width ?? null, + cameraHeight: cameraDimensions?.height ?? null, + cameraFps: cameraDimensions?.fps ?? null, subpath, mimeType: pipeline.mimeType, + cameraSubpath, + cameraMimeType: cameraPipeline?.mimeType ?? null, maxDurationMs, audioContext, chunkChain: Promise.resolve(), + cameraChunkChain: Promise.resolve(), dataRequestInterval: null, + cameraDataRequestInterval: null, chunkStartGuard: null, chunkingMode: null, lastChunkAt: null, recordedBytes: 0, + cameraRecordedBytes: 0, + cameraUploadCompleted: false, finalizePromise: null, cleanedUp: false, spoolFailed: false, memoryBackup: initialLocalRecordingState(), + cameraSpoolFailed: false, + cameraMemoryBackup: initialLocalRecordingState(), }; ownedRecording = recording; @@ -999,6 +1409,19 @@ const startRecording = async (request: StartRecordingRequest) => { // failed-recording entry (videoId, subpath) instead of download-only. await saveLiveRecordingManifest({ sessionId: spool.sessionId, + inputEventsSessionId: inputSidecar?.spool.sessionId, + audioSources: ownedAudioSidecars.map((sidecar) => sidecar.metadata), + ...(cameraSpool + ? { + cameraSessionId: cameraSpool.sessionId, + cameraMimeType: cameraPipeline?.mimeType, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + } + : {}), videoId: creation.id, shareUrl: creation.shareUrl, mimeType: pipeline.mimeType, @@ -1013,6 +1436,13 @@ const startRecording = async (request: StartRecordingRequest) => { recorder.onstop = () => resolve(); recorder.onerror = () => reject(new Error("MediaRecorder failed")); }); + if (cameraRecorder) { + recording.cameraStopPromise = new Promise((resolve, reject) => { + cameraRecorder.onstop = () => resolve(); + cameraRecorder.onerror = () => + reject(new Error("Camera MediaRecorder failed")); + }); + } // A mid-recording recorder failure must stop the session right away; // without this the rejection sits unhandled while the timer keeps // ticking over a recorder that no longer produces chunks, and nothing @@ -1073,6 +1503,51 @@ const startRecording = async (request: StartRecordingRequest) => { stopRecorderAfterError(recorder); } }; + if (cameraRecorder && cameraSpool) { + cameraRecorder.ondataavailable = (event) => { + if (event.data.size === 0) return; + recording.cameraRecordedBytes += event.data.size; + const cameraRecordedBytes = recording.cameraRecordedBytes; + recording.cameraChunkChain = recording.cameraChunkChain.then( + async () => { + if (recording.cameraSpoolFailed) { + appendCameraMemoryBackupChunk(recording, event.data); + return; + } + try { + await cameraSpool.appendChunk(event.data); + } catch (error) { + recording.cameraSpoolFailed = true; + console.warn( + "Camera recording spool failed; keeping the local backup in memory", + error, + ); + } + }, + ); + try { + cameraUploader?.handleChunk(event.data, cameraRecordedBytes); + } catch (error) { + status = { + phase: "error", + message: error instanceof Error ? error.message : String(error), + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + } + }; + recording.cameraStopPromise?.catch(() => { + if (activeRecording !== recording || recording.finalizePromise) return; + status = { + phase: "error", + message: "Camera recording stopped unexpectedly", + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }); + } status = { phase: "recording", @@ -1081,15 +1556,63 @@ const startRecording = async (request: StartRecordingRequest) => { durationMs: 0, updatedAt: startedAt, }; + const saveSourceManifest = () => + saveLiveRecordingManifest({ + sessionId: spool.sessionId, + inputEventsSessionId: inputSidecar?.spool.sessionId, + audioSources: ownedAudioSidecars.map((sidecar) => sidecar.metadata), + cameraSessionId: cameraSpool?.sessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + videoId: creation.id, + shareUrl: creation.shareUrl, + mimeType: pipeline.mimeType, + subpath, + width, + height, + fps, + startedAt, + }); + let screenStartRequestedAt = performance.now(); try { recorder.start(RECORDING_TIMESLICE_MS); recording.chunkingMode = "timeslice"; scheduleTimesliceGuard(recording); } catch { + screenStartRequestedAt = performance.now(); recorder.start(); beginManualChunking(recording); } + inputSidecar?.start(performance.timeOrigin + screenStartRequestedAt); + if (cameraRecorder) { + let cameraStartRequestedAt = performance.now(); + try { + cameraRecorder.start(RECORDING_TIMESLICE_MS); + } catch { + cameraStartRequestedAt = performance.now(); + cameraRecorder.start(); + recording.cameraDataRequestInterval = window.setInterval(() => { + requestCameraRecorderData(recording); + }, RECORDING_TIMESLICE_MS); + } + recording.cameraOffsetMs = Math.round( + cameraStartRequestedAt - screenStartRequestedAt, + ); + if (cameraUploadApi) { + cameraUploadApi.extraBody.cameraOffsetMs = recording.cameraOffsetMs; + } + } + for (const audioSidecar of ownedAudioSidecars) { + audioSidecar.start(screenStartRequestedAt); + } + if (cameraRecorder || ownedAudioSidecars.length > 0) { + await saveSourceManifest().catch(() => undefined); + } playRecordingSound("start-recording", request.settings); // The service worker that sent start-recording may have been killed // while the capture picker was open, which destroys the response @@ -1109,12 +1632,38 @@ const startRecording = async (request: StartRecordingRequest) => { ); countdownResolve?.(); await countdownPromise?.catch(() => undefined); + if (ownedRecording) { + if (ownedRecording.recorder.state !== "inactive") { + ownedRecording.recorder.stop(); + await ownedRecording.stopPromise.catch(() => undefined); + } + if ( + ownedRecording.cameraRecorder && + ownedRecording.cameraRecorder.state !== "inactive" + ) { + ownedRecording.cameraRecorder.stop(); + await ownedRecording.cameraStopPromise?.catch(() => undefined); + } + await Promise.all([ + ownedRecording.chunkChain, + ownedRecording.cameraChunkChain, + ]); + } + await Promise.all( + ownedAudioSidecars.map((sidecar) => + sidecar.cancel().catch(() => undefined), + ), + ); // Release only what this attempt acquired. No chunk can have been // captured yet — chunks only flow once recorder.start() succeeds, after // which nothing here throws — so the spool holds no recoverable data. for (const stream of ownedStreams) { stopTracks(stream); } + await Promise.all([ + ownedUploader?.cancel().catch(() => undefined), + ownedCameraUploader?.cancel().catch(() => undefined), + ]); if (ownedVideoId) { await deleteInstantRecording( request.settings, @@ -1134,15 +1683,23 @@ const startRecording = async (request: StartRecordingRequest) => { ); await ownedSpool.dispose().catch(() => undefined); } + await ownedCameraSpool?.dispose().catch(() => undefined); + await ownedInputSidecar?.dispose().catch(() => undefined); // Reset the "creating" status so later status syncs do not report a // phantom in-progress recording. - if (status.phase === "creating") { + if ( + status.phase === "creating" || + (ownedRecording && + "videoId" in status && + status.videoId === ownedRecording.videoId) + ) { status = isUserCancellationError(error) ? { phase: "idle" } : { phase: "error", message: error instanceof Error ? error.message : String(error), }; + broadcastStatus(); } throw error; } finally { @@ -1163,6 +1720,22 @@ const appendMemoryBackupChunk = (recording: ActiveRecording, chunk: Blob) => { } }; +const appendCameraMemoryBackupChunk = ( + recording: ActiveRecording, + chunk: Blob, +) => { + const previous = recording.cameraMemoryBackup; + recording.cameraMemoryBackup = appendLocalRecordingChunk(previous, chunk, { + mode: "capped", + maxBytes: MEMORY_BACKUP_MAX_BYTES, + }); + if (recording.cameraMemoryBackup.overflowed && !previous.overflowed) { + console.warn( + "In-memory camera backup exceeded its cap; dropping the local copy", + ); + } +}; + // The complete recording: the spool when it stayed healthy, otherwise the // spooled prefix plus the in-memory continuation. Null once the capped // memory backup has overflowed — a truncated local copy must not masquerade @@ -1180,6 +1753,68 @@ const recoverRecordingBlob = async (recording: ActiveRecording) => { return new Blob(parts, { type: recording.mimeType }); }; +const recoverCameraRecordingBlob = async (recording: ActiveRecording) => { + const spool = recording.cameraSpool; + if (!spool) return null; + if (!recording.cameraSpoolFailed) return spool.recoverBlob(); + if (recording.cameraMemoryBackup.overflowed) return null; + const spooledBlob = await spool.recoverBlob().catch(() => null); + const parts = spooledBlob + ? [spooledBlob, ...recording.cameraMemoryBackup.chunks] + : recording.cameraMemoryBackup.chunks; + if (parts.length === 0) return null; + return new Blob(parts, { type: recording.cameraMimeType ?? "" }); +}; + +const uploadCameraSidecar = async (options: { + videoId: VideoId; + settings: ExtensionSettings; + authToken: string; + blob: Blob; + mimeType: string; + subpath: string; + screenSubpath: string; + cameraOffsetMs: number; + durationMs: number; + width: number; + height: number; + fps: number; +}) => { + const api = { + baseUrl: options.settings.apiBaseUrl, + authToken: options.authToken, + requestTimeoutMs: DEFAULT_API_REQUEST_TIMEOUT_MS, + extraBody: { + screenSubpath: options.screenSubpath, + cameraOffsetMs: options.cameraOffsetMs, + }, + }; + const uploadSession = await initiateMultipartUpload({ + videoId: options.videoId, + contentType: options.mimeType, + subpath: options.subpath, + api, + }); + const uploader = new InstantRecordingUploader({ + videoId: options.videoId, + uploadId: uploadSession.uploadId, + provider: uploadSession.provider, + mimeType: options.mimeType, + subpath: options.subpath, + api, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + }); + await uploader.finalize({ + finalBlob: options.blob, + durationSeconds: Math.max(1, Math.round(options.durationMs / 1000)), + width: options.width, + height: options.height, + fps: options.fps, + subpath: options.subpath, + }); +}; + // Returns whether the recording bytes are persisted and retryable from the // upload page. const rememberFailedRecording = async ( @@ -1189,6 +1824,7 @@ const rememberFailedRecording = async ( if (recording.recordedBytes === 0) return false; let sessionId = recording.spool.sessionId; + let cameraSessionId = recording.cameraSpool?.sessionId; if (recording.spoolFailed) { // The original spool is missing the tail that went to memory; a retry // reading it would upload a truncated file. Persist the full recording @@ -1214,9 +1850,85 @@ const rememberFailedRecording = async ( if (!replacement) return false; sessionId = replacement.sessionId; } + if (recording.cameraSpoolFailed && recording.cameraSpool) { + const fullBlob = await recoverCameraRecordingBlob(recording).catch( + () => null, + ); + const replacement = + fullBlob && fullBlob.size >= recording.cameraRecordedBytes + ? await RecordingSpool.create({ + mimeType: recording.cameraMimeType ?? fullBlob.type, + maxPendingChunkBytes: fullBlob.size, + }) + .then(async (spool) => { + await spool.appendChunk(fullBlob); + await spool.flush(); + return spool; + }) + .catch(() => null) + : null; + await recording.cameraSpool.dispose().catch(() => undefined); + cameraSessionId = replacement?.sessionId; + } + const audioSources = await Promise.all( + recording.audioSidecars.map((sidecar) => + sidecar.prepareRetryMetadata().catch(() => null), + ), + ); + const retryableAudioSources = audioSources.filter( + (source) => source !== null, + ); + const audioRetryUnavailable = audioSources.some( + (source, index) => + !recording.audioSidecars[index]?.isUploadCompleted && source === null, + ); + const inputSidecar = recording.inputSidecar; + const inputBlob = + inputSidecar && inputSidecar.recordedBytes > 0 + ? await inputSidecar.recoverBlob().catch(() => null) + : null; + const inputRetryUnavailable = + !!inputSidecar && + inputSidecar.recordedBytes > 0 && + !inputSidecar.isUploadCompleted && + (recording.inputCaptureIncomplete || + !inputBlob || + inputBlob.size !== inputSidecar.recordedBytes); const saved = await upsertFailedRecording({ sessionId, + ...(retryableAudioSources.length > 0 + ? { audioSources: retryableAudioSources } + : {}), + ...(audioRetryUnavailable ? { audioRetryUnavailable: true } : {}), + ...(inputRetryUnavailable ? { inputEventsRetryUnavailable: true } : {}), + ...(inputSidecar && + inputSidecar.recordedBytes > 0 && + inputBlob && + inputBlob.size === inputSidecar.recordedBytes && + !recording.inputCaptureIncomplete + ? { + inputEventsSessionId: inputSidecar.spool.sessionId, + inputEventsTotalBytes: inputSidecar.recordedBytes, + } + : {}), + ...(recording.cameraRecorder && + !recording.cameraUploadCompleted && + !(cameraSessionId && recording.cameraRecordedBytes > 0) + ? { cameraRetryUnavailable: true } + : {}), + ...(cameraSessionId && recording.cameraRecordedBytes > 0 + ? { + cameraSessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + cameraTotalBytes: recording.cameraRecordedBytes, + } + : {}), videoId: recording.videoId, shareUrl: recording.shareUrl, mimeType: recording.mimeType, @@ -1237,6 +1949,21 @@ const rememberFailedRecording = async ( await deleteRecoveredRecordingSpool(dropped.sessionId).catch( () => undefined, ); + if (dropped.cameraSessionId) { + await deleteRecoveredRecordingSpool(dropped.cameraSessionId).catch( + () => undefined, + ); + } + if (dropped.inputEventsSessionId) { + await deleteRecoveredRecordingSpool(dropped.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of dropped.audioSources ?? []) { + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } } return saved.kept.some((entry) => entry.sessionId === sessionId); @@ -1244,9 +1971,13 @@ const rememberFailedRecording = async ( const finalizeRecording = async (recording: ActiveRecording) => { try { - await recording.stopPromise; + await Promise.all([ + recording.stopPromise, + recording.cameraStopPromise ?? Promise.resolve(), + ]); + await recording.inputStopPromise; await cleanupActiveRecording(recording); - await recording.chunkChain; + await Promise.all([recording.chunkChain, recording.cameraChunkChain]); // finalBlob is null when the capped memory backup overflowed. The // streamed parts still carry every byte; the uploader just falls back // to its recorded-bytes counter instead of the local blob's size. @@ -1254,6 +1985,50 @@ const finalizeRecording = async (recording: ActiveRecording) => { if ((!finalBlob || finalBlob.size === 0) && recording.recordedBytes === 0) { throw new Error("No recording data was captured"); } + if (recording.cameraRecorder && recording.cameraRecordedBytes === 0) { + throw new Error("The separate camera recording captured no data"); + } + if (recording.cameraRecordedBytes > 0) { + const cameraUploader = recording.cameraUploader; + const cameraSubpath = recording.cameraSubpath; + if ( + !recording.cameraSpool || + !cameraUploader || + !cameraSubpath || + !recording.cameraMimeType || + recording.cameraOffsetMs === null + ) { + throw new Error("The separate camera recording metadata is incomplete"); + } + await cameraUploader.finalize({ + finalBlob: null, + durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), + width: recording.cameraWidth ?? DEFAULT_WIDTH, + height: recording.cameraHeight ?? DEFAULT_HEIGHT, + fps: recording.cameraFps ?? DEFAULT_FPS, + subpath: cameraSubpath, + }); + recording.cameraUploadCompleted = true; + } else { + await recording.cameraUploader?.cancel(); + } + for (const audioSidecar of recording.audioSidecars) { + await audioSidecar.finalize( + Math.max(1, Math.round(recording.durationMs / 1000)), + ); + } + if (recording.inputSidecar && !recording.inputCaptureIncomplete) { + await recording.inputSidecar + .upload({ + videoId: recording.videoId, + screenSubpath: recording.subpath, + durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), + api: recording.inputUploadApi, + }) + .catch((error: unknown) => { + console.warn("Tab input events could not upload", error); + }); + } await recording.uploader.finalize({ finalBlob: finalBlob && finalBlob.size > 0 ? finalBlob : null, durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), @@ -1263,6 +2038,13 @@ const finalizeRecording = async (recording: ActiveRecording) => { subpath: recording.subpath, }); await recording.spool.dispose(); + await recording.cameraSpool?.dispose(); + await recording.inputSidecar?.dispose().catch(() => undefined); + await Promise.all( + recording.audioSidecars.map((sidecar) => + sidecar.disposeBackup().catch(() => undefined), + ), + ); await removeFailedRecording(recording.spool.sessionId).catch( () => undefined, ); @@ -1331,6 +2113,24 @@ const getCurrentUploadSnapshot = () => uploadStatus: undefined, }; +const stopTabInputCapture = async (recording: ActiveRecording) => { + if (recording.inputCaptureTabId === null) return; + const response: unknown = await chrome.runtime.sendMessage({ + target: "service-worker", + type: "input-capture-stop", + tabId: recording.inputCaptureTabId, + recordingId: recording.videoId, + }); + if ( + !response || + typeof response !== "object" || + !("ok" in response) || + response.ok !== true + ) { + throw new Error("Tab input events could not be flushed"); + } +}; + async function stopRecording() { // A stop during the pre-roll countdown cancels the start before any frame is // captured. Resolving the countdown wait lets startRecording's @@ -1354,13 +2154,31 @@ async function stopRecording() { } stopManualChunking(recording); + stopCameraChunking(recording); const now = Date.now(); recording.durationMs = getRecordingDuration(recording, now); recording.lastResumedAt = null; + recording.inputSidecar?.stop(performance.timeOrigin + performance.now()); + recording.inputStopPromise = + recording.inputCaptureTabId !== null + ? stopTabInputCapture(recording).catch((error: unknown) => { + recording.inputCaptureIncomplete = true; + console.warn("Tab input events could not flush", error); + }) + : null; if (recording.recorder.state !== "inactive") { recording.recorder.stop(); } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } + for (const audioSidecar of recording.audioSidecars) { + void audioSidecar.stop().catch(() => undefined); + } if (status.phase !== "error") { const snapshot = getCurrentUploadSnapshot(); @@ -1396,6 +2214,13 @@ const pauseRecording = () => { recording.durationMs = getRecordingDuration(recording, now); recording.lastResumedAt = null; recording.recorder.pause(); + recording.inputSidecar?.pause(performance.timeOrigin + performance.now()); + if (recording.cameraRecorder?.state === "recording") { + recording.cameraRecorder.pause(); + } + for (const audioSidecar of recording.audioSidecars) { + audioSidecar.pause(); + } if (status.phase === "recording") { status = { ...status, @@ -1414,6 +2239,13 @@ const resumeRecording = () => { } const now = Date.now(); recording.recorder.resume(); + recording.inputSidecar?.resume(performance.timeOrigin + performance.now()); + if (recording.cameraRecorder?.state === "paused") { + recording.cameraRecorder.resume(); + } + for (const audioSidecar of recording.audioSidecars) { + audioSidecar.resume(); + } recording.lastResumedAt = now; if (status.phase === "paused") { status = { @@ -1450,10 +2282,24 @@ const runFailedUploadRetry = async ( if (!failed?.videoId) { throw new Error("This recording is no longer available to retry."); } + if (failed.cameraRetryUnavailable) { + throw new Error( + "The separate camera recording is unavailable to retry. Download the screen recording or check your Cap.", + ); + } + if (failed.audioRetryUnavailable) { + throw new Error( + "A separate audio source is unavailable to retry. Download the saved sources or check your Cap.", + ); + } + if (failed.inputEventsRetryUnavailable) { + console.warn("Tab input events are unavailable during recording retry"); + } const orphan = await recoverRecordingSpoolSession(failed.sessionId); if (!orphan || orphan.blob.size === 0) { await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); throw new Error("The recorded data is no longer available."); } @@ -1497,6 +2343,78 @@ const runFailedUploadRetry = async ( }; try { + if (failed.cameraSessionId) { + const cameraOrphan = await recoverRecordingSpoolSession( + failed.cameraSessionId, + ); + if ( + !cameraOrphan || + cameraOrphan.blob.size === 0 || + !failed.cameraMimeType || + !failed.cameraSubpath || + typeof failed.cameraOffsetMs !== "number" || + !Number.isFinite(failed.cameraOffsetMs) + ) { + throw new Error("The separate camera recording is no longer available"); + } + await uploadCameraSidecar({ + videoId: typedVideoId, + settings, + authToken: auth.authApiKey, + blob: cameraOrphan.blob, + mimeType: failed.cameraMimeType, + subpath: failed.cameraSubpath, + screenSubpath: subpath, + cameraOffsetMs: failed.cameraOffsetMs, + durationMs: failed.durationMs, + width: failed.cameraWidth ?? failed.width ?? DEFAULT_WIDTH, + height: failed.cameraHeight ?? failed.height ?? DEFAULT_HEIGHT, + fps: failed.cameraFps ?? failed.fps ?? DEFAULT_FPS, + }); + } + for (const source of failed.audioSources ?? []) { + const audioOrphan = await recoverRecordingSpoolSession(source.sessionId); + if ( + !audioOrphan || + audioOrphan.blob.size !== source.recordedBytes || + audioOrphan.blob.size === 0 + ) { + throw new Error(`The separate ${source.kind} audio is unavailable`); + } + await uploadRecoveredAudioSidecar({ + videoId: typedVideoId, + source, + blob: audioOrphan.blob, + screenSubpath: subpath, + durationSeconds: Math.max(1, Math.round(failed.durationMs / 1000)), + api, + }); + } + if (failed.inputEventsSessionId && !failed.inputEventsRetryUnavailable) { + const inputOrphan = await recoverRecordingSpoolSession( + failed.inputEventsSessionId, + ).catch((error: unknown) => { + console.warn("Tab input events could not be recovered", error); + return null; + }); + if ( + inputOrphan && + inputOrphan.blob.size === failed.inputEventsTotalBytes && + inputOrphan.blob.size > 0 + ) { + await uploadInputEventBlob({ + videoId: typedVideoId, + screenSubpath: subpath, + durationSeconds: Math.max(1, Math.round(failed.durationMs / 1000)), + api, + blob: inputOrphan.blob, + }).catch((error: unknown) => { + console.warn("Tab input events could not upload during retry", error); + }); + } else { + console.warn("Tab input events are unavailable during recording retry"); + } + } const uploadSession = await initiateMultipartUpload({ videoId: typedVideoId, contentType: failed.mimeType, @@ -1543,7 +2461,23 @@ const runFailedUploadRetry = async ( await deleteRecoveredRecordingSpool(failed.sessionId).catch( () => undefined, ); + if (failed.cameraSessionId) { + await deleteRecoveredRecordingSpool(failed.cameraSessionId).catch( + () => undefined, + ); + } + if (failed.inputEventsSessionId) { + await deleteRecoveredRecordingSpool(failed.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of failed.audioSources ?? []) { + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); setRetryStatus({ phase: "completed", videoId: typedVideoId, @@ -1708,12 +2642,7 @@ const handleRequest = async ( if (message.type === "toggle-microphone-mute") { const recording = activeRecording; if (recording) { - for (const stream of recording.streams) { - for (const track of stream.getAudioTracks()) { - track.enabled = !message.muted; - } - } - for (const track of recording.recordingStream.getAudioTracks()) { + for (const track of recording.microphoneStream?.getAudioTracks() ?? []) { track.enabled = !message.muted; } } @@ -1724,6 +2653,24 @@ const handleRequest = async ( }; chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (message?.type === "input-events-batch") { + const batch = parseCapturedTabInputBatch(message); + const recording = activeRecording; + if (!batch || !recording?.inputSidecar) { + sendResponse({ ok: false, error: "Tab input recording is unavailable" }); + return false; + } + recording.inputSidecar + .accept(batch, _sender.tab?.id) + .then(() => sendResponse({ ok: true })) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } if (!isOffscreenRequest(message)) return false; handleRequest(message) diff --git a/apps/chrome-extension/src/options/main.tsx b/apps/chrome-extension/src/options/main.tsx index 05a5d1ad081..008d62647ef 100644 --- a/apps/chrome-extension/src/options/main.tsx +++ b/apps/chrome-extension/src/options/main.tsx @@ -1,6 +1,7 @@ import { deleteRecoveredRecordingSpool, - recoverOrphanedRecordingSpools, + listRecordingSpoolSessions, + recoverRecordingSpoolSession, } from "@cap/recorder-core"; import { useCallback, useEffect, useState } from "react"; import { createRoot } from "react-dom/client"; @@ -15,9 +16,11 @@ import { type FailedRecording, loadAuth, loadFailedRecordings, + loadLiveRecordingManifests, loadSettings, loadSharedRecordingState, removeFailedRecording, + removeLiveRecordingManifest, saveSettings, } from "../shared/storage"; import type { ExtensionAuth, ExtensionSettings } from "../shared/types"; @@ -51,13 +54,31 @@ const isActiveRecordingPhase = (phase: string | undefined) => // hold bytes, and also surfaces spools stranded by a crash before the // offscreen sweep could record them (those have no metadata yet). const loadRecoveredRecordings = async (): Promise => { - const [failed, spools, recordingState] = await Promise.all([ + const [failed, spools, manifests, recordingState] = await Promise.all([ loadFailedRecordings(), - recoverOrphanedRecordingSpools(), + listRecordingSpoolSessions(), + loadLiveRecordingManifests(), loadSharedRecordingState().catch(() => null), ]); - const spoolSessions = new Set(spools.map((spool) => spool.sessionId)); - const knownSessions = new Set(failed.map((entry) => entry.sessionId)); + const spoolSessions = new Set( + spools + .filter((spool) => spool.totalBytes > 0) + .map((spool) => spool.sessionId), + ); + const knownSessions = new Set( + failed.flatMap((entry) => [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ]), + ); + for (const manifest of manifests) { + if (!spoolSessions.has(manifest.sessionId)) continue; + if (manifest.cameraSessionId) knownSessions.add(manifest.cameraSessionId); + for (const source of manifest.audioSources ?? []) { + knownSessions.add(source.sessionId); + } + } const entries = failed.filter((entry) => spoolSessions.has(entry.sessionId)); // Skip unknown spools while a recording is live anywhere: an in-flight @@ -139,12 +160,29 @@ function RecoveredRecordingsSection() { } }; - const download = (entry: FailedRecording) => + const download = ( + entry: FailedRecording, + kind: "screen" | "camera" | "mic" | "systemAudio", + ) => runAction(entry.sessionId, async () => { - const spools = await recoverOrphanedRecordingSpools(); - const spool = spools.find( - (candidate) => candidate.sessionId === entry.sessionId, + const audioSource = entry.audioSources?.find( + (source) => source.kind === kind, ); + const sessionId = + kind === "screen" + ? entry.sessionId + : kind === "camera" + ? entry.cameraSessionId + : audioSource?.sessionId; + const mimeType = + kind === "screen" + ? entry.mimeType + : kind === "camera" + ? entry.cameraMimeType + : audioSource?.mimeType; + const spool = sessionId + ? await recoverRecordingSpoolSession(sessionId) + : null; if (!spool || spool.blob.size === 0) { refresh(); throw new Error("The recorded data is no longer available."); @@ -152,10 +190,21 @@ function RecoveredRecordingsSection() { const url = URL.createObjectURL(spool.blob); const anchor = document.createElement("a"); anchor.href = url; - anchor.download = `cap-recording-${ + const recordingName = entry.videoId ?? - new Date(entry.createdAt).toISOString().replace(/[:.]/g, "-") - }.${fileExtensionForMimeType(entry.mimeType)}`; + new Date(entry.createdAt).toISOString().replace(/[:.]/g, "-"); + const separateSources = + !!entry.cameraSessionId || + entry.cameraRetryUnavailable === true || + (entry.audioSources?.length ?? 0) > 0 || + entry.audioRetryUnavailable === true; + const label = + kind === "screen" + ? separateSources + ? "-screen" + : "" + : `-${kind === "systemAudio" ? "system-audio" : kind}`; + anchor.download = `cap-recording-${recordingName}${label}.${fileExtensionForMimeType(mimeType ?? spool.mimeType)}`; anchor.click(); window.setTimeout(() => URL.revokeObjectURL(url), 10_000); return null; @@ -163,8 +212,15 @@ function RecoveredRecordingsSection() { const remove = (entry: FailedRecording) => runAction(entry.sessionId, async () => { - await deleteRecoveredRecordingSpool(entry.sessionId); + await Promise.all( + [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ].map((sessionId) => deleteRecoveredRecordingSpool(sessionId)), + ); await removeFailedRecording(entry.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(entry.sessionId).catch(() => undefined); refresh(); return null; }); @@ -222,26 +278,56 @@ function RecoveredRecordingsSection() {
- {entry.videoId && ( - + {entry.videoId && + !entry.cameraRetryUnavailable && + !entry.audioRetryUnavailable && ( + + )} + {(entry.cameraRetryUnavailable || + entry.audioRetryUnavailable) && ( + + A separate source is missing. Download available sources. + )} + {entry.cameraSessionId && !entry.cameraRetryUnavailable && ( + + )} + {entry.audioSources?.map((source) => ( + + ))} +

diff --git a/apps/desktop-gpui/src/assets.rs b/apps/desktop-gpui/src/assets.rs index 65eabb37589..9c814a2e8ad 100644 --- a/apps/desktop-gpui/src/assets.rs +++ b/apps/desktop-gpui/src/assets.rs @@ -85,7 +85,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "laptop.svg", "shuffle.svg", "gift.svg", - "history.svg", "hotkeys.svg", "image.svg", "info.svg", @@ -99,7 +98,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "message-square-plus.svg", "mic-off.svg", "microphone.svg", - "minimize.svg", "minus.svg", "monitor.svg", "more-vertical.svg", @@ -189,7 +187,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": // The main window's hand-drawn traffic lights: the x and expand glyphs // `CaptionControlsMacOS.tsx` inlines, shown while the group is hovered. "traffic-close.svg", - "traffic-zoom.svg", // The remaining settings pages (`settings_pages.rs`). `circle-check` is // Cap's own (`packages/ui-solid/icons/circle-check.svg`, hotkeys.tsx's // IconCapCircleCheck); the rest are the Lucide 24x24 originals the pages' @@ -335,6 +332,7 @@ mod tests { // `ui::SelectionHeader` names the check and the trash itself. include_str!("ui/selection_header.rs"), include_str!("ui/radio_cards.rs"), + include_str!("ui/windows_caption.rs"), // The onboarding window's welcome cards and permissions surface; the // per-permission row glyphs are named on `OSPermission::icon`. include_str!("onboarding_window.rs"), diff --git a/apps/desktop/src-tauri/src/clip_thumbnails.rs b/apps/desktop/src-tauri/src/clip_thumbnails.rs index 65860a5ad13..e4e0ac657d1 100644 --- a/apps/desktop/src-tauri/src/clip_thumbnails.rs +++ b/apps/desktop/src-tauri/src/clip_thumbnails.rs @@ -1,14 +1,19 @@ -use std::{path::Path, sync::LazyLock}; +use std::sync::LazyLock; use cap_project::{RecordingMetaInner, StudioRecordingMeta}; use tokio::sync::Semaphore; use crate::editor_window::WindowEditorInstance; -const THUMB_MAX_WIDTH: u32 = 240; -const THUMB_JPEG_QUALITY: u8 = 70; -const SEEK_DECODE_PACKET_LIMIT: usize = 4096; -const SEEK_DECODE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +#[path = "../../../../crates/editor/src/clip_thumbnails_shared.rs"] +mod clip_thumbnails_shared; + +use clip_thumbnails_shared::decode_clip_thumbnail; +#[cfg(test)] +use clip_thumbnails_shared::{ + SEEK_DECODE_PACKET_LIMIT, decode_clip_thumbnail_with_budget, persist_clip_thumbnail, + thumbnail_frame_reaches_target, +}; static THUMBNAIL_SEMAPHORE: LazyLock = LazyLock::new(|| Semaphore::new(4)); @@ -71,222 +76,6 @@ pub async fn get_clip_thumbnail( Ok(cache_path.to_string_lossy().into_owned()) } -fn decode_clip_thumbnail(input: &Path, time: f64, output: &Path) -> Result<(), String> { - decode_clip_thumbnail_with_budget( - input, - time, - output, - SEEK_DECODE_PACKET_LIMIT, - SEEK_DECODE_TIMEOUT, - ) -} - -fn decode_clip_thumbnail_with_budget( - input: &Path, - time: f64, - output: &Path, - packet_limit: usize, - timeout: std::time::Duration, -) -> Result<(), String> { - use ffmpeg::rescale::{Rescale, TIME_BASE}; - - let mut ictx = - ffmpeg::format::input(input).map_err(|e| format!("Failed to open video: {e}"))?; - - let stream = ictx - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("No video stream found")?; - let stream_index = stream.index(); - let stream_time_base = stream.time_base(); - let stream_start = match stream.start_time() { - ffmpeg::ffi::AV_NOPTS_VALUE => 0, - timestamp => timestamp, - }; - let target_timestamp = ((time * 1_000_000.0) as i64) - .rescale((1, 1_000_000), stream_time_base) - .saturating_add(stream_start); - - let mut decoder = ffmpeg::codec::context::Context::from_parameters(stream.parameters()) - .map_err(|e| e.to_string())? - .decoder() - .video() - .map_err(|e| e.to_string())?; - - let src_width = decoder.width(); - let src_height = decoder.height(); - if src_width == 0 || src_height == 0 { - return Err("Invalid video dimensions".to_string()); - } - - let scale = (THUMB_MAX_WIDTH as f32 / src_width as f32).min(1.0); - let target_width = ((src_width as f32 * scale).round() as u32).max(1); - let target_height = ((src_height as f32 * scale).round() as u32).max(1); - - let mut scaler = ffmpeg::software::scaling::context::Context::get( - decoder.format(), - src_width, - src_height, - ffmpeg::format::Pixel::RGB24, - target_width, - target_height, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ) - .map_err(|e| e.to_string())?; - - if time > 0.0 { - let position_us = (time * 1_000_000.0) as i64; - let seek_target = target_timestamp.rescale(stream_time_base, TIME_BASE); - decoder.flush(); - ictx.seek(seek_target, ..seek_target) - .map_err(|e| format!("Failed to seek to {position_us}us: {e}"))?; - } - - let mut frame = ffmpeg::frame::Video::empty(); - let mut decoded = ffmpeg::frame::Video::empty(); - let mut got_frame = false; - let mut reached_target = false; - let mut decoder_finished = false; - let mut packets_tried = 0usize; - let decode_started = std::time::Instant::now(); - - 'outer: for (packet_stream, packet) in ictx.packets() { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - if packet_stream.index() != stream_index { - continue; - } - - packets_tried += 1; - - if decoder.send_packet(&packet).is_err() { - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - continue; - } - - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - reached_target = true; - break 'outer; - } - } - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(ffmpeg::Error::Eof) => { - decoder_finished = true; - break 'outer; - } - Err(e) => { - if packets_tried >= packet_limit { - return Err(format!("Failed to decode frame: {e}")); - } - break; - } - } - } - - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - } - - if !reached_target && !decoder_finished { - decoder - .send_eof() - .map_err(|e| format!("Failed to flush decoder: {e}"))?; - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - break; - } - } - Err(ffmpeg::Error::Eof) => break, - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(e) => return Err(format!("Failed to flush decoder: {e}")), - } - } - } - - if !got_frame { - return Err("No decodable frames found".to_string()); - } - - let mut rgb_frame = ffmpeg::frame::Video::empty(); - scaler - .run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - let width = rgb_frame.width() as usize; - let height = rgb_frame.height() as usize; - let src_stride = rgb_frame.stride(0); - let dst_stride = width * 3; - if src_stride < dst_stride { - return Err(format!( - "Unexpected RGB stride: src_stride={src_stride}, expected >= {dst_stride}" - )); - } - let mut img_buffer = vec![0u8; height * dst_stride]; - for y in 0..height { - let src_slice = &rgb_frame.data(0)[y * src_stride..y * src_stride + dst_stride]; - img_buffer[y * dst_stride..(y + 1) * dst_stride].copy_from_slice(src_slice); - } - - use image::ImageEncoder; - - let mut jpeg_bytes = Vec::new(); - image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, THUMB_JPEG_QUALITY) - .write_image( - &img_buffer, - width as u32, - height as u32, - image::ExtendedColorType::Rgb8, - ) - .map_err(|e| format!("Failed to encode thumbnail: {e}"))?; - - persist_clip_thumbnail(output, &jpeg_bytes) -} - -fn thumbnail_frame_reaches_target(frame: &ffmpeg::frame::Video, target_timestamp: i64) -> bool { - frame - .timestamp() - .or_else(|| frame.pts()) - .is_none_or(|timestamp| timestamp >= target_timestamp) -} - -fn persist_clip_thumbnail(output: &Path, jpeg_bytes: &[u8]) -> Result<(), String> { - use std::io::Write; - - let parent = output.parent().unwrap_or_else(|| Path::new(".")); - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create thumbnail directory: {e}"))?; - - let mut staged = tempfile::NamedTempFile::new_in(parent) - .map_err(|e| format!("Failed to stage thumbnail: {e}"))?; - staged - .write_all(jpeg_bytes) - .map_err(|e| format!("Failed to write thumbnail: {e}"))?; - staged - .persist(output) - .map_err(|e| format!("Failed to persist thumbnail: {e}"))?; - - Ok(()) -} - #[cfg(test)] mod persistence_tests { use super::*; diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index cebe7d9c065..1c85c5bce2f 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -6,10 +6,9 @@ use cap_enc_ffmpeg::{ }; use cap_media_info::{AudioInfo, FFRational, Pixel, VideoInfo, ensure_even}; use cap_project::{ - AudioMeta, ClipConfiguration, CursorEvents, CursorMeta, Cursors, InstantRecordingMeta, - MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, RecordingMeta, - RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, - TimelineConfiguration, TimelineSegment, VideoMeta, XY, + AudioMeta, ClipConfiguration, Cursors, InstantRecordingMeta, MultipleSegment, MultipleSegments, + Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SingleSegment, + StudioRecordingMeta, StudioRecordingStatus, TimelineConfiguration, TimelineSegment, VideoMeta, }; use ffmpeg::{ ChannelLayout, @@ -17,11 +16,10 @@ use ffmpeg::{ format::{self as avformat}, }; use image::ImageEncoder; -use relative_path::{Component as RelativeComponent, RelativePathBuf}; +use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ - collections::HashMap, path::{Path, PathBuf}, str::FromStr, }; @@ -38,9 +36,6 @@ use crate::{ const VIDEO_IMPORT_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "wmv", "m4v", "flv"]; const IMAGE_IMPORT_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff"]; -const AUDIO_IMPORT_EXTENSIONS: &[&str] = &["ogg", "m4a", "mp3", "wav", "aac", "flac"]; -const KEYBOARD_IMPORT_EXTENSIONS: &[&str] = &["bin", "json"]; -const CURSOR_EVENTS_IMPORT_EXTENSIONS: &[&str] = &["json"]; const MAX_IMAGE_DIMENSION: u32 = 16_384; #[derive(Serialize, Deserialize, Type, Clone, Debug)] @@ -161,89 +156,6 @@ fn is_cap_project_path(path: &Path) -> bool { path.is_dir() && path.join("recording-meta.json").is_file() } -fn normalized_metadata_relative_path( - path: &RelativePathBuf, - asset_kind: &str, -) -> Result { - let normalized = path.as_str().replace('\\', "/"); - let path = RelativePathBuf::from(normalized); - let raw = path.as_str(); - if raw.is_empty() - || raw.starts_with('/') - || raw.contains(':') - || path - .components() - .any(|component| matches!(component, RelativeComponent::ParentDir)) - { - return Err(format!( - "Invalid {asset_kind} path in recording metadata: {raw}" - )); - } - - Ok(path) -} - -fn source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result, String> { - let source_relative_path = normalized_metadata_relative_path(source_relative_path, asset_kind)?; - - if !has_supported_extension(Path::new(source_relative_path.as_str()), allowed_extensions) { - return Err(format!( - "Unsupported {asset_kind} file type: {}", - source_relative_path.as_str() - )); - } - - let source_path = source_relative_path.to_path(source_project_path); - if !source_path.is_file() { - return Ok(None); - } - - let source_root = source_project_path - .canonicalize() - .map_err(|e| format!("Failed to resolve source project path: {e}"))?; - let canonical_source_path = source_path - .canonicalize() - .map_err(|e| format!("Failed to resolve {asset_kind} path: {e}"))?; - - if !canonical_source_path.starts_with(&source_root) { - return Err(format!( - "{asset_kind} path escapes source project: {}", - source_relative_path.as_str() - )); - } - - Ok(Some(canonical_source_path)) -} - -fn required_source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result { - source_asset_path( - source_project_path, - source_relative_path, - asset_kind, - allowed_extensions, - )? - .ok_or_else(|| { - format!( - "Missing {asset_kind} file: {}", - source_relative_path.to_path(source_project_path).display() - ) - }) -} - -fn legacy_cursor_relative_path(path: &str) -> Result { - normalized_metadata_relative_path(&RelativePathBuf::from(path), "cursor image") -} - fn editor_project_path_from_window(window: &Window) -> Result { let CapWindowId::Editor { id } = CapWindowId::from_str(window.label()).map_err(|e| e.to_string())? @@ -331,42 +243,6 @@ fn full_timeline_for_segments( .collect() } -fn get_source_video_duration_secs( - source_meta: &RecordingMeta, - video: &VideoMeta, -) -> Result { - let source_path = required_source_asset_path( - &source_meta.project_path, - &video.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )?; - get_video_duration_secs(&source_path) -} - -fn full_timeline_for_source_segments( - source_meta: &RecordingMeta, - segments: &[MultipleSegment], -) -> Result, String> { - segments - .iter() - .enumerate() - .map(|(index, segment)| { - let duration = get_source_video_duration_secs(source_meta, &segment.display)?; - Ok(TimelineSegment { - recording_clip: index as u32, - timescale: 1.0, - start: 0.0, - end: duration, - name: None, - speed_audio_mode: None, - hide_cursor: None, - volume: None, - }) - }) - .collect() -} - fn ensure_project_timeline<'a>( config: &'a mut ProjectConfiguration, project_path: &Path, @@ -440,600 +316,6 @@ fn unique_segment_dir( } } -fn relative_file_extension(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .extension() - .and_then(|ext| ext.to_str()) - .filter(|ext| !ext.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn relative_file_name(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .file_name() - .and_then(|name| name.to_str()) - .filter(|name| !name.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn unique_file_name(dir: &Path, preferred: &str) -> String { - let sanitized = sanitize_filename(preferred); - let sanitized = if sanitized.is_empty() { - "file".to_string() - } else { - sanitized - }; - - let path = Path::new(&sanitized); - let stem = path - .file_stem() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .unwrap_or("file") - .to_string(); - let extension = path - .extension() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .map(|value| value.to_string()); - - let mut counter = 0; - loop { - let candidate = if counter == 0 { - sanitized.clone() - } else if let Some(extension) = &extension { - format!("{stem}-{counter}.{extension}") - } else { - format!("{stem}-{counter}") - }; - - if !dir.join(&candidate).exists() { - return candidate; - } - - counter += 1; - } -} - -fn copy_file_to_relative_path( - source_path: &Path, - target_project_path: &Path, - target_relative_path: &RelativePathBuf, -) -> Result<(), String> { - let target_path = target_relative_path.to_path(target_project_path); - - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create import directory: {e}"))?; - } - - std::fs::copy(source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy {}: {e}", source_path.display())) -} - -fn copy_video_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &VideoMeta, - target_relative_dir: &str, - name: &str, - required: bool, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )? - else { - if required { - return Err(format!( - "Missing video file: {}", - source.path.to_path(source_project_path).display() - )); - } - return Ok(None); - }; - - let can_decode = probe_video_can_decode(&source_path) - .map_err(|e| format!("Cannot decode video {}: {e}", source_path.display()))?; - if !can_decode { - if required { - return Err(format!("Unsupported video file: {}", source_path.display())); - } - return Ok(None); - } - - let extension = relative_file_extension(&source.path, "mp4"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_audio_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &AudioMeta, - target_relative_dir: &str, - name: &str, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "audio", - AUDIO_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let extension = relative_file_extension(&source.path, "ogg"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_keyboard_path( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, -) -> Result, String> { - if let Some(source_relative_path) = &source_segment.keyboard { - let file_name = - relative_file_name(source_relative_path, cap_project::KEYBOARD_EVENTS_FILE_NAME); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(&file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - }; - - let Some(display_dir) = source_segment.display.path.parent() else { - return Ok(None); - }; - - for file_name in [ - cap_project::KEYBOARD_EVENTS_FILE_NAME, - cap_project::LEGACY_KEYBOARD_EVENTS_FILE_NAME, - ] { - let source_relative_path = display_dir.join(file_name); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - } - - Ok(None) -} - -fn normalize_cursors_to_correct(cursors: &mut Cursors) -> &mut HashMap { - if let Cursors::Old(old) = cursors { - let converted = old - .iter() - .map(|(id, path)| { - ( - id.clone(), - CursorMeta { - image_path: RelativePathBuf::from(path.as_str()), - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ) - }) - .collect(); - *cursors = Cursors::Correct(converted); - } - - match cursors { - Cursors::Correct(map) => map, - Cursors::Old(_) => unreachable!(), - } -} - -fn unique_cursor_id( - cursors: &HashMap, - import_token: &str, - source_id: &str, -) -> String { - let source_id = if source_id.is_empty() { - "cursor" - } else { - source_id - }; - let base = format!("{import_token}-{source_id}"); - if !cursors.contains_key(&base) { - return base; - } - - let mut counter = 1; - loop { - let candidate = format!("{base}-{counter}"); - if !cursors.contains_key(&candidate) { - return candidate; - } - counter += 1; - } -} - -fn copy_source_cursor_images( - source_meta: &RecordingMeta, - source_cursors: &Cursors, - target_project_path: &Path, - target_cursors: &mut Cursors, - import_token: &str, -) -> Result, String> { - let target_cursor_dir = target_project_path.join("content").join("cursors"); - std::fs::create_dir_all(&target_cursor_dir) - .map_err(|e| format!("Failed to create cursor directory: {e}"))?; - - let target_cursors = normalize_cursors_to_correct(target_cursors); - let mut id_map = HashMap::new(); - - match source_cursors { - Cursors::Correct(source_map) => { - for (source_id, cursor) in source_map { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &cursor.image_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&cursor.image_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: cursor.hotspot, - shape: cursor.shape, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - Cursors::Old(source_map) => { - for (source_id, source_path) in source_map { - let source_relative_path = legacy_cursor_relative_path(source_path)?; - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&source_relative_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - } - - Ok(id_map) -} - -fn copy_cursor_events_path( - source_meta: &RecordingMeta, - source_relative_path: &RelativePathBuf, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result, String> { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "cursor events", - CURSOR_EVENTS_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!("{target_relative_dir}/cursor.json")); - let target_path = target_relative_path.to_path(target_project_path); - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create cursor event directory: {e}"))?; - } - - if cursor_id_map.is_empty() { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - return Ok(Some(target_relative_path)); - } - - match CursorEvents::load_from_file(&source_path) { - Ok(mut events) => { - for event in &mut events.moves { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - for event in &mut events.clicks { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - - let file = std::fs::File::create(&target_path) - .map_err(|e| format!("Failed to create cursor event file: {e}"))?; - serde_json::to_writer_pretty(file, &events) - .map_err(|e| format!("Failed to write cursor event file: {e}"))?; - } - Err(_) => { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - } - } - - Ok(Some(target_relative_path)) -} - -fn single_segment_to_multiple(segment: &SingleSegment) -> MultipleSegment { - MultipleSegment { - display: segment.display.clone(), - camera: segment.camera.clone(), - mic: segment.audio.clone(), - system_audio: None, - cursor: segment.cursor.clone(), - keyboard: None, - display_notch: None, - } -} - -fn studio_segments_for_import(studio_meta: &StudioRecordingMeta) -> Vec { - match studio_meta { - StudioRecordingMeta::SingleSegment { segment } => { - vec![single_segment_to_multiple(segment)] - } - StudioRecordingMeta::MultipleSegments { inner } => inner.segments.clone(), - } -} - -fn source_timeline_segments_for_import( - source_meta: &RecordingMeta, - source_segments: &[MultipleSegment], -) -> Result, String> { - let source_config = ProjectConfiguration::load(&source_meta.project_path).unwrap_or_default(); - let Some(timeline) = source_config.timeline else { - return full_timeline_for_source_segments(source_meta, source_segments); - }; - - if timeline.segments.is_empty() { - return full_timeline_for_source_segments(source_meta, source_segments); - } - - let mut duration_cache = HashMap::new(); - let mut imported_segments = Vec::new(); - - for segment in timeline.segments { - let source_index = segment.recording_clip; - let Some(source_segment) = source_segments.get(source_index as usize) else { - continue; - }; - - let max_duration = if let Some(duration) = duration_cache.get(&source_index) { - *duration - } else { - let duration = get_source_video_duration_secs(source_meta, &source_segment.display)?; - duration_cache.insert(source_index, duration); - duration - }; - - if max_duration <= 0.0 { - continue; - } - - let raw_start = if segment.start.is_finite() { - segment.start - } else { - 0.0 - }; - let raw_end = if segment.end.is_finite() { - segment.end - } else { - max_duration - }; - let start = raw_start.clamp(0.0, max_duration); - let end = raw_end.clamp(start, max_duration); - if end <= start { - continue; - } - - imported_segments.push(TimelineSegment { - recording_clip: source_index, - timescale: if segment.timescale.is_finite() && segment.timescale > 0.0 { - segment.timescale - } else { - 1.0 - }, - start, - end, - name: None, - speed_audio_mode: segment.speed_audio_mode, - hide_cursor: segment.hide_cursor, - volume: segment.volume, - }); - } - - if imported_segments.is_empty() { - full_timeline_for_source_segments(source_meta, source_segments) - } else { - Ok(imported_segments) - } -} - -fn copy_source_segment( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result { - let display = copy_video_meta( - &source_meta.project_path, - target_project_path, - &source_segment.display, - target_relative_dir, - "display", - true, - )? - .ok_or_else(|| "Missing display video".to_string())?; - - let camera = source_segment - .camera - .as_ref() - .map(|camera| { - copy_video_meta( - &source_meta.project_path, - target_project_path, - camera, - target_relative_dir, - "camera", - false, - ) - }) - .transpose()? - .flatten(); - - let mic = source_segment - .mic - .as_ref() - .map(|mic| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - mic, - target_relative_dir, - "mic", - ) - }) - .transpose()? - .flatten(); - - let system_audio = source_segment - .system_audio - .as_ref() - .map(|system_audio| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - system_audio, - target_relative_dir, - "system-audio", - ) - }) - .transpose()? - .flatten(); - - let cursor = source_segment - .cursor - .as_ref() - .map(|cursor| { - copy_cursor_events_path( - source_meta, - cursor, - target_project_path, - target_relative_dir, - cursor_id_map, - ) - }) - .transpose()? - .flatten(); - - let keyboard = copy_keyboard_path( - source_meta, - source_segment, - target_project_path, - target_relative_dir, - )?; - - Ok(MultipleSegment { - display, - camera, - mic, - system_audio, - cursor, - keyboard, - display_notch: source_segment.display_notch, - }) -} - fn get_video_stream_info( input: &avformat::context::Input, ) -> Result<(usize, VideoInfo), ImportError> { @@ -1771,121 +1053,21 @@ async fn append_cap_project_to_editor_project( ) -> Result { let source_meta = RecordingMeta::load_for_project(&source_project_path) .map_err(|e| format!("Failed to load source project metadata: {e}"))?; - - let RecordingMetaInner::Studio(source_studio_meta) = &source_meta.inner else { - return match &source_meta.inner { - RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { - append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()) - .await - } - RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { - Err("Source Cap project is still recording".to_string()) - } - RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { - Err(format!("Source Cap project failed: {error}")) - } - RecordingMetaInner::Studio(_) => unreachable!(), - }; - }; - - let source_segments = studio_segments_for_import(source_studio_meta); - if source_segments.is_empty() { - return Err("Source Cap project has no recording segments".to_string()); - } - - let source_timeline = source_timeline_segments_for_import(&source_meta, &source_segments)?; - let source_cursors = match source_studio_meta.as_ref() { - StudioRecordingMeta::MultipleSegments { inner } => Some(&inner.cursors), - StudioRecordingMeta::SingleSegment { .. } => None, - }; - - let mut target_meta = RecordingMeta::load_for_project(&target_project_path) - .map_err(|e| format!("Failed to load target project metadata: {e}"))?; - let mut config = ProjectConfiguration::load(&target_project_path).unwrap_or_default(); - let existing_segments = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - inner.segments.clone() - }; - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - - let (base_index, copied_segments, source_to_target_index) = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - let base_index = inner.segments.len() as u32; - let import_token = format!("import-{}", uuid::Uuid::new_v4().simple()); - let cursor_id_map = if let Some(source_cursors) = source_cursors { - copy_source_cursor_images( - &source_meta, - source_cursors, - &target_project_path, - &mut inner.cursors, - &import_token, - )? - } else { - HashMap::new() - }; - - let mut copied_segments = Vec::new(); - let mut source_to_target_index = HashMap::new(); - - for (source_index, source_segment) in source_segments.iter().enumerate() { - let target_index = base_index + copied_segments.len() as u32; - let (_, target_relative_dir) = - unique_segment_dir(&target_project_path, target_index) - .map_err(|e| format!("Failed to create imported segment directory: {e}"))?; - let copied_segment = copy_source_segment( - &source_meta, - source_segment, - &target_project_path, - &target_relative_dir, - &cursor_id_map, - )?; - - inner.segments.push(copied_segment.clone()); - copied_segments.push(copied_segment); - source_to_target_index.insert(source_index as u32, target_index); + match &source_meta.inner { + RecordingMetaInner::Studio(_) => cap_editor::append_studio_cap_project_to_editor_project( + target_project_path, + source_project_path, + ), + RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { + append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()).await } - - (base_index, copied_segments, source_to_target_index) - }; - - if copied_segments.is_empty() { - return Err("Source Cap project has no importable recording segments".to_string()); - } - - { - let timeline = - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - for source_segment in source_timeline { - let Some(target_index) = source_to_target_index.get(&source_segment.recording_clip) - else { - continue; - }; - - timeline.segments.push(TimelineSegment { - recording_clip: *target_index, - timescale: source_segment.timescale, - start: source_segment.start, - end: source_segment.end, - name: None, - speed_audio_mode: source_segment.speed_audio_mode, - hide_cursor: source_segment.hide_cursor, - volume: source_segment.volume, - }); + RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { + Err("Source Cap project is still recording".to_string()) + } + RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { + Err(format!("Source Cap project failed: {error}")) } } - - add_clip_configs(&mut config, base_index, &copied_segments); - - target_meta - .save_for_project() - .map_err(|e| format!("Failed to save project metadata: {e:?}"))?; - config - .write(&target_project_path) - .map_err(|e| format!("Failed to save project config: {e}"))?; - - Ok(copied_segments.len()) } #[tauri::command] @@ -2142,125 +1324,4 @@ mod tests { assert_eq!((referenced.width(), referenced.height()), (12, 8)); assert_eq!(referenced.pts(), Some(93)); } - - #[test] - fn source_asset_path_allows_file_inside_source_project() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.mp4"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_allows_backslash_separators() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = - RelativePathBuf::from("content\\segments\\segment-0\\display.mp4"); - let source_path = RelativePathBuf::from("content/segments/segment-0/display.mp4") - .to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_rejects_parent_traversal() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("../secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn source_asset_path_rejects_absolute_path() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("/tmp/secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn legacy_cursor_relative_path_rejects_windows_absolute_path() { - let error = legacy_cursor_relative_path("C:\\Users\\me\\cursor.png").unwrap_err(); - - assert!(error.contains("Invalid cursor image path")); - } - - #[test] - fn source_asset_path_rejects_unsupported_extension() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.txt"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Unsupported video file type")); - } - - #[cfg(unix)] - #[test] - fn source_asset_path_rejects_symlink_escape() { - let source_project = tempfile::tempdir().unwrap(); - let external_dir = tempfile::tempdir().unwrap(); - let external_file = external_dir.path().join("cursor.png"); - std::fs::write(&external_file, b"cursor").unwrap(); - - let source_relative_path = RelativePathBuf::from("content/cursors/cursor.png"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::os::unix::fs::symlink(&external_file, &source_path).unwrap(); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("cursor image path escapes source project")); - } } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 87cae9cedb8..39f5aeb047b 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6819,15 +6819,7 @@ fn apply_screen_recording_presentation_defaults( } pub fn default_project_config() -> ProjectConfiguration { - let mut config = ProjectConfiguration::default(); - - apply_screen_recording_presentation_defaults(&mut config, None, false, None); - - if config.background.rounding <= f64::EPSILON { - config.background.rounding = DEFAULT_SCREEN_RECORDING_BACKGROUND_ROUNDING_PERCENT; - } - - config + cap_editor::default_screen_recording_project_config() } #[tauri::command] diff --git a/apps/desktop/src/routes/editor/CaptionsTab.tsx b/apps/desktop/src/routes/editor/CaptionsTab.tsx index ea3d58c0e02..670b42a918b 100644 --- a/apps/desktop/src/routes/editor/CaptionsTab.tsx +++ b/apps/desktop/src/routes/editor/CaptionsTab.tsx @@ -89,6 +89,10 @@ interface LanguageOption { } const MODEL_DOWNLOAD_STATUS_POLL_MS = 1000; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; +const webCaptionsEnabled = () => + (window as Window & { capWebEditorCaptionsEnabled?: boolean }) + .capWebEditorCaptionsEnabled === true; const MODEL_OPTIONS: ModelOption[] = [ { @@ -154,6 +158,10 @@ const LANGUAGE_OPTIONS: LanguageOption[] = [ { code: "ta", label: "Tamil" }, ]; +const WEB_LANGUAGE_OPTIONS = LANGUAGE_OPTIONS.filter( + (option) => option.code !== "pa", +); + const STYLE_PRESET_KEYS = new Set([ "font", "fontWeight", @@ -250,6 +258,11 @@ export function CaptionsTab(props: { }) { const { project, setProject, editorInstance, editorState, setEditorState } = useEditorContext(); + const [webCaptionsProEnabled, setWebCaptionsProEnabled] = createSignal( + isWebEditor && webCaptionsEnabled(), + ); + const syncWebCaptionPlan = () => + setWebCaptionsProEnabled(webCaptionsEnabled()); const selectedCaptionIndex = () => editorState.timeline.selection?.type === "caption" && @@ -561,6 +574,30 @@ export function CaptionsTab(props: { }; onMount(async () => { + if (isWebEditor) { + window.addEventListener( + "cap-web-editor-captions-plan", + syncWebCaptionPlan, + ); + syncWebCaptionPlan(); + const savedLanguage = localStorage.getItem( + "selectedTranscriptionLanguage", + ); + if ( + savedLanguage && + WEB_LANGUAGE_OPTIONS.some((option) => option.code === savedLanguage) + ) { + setSelectedLanguage(savedLanguage); + } + if (editorInstance?.recordings) { + setHasAudio( + editorInstance.recordings.segments.some( + (segment) => segment.mic !== null || segment.system_audio !== null, + ), + ); + } + return; + } try { unlistenDownloadProgress = await events.downloadProgress.listen( (event) => { @@ -621,6 +658,12 @@ export function CaptionsTab(props: { }); onCleanup(() => { + if (isWebEditor) { + window.removeEventListener( + "cap-web-editor-captions-plan", + syncWebCaptionPlan, + ); + } if (unlistenDownloadProgress) unlistenDownloadProgress(); stopDownloadStatusPolling(); }); @@ -777,264 +820,360 @@ export function CaptionsTab(props: { -

- - - options={availableModelOptions().map((model) => model.name)} - value={selectedModel()} - onChange={(value: string | null) => { - if (value) setSelectedModel(value); - }} - itemComponent={(props) => { - const model = availableModelOptions().find( - (option) => option.name === props.item.rawValue, - ); - - return ( - - as={KSelect.Item} - item={props.item} - > -
-
-
- - {model?.label ?? props.item.rawValue} - + +
+

+ Cap Pro captions use the same AssemblyAI transcription as your + shareable link. +

+ + + Upgrade to Cap Pro + + + + + + options={WEB_LANGUAGE_OPTIONS.map((option) => option.code)} + value={selectedLanguage()} + onChange={(value: string | null) => { + if (value) setSelectedLanguage(value); + }} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + { + WEB_LANGUAGE_OPTIONS.find( + (option) => option.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = WEB_LANGUAGE_OPTIONS.find( + (option) => option.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + +
+ + +
+ + + options={availableModelOptions().map((model) => model.name)} + value={selectedModel()} + onChange={(value: string | null) => { + if (value) setSelectedModel(value); + }} + itemComponent={(props) => { + const model = availableModelOptions().find( + (option) => option.name === props.item.rawValue, + ); + + return ( + + as={KSelect.Item} + item={props.item} + > +
+
+
+ + {model?.label ?? props.item.rawValue} + + + + + + +
- - - +
+ {model?.description} +
-
- {model?.description} -
+ + {model?.size} +
- - - {model?.size} - + + ); + }} + > + +
+
+ + {selectedModelOption()?.label || "Select a model"} + + + + +
- - ); - }} - > - -
-
- - {selectedModelOption()?.label || "Select a model"} - - - - +
+ {selectedModelOption()?.description} +
-
- {selectedModelOption()?.description} -
+ + {selectedModelOption()?.size} + -
- - - {selectedModelOption()?.size} - - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - as={KSelect.Listbox} /> - - - - + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + as={KSelect.Listbox} /> + + + + + + +

+ Parakeet caption models are unavailable on Intel Macs. Whisper + models remain available. +

+ -

- Parakeet caption models are unavailable on Intel Macs. Whisper - models remain available. + One time download to your system. All captions are stored locally.

- - -

- One time download to your system. All captions are stored locally. -

- - - - options={LANGUAGE_OPTIONS.map((l) => l.code)} - value={selectedLanguage()} - onChange={(value: string | null) => { - if (value) setSelectedLanguage(value); - }} - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - { - LANGUAGE_OPTIONS.find((l) => l.code === props.item.rawValue) - ?.label - } - - - )} - > - - class="truncate"> - {(state) => { - const language = LANGUAGE_OPTIONS.find( - (l) => l.code === state.selectedOption(), - ); - return {language?.label || "Select a language"}; - }} - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - - class="overflow-y-auto max-h-48" - as={KSelect.Listbox} - /> - - - - - - - - -
-
+ { + LANGUAGE_OPTIONS.find( + (l) => l.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = LANGUAGE_OPTIONS.find( + (l) => l.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + + + +
+ class="overflow-hidden w-full h-1.5 rounded-full bg-ed-ctl" + role="progressbar" + aria-valuemin="0" + aria-valuemax="100" + aria-valuenow={downloadPercent()} + > +
+
+

+ {downloadMessage() || + "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} +

-

- {downloadMessage() || - "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} -

-
+ +
+ } + > +
+ + -
- } - > -
- - - -
- - - - {selectedModelOption()?.label ?? "Caption"} model downloaded +
+ + + + {selectedModelOption()?.label ?? "Caption"} model downloaded + - - } - onClick={deleteModel} - disabled={ - isGenerating() || - isDownloading() || - deletingModel() === selectedModel() - } - > - {deletingModel() === selectedModel() ? "Deleting..." : "Delete"} - + } + onClick={deleteModel} + disabled={ + isGenerating() || + isDownloading() || + deletingModel() === selectedModel() + } + > + {deletingModel() === selectedModel() + ? "Deleting..." + : "Delete"} + +
-
- -
+ +
+ -
+
@@ -1466,6 +1605,7 @@ export function CaptionsTab(props: { so the chunk load stays inside @@ -302,6 +306,13 @@ type WallpaperOption = { const isCurrentDesktopBackgroundPath = (path: string | null | undefined) => { if (!path) return false; + if ( + isWebEditor && + /^content\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(png|jpg|webp|gif|bmp|tiff)$/.test( + path, + ) + ) + return true; const filename = path.split(/[\\/]/).pop(); if (!filename) return false; return ( @@ -1998,6 +2009,8 @@ function BackgroundConfig(props: { const assetsDir = `${editorInstance.path}/assets`; try { + if (isWebEditor) + return await invoke("webEditorStoredDesktopBackground"); const importedPrefix = `${CURRENT_DESKTOP_BACKGROUND_BASENAME}-`; let newest: { path: string; timestamp: number } | null = null; for (const entry of await readDir(assetsDir)) { @@ -2180,9 +2193,20 @@ function BackgroundConfig(props: { if (importingDesktopBackground()) return; setImportingDesktopBackground(true); try { - const path = await commands.importCurrentDesktopBackground( - editorInstance.path, - ); + let source = editorInstance.path; + if (isWebEditor) { + const selected = await open({ + filters: [ + { + name: "Images", + extensions: [...BACKGROUND_IMAGE_EXTENSIONS, "jpeg"], + }, + ], + }); + if (typeof selected !== "string") return; + source = selected; + } + const path = await commands.importCurrentDesktopBackground(source); const addingFromBlankBackground = isNoneBackground(); batch(() => { setCurrentDesktopBackgroundPath(path); @@ -2435,7 +2459,9 @@ function BackgroundConfig(props: {
- Use the wallpaper from your desktop + {isWebEditor + ? "Choose your desktop wallpaper" + : "Use the wallpaper from your desktop"} {importingDesktopBackground() ? "Importing..." - : "Import desktop background"} + : isWebEditor + ? "Choose desktop wallpaper" + : "Import desktop background"}
} @@ -3273,11 +3301,15 @@ function CameraConfig(props: { scrollRef: HTMLDivElement }) { - options={cameraBackgroundOptions(ostype() === "macos")} + options={cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + )} optionValue="value" optionTextValue="name" value={ - cameraBackgroundOptions(ostype() === "macos").find( + cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + ).find( (option) => option.value === (project.camera.backgroundBlur?.mode ?? "off"), diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index aac65a6c360..b8a6365dac0 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -91,6 +91,7 @@ const DEFAULT_TIMELINE_CONTENT_HEIGHT = 124; // Vertical gutter between the player row and the timeline card, plus the // gutter below the timeline card; both live inside the measured layout box. const LAYOUT_GUTTERS = 16; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; const scheduleIdleWork = (callback: () => void) => { const win = window as Window & { @@ -790,6 +791,19 @@ function Inner(props: { throttledConfigUpdate(time); trailingConfigUpdate(time); }; + if (isWebEditor) { + const refreshCaptionPlan = () => { + void commands.checkUpgradedAndUpdate().catch(() => undefined); + }; + onMount(refreshCaptionPlan); + createEventListener(window, "focus", refreshCaptionPlan); + createEventListener(document, "visibilitychange", () => { + if (!document.hidden) refreshCaptionPlan(); + }); + createEventListener(window, "cap-web-editor-captions-plan", () => + doConfigUpdate(frameNumberToRender()), + ); + } createEffect( on( diff --git a/apps/desktop/src/routes/editor/EditorErrorScreen.tsx b/apps/desktop/src/routes/editor/EditorErrorScreen.tsx index cc2a21478c0..38e9ef8a616 100644 --- a/apps/desktop/src/routes/editor/EditorErrorScreen.tsx +++ b/apps/desktop/src/routes/editor/EditorErrorScreen.tsx @@ -2,6 +2,7 @@ import { Button } from "@cap/ui-solid"; import { createMutation } from "@tanstack/solid-query"; import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; +import { getCurrentWindow } from "@tauri-apps/api/window"; import { revealItemInDir } from "@tauri-apps/plugin-opener"; import { type as ostype } from "@tauri-apps/plugin-os"; import { createEffect, onCleanup, Show } from "solid-js"; @@ -15,6 +16,7 @@ import IconLoaderCircle from "~icons/lucide/loader-circle"; import IconRefreshCw from "~icons/lucide/refresh-cw"; const NEEDS_RECOVERY_PATTERN = /may need to be recovered/i; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; function isRecoveryNeededError(error: string): boolean { return NEEDS_RECOVERY_PATTERN.test(error); @@ -26,7 +28,7 @@ export function EditorErrorScreen(props: { }) { const storageShortage = () => isRecordingStorageError(props.error); const needsRecovery = () => - storageShortage() || isRecoveryNeededError(props.error); + !isWebEditor && (storageShortage() || isRecoveryNeededError(props.error)); const isMac = () => ostype() === "macos"; let disposed = false; @@ -259,52 +261,69 @@ export function EditorErrorScreen(props: {
-
-
-

- Manual Investigation -

-

- You can open the recording folder to inspect the raw files - directly. -

- -
-

- {props.projectPath} + +

+
+

+ Manual Investigation +

+

+ You can open the recording folder to inspect the raw files + directly.

- +

+ {props.projectPath} +

+ + Tip: Double-click inside the folder to browse the + contents. +

+ } + >

- Tip: Double-click inside the folder to browse the - contents. + Tip: Right-click and select "Show Enclosing Folder" to see + the .cap bundle contents.

- } - > -

- Tip: Right-click and select "Show Enclosing Folder" to see - the .cap bundle contents. -

- + +
-
- -
+ +
-
- -
+
+ +
+ + +
+ +
+
diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index 68e60f5f6ac..79beccdd738 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -54,6 +54,7 @@ import { RESOLUTION_OPTIONS } from "./Header"; import { Dialog } from "./ui"; class SilentError extends Error {} +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; const EXPORT_CTA_CLASS = "flex w-full h-10 items-center justify-center gap-2 rounded-[10px] text-[13px] font-medium transition-colors outline-hidden focus-visible:ring-2 focus-visible:ring-ed-accent/40 disabled:opacity-50 disabled:cursor-not-allowed"; @@ -269,6 +270,8 @@ export function ExportPage() { if (disablesLinkExport() && _settings.exportTo === "link") ret.exportTo = "file"; + if (isWebEditor && _settings.exportTo === "clipboard") + ret.exportTo = "file"; if (shouldUseGifMode()) { if (!["720p", "1080p"].includes(_settings.resolution.value)) { @@ -842,8 +845,10 @@ export function ExportPage() { await refetchMeta(); setReuploading(!!meta().sharing); - const existingAuth = await authStore.get(); - if (!existingAuth) createSignInMutation(); + const existingAuth = isWebEditor + ? { user_id: "web" } + : await authStore.get(); + if (!existingAuth && !isWebEditor) createSignInMutation(); trackEvent("create_shareable_link_clicked", { resolution: settings.resolution, fps: settings.fps, @@ -962,13 +967,21 @@ export function ExportPage() { label: option.value === "link" && meta().sharing ? "Reupload" : option.label, icon: option.icon, - disabled: option.value === "link" && disablesLinkExport(), + disabled: + (option.value === "link" && disablesLinkExport()) || + (isWebEditor && option.value === "clipboard"), disabledReason: - option.value === "link" && disablesLinkExport() - ? cursorOnly() - ? "Cursor-only exports can only be saved to a file or clipboard" - : "Transparent exports can only be saved to a file or clipboard" - : undefined, + isWebEditor && option.value === "clipboard" + ? "Use File to download. Browser clipboard cannot paste exports as files" + : option.value === "link" && disablesLinkExport() + ? cursorOnly() + ? isWebEditor + ? "Cursor-only exports can only be saved to a file" + : "Cursor-only exports can only be saved to a file or clipboard" + : isWebEditor + ? "Transparent exports can only be saved to a file" + : "Transparent exports can only be saved to a file or clipboard" + : undefined, })); const formatOptions = () => @@ -1446,7 +1459,7 @@ export function ExportPage() {
- {settings.exportTo === "link" && !auth.data ? ( + {settings.exportTo === "link" && !auth.data && !isWebEditor ? ( + )} +
+ {phase === "idle" || (phase === "error" && !captured) ? ( +
+ + + +
+ ) : null} + {previewStream && ( +
+ + Download screen clip + + {captured.camera && downloadUrls.camera && ( + + Download camera clip + + )} +
+ )} +
+ {phase === "idle" || (phase === "error" && !captured) ? ( + + ) : null} + {phase === "recording" && ( + + )} + {phase === "paused" && ( + + )} + {(phase === "recording" || phase === "paused") && ( + + )} + {phase === "error" && captured && capturedCanImport && ( + + )} +
+
+
+ ); +} diff --git a/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx new file mode 100644 index 00000000000..c0e27a50c47 --- /dev/null +++ b/apps/web/app/s/[videoId]/edit/studio/StudioEditorClient.tsx @@ -0,0 +1,687 @@ +"use client"; + +import dynamic from "next/dynamic"; +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useRef, useState } from "react"; +import type { WebEditorCapImportProgress } from "@/lib/editor-cap-import-client"; +import { + hasEditorCaptionContent, + stripEditorCaptionContent, +} from "@/lib/editor-caption-access"; +import type { EditorClipCapture } from "@/lib/editor-clip-recorder"; +import { + captureEditorLocalDraft, + clearEditorLocalDraft, + type EditorLocalDraft, + readEditorLocalDraft, +} from "@/lib/editor-local-draft"; +import { startWebEditorPreparation } from "@/lib/editor-preparation-client"; +import type { WebEditorVideoImportProgress } from "@/lib/editor-video-import-client"; +import { EditorClipRecorder } from "./EditorClipRecorder"; +import { EditorHostBridge } from "./editor-host"; + +const UpgradeModal = dynamic( + () => + import("@/components/UpgradeModal").then((module) => module.UpgradeModal), + { ssr: false }, +); + +type PreparationStatus = { + status: "preparing" | "ready" | "error" | "canceled" | "closed"; + sessionId?: string; +}; + +function isPreparationStatus(value: unknown): value is PreparationStatus { + return ( + typeof value === "object" && + value !== null && + "status" in value && + ["preparing", "ready", "error", "canceled", "closed"].includes( + String(value.status), + ) + ); +} + +function waitForPoll(signal: AbortSignal) { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new Error("Editor preparation was canceled")); + return; + } + const timer = window.setTimeout(() => { + signal.removeEventListener("abort", canceled); + resolve(); + }, 500); + const canceled = () => { + window.clearTimeout(timer); + reject(new Error("Editor preparation was canceled")); + }; + signal.addEventListener("abort", canceled, { once: true }); + }); +} + +export function StudioEditorClient(props: { + videoId: string; + userId: string; + captionsEnabled: boolean; + savedAt: string | null; + preparingTitle: string; + preparingDuration: number; + preparingTracks: Array<"display" | "camera">; +}) { + const { + videoId, + userId, + captionsEnabled, + savedAt, + preparingTitle, + preparingDuration, + preparingTracks, + } = props; + const router = useRouter(); + const [sessionId, setSessionId] = useState(null); + const [waitingForCapacity, setWaitingForCapacity] = useState(false); + const [error, setError] = useState(null); + const [errorFrameReady, setErrorFrameReady] = useState(false); + const [recoveryConflict, setRecoveryConflict] = + useState(null); + const [restoringBrowserDraft, setRestoringBrowserDraft] = useState(false); + const [videoImport, setVideoImport] = useState< + WebEditorVideoImportProgress | WebEditorCapImportProgress | null + >(null); + const [recordClipOpen, setRecordClipOpen] = useState(false); + const [upgradeOpen, setUpgradeOpen] = useState(false); + const preparationRef = useRef(null); + const sessionRef = useRef(null); + const bridgeRef = useRef(null); + const iframeRef = useRef(null); + const errorFrameRef = useRef(null); + const frameConnectRef = useRef<{ + document: Document; + sessionId: string; + } | null>(null); + const restoreInProgressRef = useRef(false); + const savedAtRef = useRef(savedAt); + const captureDraftRef = useRef<() => boolean>(() => true); + const closedRef = useRef(false); + const restartAfterImport = useCallback(async () => { + const activeSession = sessionRef.current; + if (activeSession) { + const response = await fetch( + `/api/editor/sessions/${encodeURIComponent(activeSession)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", cache: "no-store" }, + ); + if (!response.ok && response.status !== 404) { + throw new Error( + "Imported clip was saved, but the editor could not restart", + ); + } + sessionRef.current = null; + } + window.location.reload(); + }, [videoId]); + + useEffect(() => { + const controller = new AbortController(); + savedAtRef.current = savedAt; + closedRef.current = false; + setSessionId(null); + setWaitingForCapacity(false); + setError(null); + setRecoveryConflict(null); + setRestoringBrowserDraft(false); + restoreInProgressRef.current = false; + setVideoImport(null); + setRecordClipOpen(false); + setUpgradeOpen(false); + let lastCapturedConfig: string | null = null; + let lastCapturedSavedAt: string | null = null; + const captureDraft = () => { + let serialized: string | null; + try { + const editorWindow = iframeRef.current?.contentWindow as + | (Window & { + capSolidEditor?: { unsavedProject?: () => string | null }; + }) + | null; + serialized = editorWindow?.capSolidEditor?.unsavedProject?.() ?? null; + } catch { + return false; + } + if (!serialized) return true; + if ( + serialized === lastCapturedConfig && + savedAtRef.current === lastCapturedSavedAt + ) + return true; + const captured = captureEditorLocalDraft( + window.localStorage, + userId, + videoId, + savedAtRef.current, + serialized, + ); + if (captured) { + lastCapturedConfig = serialized; + lastCapturedSavedAt = savedAtRef.current; + } + return captured; + }; + captureDraftRef.current = captureDraft; + const beforeUnload = (event: BeforeUnloadEvent) => { + if (captureDraft()) return; + event.preventDefault(); + event.returnValue = ""; + }; + const close = () => { + if (closedRef.current) return; + captureDraft(); + closedRef.current = true; + controller.abort(); + bridgeRef.current?.dispose(); + bridgeRef.current = null; + const activeSession = sessionRef.current; + const activePreparation = preparationRef.current; + sessionRef.current = null; + preparationRef.current = null; + if (activeSession) { + void fetch( + `/api/editor/sessions/${encodeURIComponent(activeSession)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + } else if (activePreparation) { + void fetch( + `/api/editor/preparations/${encodeURIComponent(activePreparation)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + } + }; + const prepare = async () => { + try { + const created = await startWebEditorPreparation( + videoId, + controller.signal, + setWaitingForCapacity, + ); + preparationRef.current = created.id; + if (controller.signal.aborted) { + void fetch( + `/api/editor/preparations/${encodeURIComponent(created.id)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + preparationRef.current = null; + return; + } + const deadline = Date.now() + 5 * 60 * 1000; + while (!controller.signal.aborted && Date.now() < deadline) { + const statusResponse = await fetch( + `/api/editor/preparations/${encodeURIComponent(created.id)}?videoId=${encodeURIComponent(videoId)}`, + { signal: controller.signal }, + ); + if (!statusResponse.ok) { + throw new Error("Editor preparation status is unavailable"); + } + const status: unknown = await statusResponse.json(); + if (!isPreparationStatus(status)) { + throw new Error("Editor preparation status was invalid"); + } + if (status.status === "ready") { + if (!status.sessionId) { + throw new Error("Editor session was not returned"); + } + if (controller.signal.aborted) { + void fetch( + `/api/editor/sessions/${encodeURIComponent(status.sessionId)}?videoId=${encodeURIComponent(videoId)}`, + { method: "DELETE", keepalive: true }, + ); + return; + } + sessionRef.current = status.sessionId; + preparationRef.current = null; + const draft = readEditorLocalDraft( + window.localStorage, + userId, + videoId, + ); + if (draft) { + if (!captionsEnabled && hasEditorCaptionContent(draft.config)) { + setRecoveryConflict(draft); + setError( + "Browser edits include captions, which require Cap Pro. Restore your other edits without captions, or open the latest saved version.", + ); + return; + } + const recovered = await fetch( + `/api/editor/sessions/${encodeURIComponent(status.sessionId)}/config`, + { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId, + config: draft.config, + expectedSavedAt: draft.baseSavedAt, + }), + signal: controller.signal, + }, + ); + if (recovered.status === 409) { + setRecoveryConflict(draft); + setError( + "This recording changed after the browser saved pending edits. Choose which version to open.", + ); + return; + } + if (!recovered.ok) + throw new Error( + "Pending edits are saved in this browser, but the editor could not restore them. Try again.", + ); + const result: unknown = await recovered.json(); + if ( + typeof result === "object" && + result !== null && + "savedAt" in result && + typeof result.savedAt === "string" + ) { + savedAtRef.current = result.savedAt; + } + clearEditorLocalDraft(window.localStorage, userId, videoId); + } + if (controller.signal.aborted) return; + setSessionId(status.sessionId); + return; + } + if (status.status !== "preparing") { + throw new Error("Editor preparation failed"); + } + await waitForPoll(controller.signal); + } + throw new Error("Editor preparation timed out"); + } catch (cause) { + if (!controller.signal.aborted) { + setError( + cause instanceof Error ? cause.message : "Editor is unavailable", + ); + } + } + }; + const timer = window.setTimeout(() => void prepare(), 0); + window.addEventListener("beforeunload", beforeUnload); + window.addEventListener("popstate", captureDraft); + window.addEventListener("pagehide", close); + return () => { + window.clearTimeout(timer); + window.removeEventListener("beforeunload", beforeUnload); + window.removeEventListener("popstate", captureDraft); + window.removeEventListener("pagehide", close); + close(); + captureDraftRef.current = () => true; + }; + }, [captionsEnabled, savedAt, userId, videoId]); + + const restoreBrowserDraft = useCallback(async () => { + const activeSession = sessionRef.current; + const draft = recoveryConflict; + if (!activeSession || !draft || restoreInProgressRef.current) return; + const config = + !captionsEnabled && hasEditorCaptionContent(draft.config) + ? stripEditorCaptionContent(draft.config) + : draft.config; + restoreInProgressRef.current = true; + setRestoringBrowserDraft(true); + try { + const configUrl = `/api/editor/sessions/${encodeURIComponent(activeSession)}/config`; + const revisionResponse = await fetch( + `${configUrl}?videoId=${encodeURIComponent(videoId)}`, + { cache: "no-store" }, + ).catch(() => null); + const revision: unknown = await revisionResponse + ?.json() + .catch(() => null); + if ( + !revisionResponse?.ok || + typeof revision !== "object" || + revision === null || + !("savedAt" in revision) || + (revision.savedAt !== null && typeof revision.savedAt !== "string") + ) { + setError("The latest saved edits could not be checked. Try again."); + return; + } + if (closedRef.current || sessionRef.current !== activeSession) return; + const response = await fetch(configUrl, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId, + config, + expectedSavedAt: revision.savedAt, + }), + }).catch(() => null); + if (response?.status === 409) { + setError( + "This recording changed again while restoring browser edits. Choose which version to open.", + ); + return; + } + if (!response?.ok) { + setError("Browser edits could not be restored. Try again."); + return; + } + const saved: unknown = await response.json().catch(() => null); + if ( + typeof saved !== "object" || + saved === null || + !("savedAt" in saved) || + typeof saved.savedAt !== "string" + ) { + setError( + "Browser edits were saved, but the result could not be checked. Try again.", + ); + return; + } + if (closedRef.current || sessionRef.current !== activeSession) return; + savedAtRef.current = saved.savedAt; + clearEditorLocalDraft(window.localStorage, userId, videoId); + setRecoveryConflict(null); + setError(null); + setSessionId(activeSession); + } finally { + restoreInProgressRef.current = false; + setRestoringBrowserDraft(false); + } + }, [captionsEnabled, recoveryConflict, userId, videoId]); + + const sendErrorToFrame = useCallback( + (iframe: HTMLIFrameElement) => { + const document = iframe.contentDocument; + if ( + !error || + !document || + document.readyState !== "complete" || + new URL(document.URL).pathname !== "/editor-solid/index.html" + ) + return; + iframe.contentWindow?.postMessage( + { + kind: "cap-editor-error", + version: 1, + message: error, + hasBrowserDraftConflict: recoveryConflict !== null, + restoringBrowserDraft, + }, + window.location.origin, + ); + }, + [error, recoveryConflict, restoringBrowserDraft], + ); + + useEffect(() => { + if (!error) { + setErrorFrameReady(false); + return; + } + bridgeRef.current?.dispose(); + bridgeRef.current = null; + frameConnectRef.current = null; + }, [error]); + + useEffect(() => { + if (errorFrameRef.current) sendErrorToFrame(errorFrameRef.current); + }, [sendErrorToFrame]); + + useEffect(() => { + if (!error) return; + const onErrorFrameMessage = (event: MessageEvent) => { + if ( + event.origin !== window.location.origin || + event.source !== errorFrameRef.current?.contentWindow || + typeof event.data !== "object" || + event.data === null + ) + return; + const message = event.data as Record; + if (message.version !== 1) return; + if (message.kind === "cap-editor-error-ready") { + setErrorFrameReady(true); + return; + } + if (message.kind !== "cap-editor-error-action") return; + if (message.action === "retry") { + window.location.reload(); + } else if (message.action === "back-to-recording") { + router.push(`/s/${encodeURIComponent(videoId)}`); + } else if ( + message.action === "restore-browser" && + recoveryConflict && + !restoringBrowserDraft + ) { + void restoreBrowserDraft(); + } else if ( + message.action === "open-latest" && + recoveryConflict && + !restoringBrowserDraft + ) { + clearEditorLocalDraft(window.localStorage, userId, videoId); + window.location.reload(); + } + }; + window.addEventListener("message", onErrorFrameMessage); + return () => window.removeEventListener("message", onErrorFrameMessage); + }, [ + error, + recoveryConflict, + restoringBrowserDraft, + restoreBrowserDraft, + router, + userId, + videoId, + ]); + + const onFrameLoad = useCallback( + (iframe: HTMLIFrameElement) => { + const document = iframe.contentDocument; + if ( + !document || + document.readyState !== "complete" || + new URL(document.URL).pathname !== "/editor-solid/index.html" || + closedRef.current + ) + return; + iframe.contentWindow?.postMessage( + { + kind: "cap-editor-preparing", + version: 1, + title: preparingTitle, + durationSeconds: preparingDuration, + tracks: preparingTracks, + }, + window.location.origin, + ); + if (!sessionId) return; + const previous = frameConnectRef.current; + if (previous?.document === document) { + if (previous.sessionId === sessionId) return; + frameConnectRef.current = null; + iframe.src = "/editor-solid/index.html"; + return; + } + frameConnectRef.current = { document, sessionId }; + bridgeRef.current?.dispose(); + const bridge = new EditorHostBridge( + videoId, + sessionId, + userId, + (reason) => + router.push( + reason === "deleted" + ? "/dashboard/caps" + : `/s/${encodeURIComponent(videoId)}`, + ), + (cause) => { + captureDraftRef.current(); + setError(cause.message); + }, + (progress) => setVideoImport(progress), + () => setRecordClipOpen(true), + restartAfterImport, + captionsEnabled, + () => setUpgradeOpen(true), + (nextSavedAt) => { + savedAtRef.current = nextSavedAt; + }, + () => savedAtRef.current, + ); + bridgeRef.current = bridge; + void bridge.connect(iframe).catch((cause) => { + bridge.dispose(); + if (bridgeRef.current === bridge) { + captureDraftRef.current(); + setError( + cause instanceof Error ? cause.message : "Editor could not connect", + ); + } + }); + }, + [ + captionsEnabled, + preparingDuration, + preparingTitle, + preparingTracks, + restartAfterImport, + router, + sessionId, + userId, + videoId, + ], + ); + + useEffect(() => { + if (sessionId && iframeRef.current) onFrameLoad(iframeRef.current); + }, [onFrameLoad, sessionId]); + + if (error) { + return ( +
+