diff --git a/apps/desktop-gpui/src/editor_preparing/audio.rs b/apps/desktop-gpui/src/editor_preparing/audio.rs index 62929f287b1..bb5206e20e6 100644 --- a/apps/desktop-gpui/src/editor_preparing/audio.rs +++ b/apps/desktop-gpui/src/editor_preparing/audio.rs @@ -174,13 +174,16 @@ mod tests { impl TestDirectory { fn new() -> Self { + static NEXT_DIRECTORY: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); let nonce = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); let path = std::env::temp_dir().join(format!( - "cap-gpui-preparing-audio-{}-{nonce}", - std::process::id() + "cap-gpui-preparing-audio-{}-{nonce}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, std::sync::atomic::Ordering::Relaxed), )); std::fs::create_dir(&path).unwrap(); Self(path) diff --git a/apps/media-server/src/__tests__/lib/idle-timeout.test.ts b/apps/media-server/src/__tests__/lib/idle-timeout.test.ts new file mode 100644 index 00000000000..54c6c564749 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/idle-timeout.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from "bun:test"; +import { withIdleTimeout } from "../../lib/media-common"; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +describe("withIdleTimeout", () => { + test("lets slow work finish for as long as it keeps reporting progress", async () => { + const result = await withIdleTimeout(async (touch) => { + for (let step = 0; step < 8; step++) { + await wait(30); + touch(); + } + return "done"; + }, 80); + expect(result).toBe("done"); + }); + + test("stops work that goes quiet and runs its cleanup", async () => { + let cleaned = false; + const started = performance.now(); + await expect( + withIdleTimeout( + async (touch) => { + touch(); + await wait(1_000); + return "late"; + }, + 60, + () => { + cleaned = true; + }, + ), + ).rejects.toThrow("Stopped making progress"); + expect(performance.now() - started).toBeLessThan(500); + expect(cleaned).toBe(true); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/job-manager.test.ts b/apps/media-server/src/__tests__/lib/job-manager.test.ts index ff2b43e690b..5f2dc321540 100644 --- a/apps/media-server/src/__tests__/lib/job-manager.test.ts +++ b/apps/media-server/src/__tests__/lib/job-manager.test.ts @@ -7,11 +7,14 @@ import { createJob, deleteJob, getJob, + JOB_PROGRESS_STALL_MS, type JobProgress, + markJobProgress, type RecordingWorkerAcknowledgement, sendWebhook, touchJob, updateJob, + watchJobProgress, } from "../../lib/job-manager"; const createdJobs: string[] = []; @@ -521,4 +524,59 @@ describe("job cleanup", () => { expect(currentJob?.phase).toBe("error"); expect(currentJob?.error).toContain("maximum lifetime of 60 minutes"); }); + + test("keeps a job that is still making progress running for hours", () => { + const job = createTrackedJob("job-long-but-moving"); + const now = Date.now(); + job.phase = "processing"; + watchJobProgress(job.jobId); + job.createdAt = now - 3 * 60 * 60 * 1000; + job.updatedAt = now; + job.progressAt = now - 60_000; + + expect(cleanupExpiredJobs()).toBe(0); + expect(getJob(job.jobId)?.phase).toBe("processing"); + }); + + test("fails a job that stopped making progress, however short it is", () => { + const job = createTrackedJob("job-short-but-stuck"); + const now = Date.now(); + job.phase = "processing"; + watchJobProgress(job.jobId); + job.createdAt = now - 20 * 60 * 1000; + job.updatedAt = now; + job.progressAt = now - JOB_PROGRESS_STALL_MS - 60_000; + + expect(cleanupExpiredJobs()).toBe(1); + expect(getJob(job.jobId)).toMatchObject({ + phase: "error", + message: "Processing failed (stalled)", + }); + expect(getJob(job.jobId)?.error).toContain("stopped making progress"); + }); + + test("counts rising progress and new phases as progress, not heartbeats", () => { + const job = createTrackedJob("job-progress-signals"); + job.phase = "processing"; + job.progress = 20; + watchJobProgress(job.jobId); + const old = Date.now() - 10 * 60 * 1000; + + job.progressAt = old; + touchJob(job.jobId); + updateJob(job.jobId, { message: "Still here" }); + updateJob(job.jobId, { progress: 20 }); + expect(job.progressAt).toBe(old); + + updateJob(job.jobId, { progress: 21 }); + expect(job.progressAt).toBeGreaterThan(old); + + job.progressAt = old; + updateJob(job.jobId, { phase: "uploading" }); + expect(job.progressAt).toBeGreaterThan(old); + + job.progressAt = old; + markJobProgress(job.jobId); + expect(job.progressAt).toBeGreaterThan(old); + }); }); diff --git a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts index dd9844d100c..a18837e4eb9 100644 --- a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts @@ -32,6 +32,7 @@ import { repairContainer, uploadFileToS3, uploadFileToStorage, + uploadTimeoutMs, uploadToS3, } from "../../lib/media-video"; @@ -1008,6 +1009,80 @@ describe("processVideo integration tests", () => { expect(existsSync(tempFile.path)).toBe(false); }, 60000); + test("lets an encode run past its idle limit while frames keep advancing", async () => { + const dir = mkdtempSync(join(tmpdir(), "advancing-encode-")); + const longInput = join(dir, "input.mp4"); + execFileSync("ffmpeg", [ + "-hide_banner", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=1280x720:rate=30", + "-t", + "60", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-y", + longInput, + ]); + const idleTimeoutMs = 1_500; + try { + const metadata = await probeVideo(`file://${longInput}`); + const progressUpdates: number[] = []; + const started = performance.now(); + + const tempFile = await processVideo( + longInput, + metadata, + { maxWidth: 960, maxHeight: 540, preset: "medium", idleTimeoutMs }, + (progress) => progressUpdates.push(progress), + ); + tempFiles.push(tempFile.path); + + expect(performance.now() - started).toBeGreaterThan(idleTimeoutMs * 2); + expect(progressUpdates.length).toBeGreaterThan(2); + for (let index = 1; index < progressUpdates.length; index++) { + expect(progressUpdates[index]).toBeGreaterThan( + progressUpdates[index - 1] ?? -1, + ); + } + await tempFile.cleanup(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 120000); + + test("stops an encode whose input stops producing frames", async () => { + const metadata = await probeVideo(`file://${TEST_VIDEO_WITH_AUDIO}`); + const dir = mkdtempSync(join(tmpdir(), "stalled-encode-")); + const stalledInput = join(dir, "input.mp4"); + execFileSync("mkfifo", [stalledInput]); + try { + const started = performance.now(); + await expect( + processVideo(stalledInput, metadata, { + remuxOnly: true, + idleTimeoutMs: 1_000, + }), + ).rejects.toThrow("Stopped making progress"); + expect(performance.now() - started).toBeLessThan(15_000); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 30000); + + test("gives large uploads time in proportion to their size", () => { + expect(uploadTimeoutMs(0)).toBe(10 * 60 * 1000); + expect(uploadTimeoutMs(200 * 1024 * 1024)).toBe(10 * 60 * 1000); + expect(uploadTimeoutMs(3 * 1024 ** 3)).toBe(3 * 1024 * 1000); + }); + test("respects CRF setting", async () => { const metadata = await probeVideo(`file://${TEST_VIDEO_WITH_AUDIO}`); diff --git a/apps/media-server/src/__tests__/lib/video-input-validation.integration.test.ts b/apps/media-server/src/__tests__/lib/video-input-validation.integration.test.ts index be9c875a566..7256e901176 100644 --- a/apps/media-server/src/__tests__/lib/video-input-validation.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/video-input-validation.integration.test.ts @@ -208,4 +208,37 @@ describe("raw recording input validation", () => { "timed out", ); }); + + test("reports decoding progress while it checks a clean recording", async () => { + let reports = 0; + await expect( + validateVideoInput(clean, undefined, undefined, { + idleTimeoutMs: 5_000, + onProgress: () => reports++, + }), + ).resolves.toBeUndefined(); + expect(reports).toBeGreaterThan(0); + }); + + test("still rejects a damaged recording while tracking progress", async () => { + await expect( + validateVideoInput(corrupt, undefined, undefined, { + idleTimeoutMs: 5_000, + onProgress: () => {}, + }), + ).rejects.toThrow("original upload has been preserved"); + }); + + test("stops a check whose input stops producing frames", async () => { + const stalled = join(directory, "stalled.webm"); + execFileSync("mkfifo", [stalled]); + let reports = 0; + await expect( + validateVideoInput(stalled, undefined, undefined, { + idleTimeoutMs: 1_000, + onProgress: () => reports++, + }), + ).rejects.toThrow("Stopped making progress"); + expect(reports).toBe(0); + }, 30_000); }); diff --git a/apps/media-server/src/__tests__/routes/video-progress-webhooks.test.ts b/apps/media-server/src/__tests__/routes/video-progress-webhooks.test.ts new file mode 100644 index 00000000000..1354323ee73 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/video-progress-webhooks.test.ts @@ -0,0 +1,98 @@ +import { afterAll, afterEach, describe, expect, spyOn, test } from "bun:test"; +import { copyFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import app from "../../app"; +import * as jobManager from "../../lib/job-manager"; +import { deleteJob, getJob } from "../../lib/job-manager"; +import * as mediaVideo from "../../lib/media-video"; + +const fixture = join(import.meta.dir, "..", "fixtures", "test-with-audio.mp4"); +const directory = mkdtempSync(join(tmpdir(), "cap-progress-webhooks-")); +const received: { videoId: string; phase: string; message?: string }[] = []; +const receiver = Bun.serve({ + port: 0, + async fetch(request) { + received.push(await request.json()); + return Response.json({ success: true }); + }, +}); + +afterAll(() => { + receiver.stop(true); + rmSync(directory, { recursive: true, force: true }); +}); + +const spies: { mockRestore: () => void }[] = []; +afterEach(() => { + for (const spy of spies.splice(0)) spy.mockRestore(); +}); + +async function encodeWithRapidProgress(priority: "normal" | "bulk") { + const videoId = `progress-${priority}`; + const input = join(directory, `${priority}-input.mp4`); + const output = join(directory, `${priority}-output.mp4`); + copyFileSync(fixture, input); + copyFileSync(fixture, output); + spies.push( + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true), + spyOn(mediaVideo, "downloadVideoToTemp").mockResolvedValue({ + path: input, + cleanup: async () => {}, + }), + spyOn(mediaVideo, "processVideo").mockImplementation( + async (_input, _metadata, _options, onProgress) => { + for (let step = 1; step <= 40; step++) { + onProgress?.(step * 2.5, `Encoding: ${step * 2.5}%`); + await Bun.sleep(60); + } + return { path: output, cleanup: async () => {} }; + }, + ), + spyOn(mediaVideo, "uploadFileToS3").mockResolvedValue({}), + ); + const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET ?? "test-secret"; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; + const response = await app.fetch( + new Request("http://localhost/video/process", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-media-server-secret": secret, + }, + body: JSON.stringify({ + videoId, + userId: "progress-test", + videoUrl: "https://example.com/raw.mp4", + outputPresignedUrl: "https://example.com/result.mp4", + webhookUrl: `http://127.0.0.1:${receiver.port}/progress`, + webhookSecret: secret, + priority, + }), + }), + ); + expect(response.status).toBe(200); + const { jobId } = (await response.json()) as { jobId: string }; + const deadline = Date.now() + 15_000; + while (Date.now() < deadline && getJob(jobId)?.phase !== "complete") { + await Bun.sleep(20); + } + expect(getJob(jobId)?.phase).toBe("complete"); + deleteJob(jobId); + return received.filter( + (update) => + update.videoId === videoId && update.message?.startsWith("Encoding"), + ).length; +} + +describe("encoding progress webhooks", () => { + test("keeps regular uploads updating every second for the share page", async () => { + const sent = await encodeWithRapidProgress("normal"); + expect(sent).toBeGreaterThanOrEqual(2); + expect(sent).toBeLessThanOrEqual(4); + }, 30_000); + + test("sends bulk imports one update per five seconds", async () => { + expect(await encodeWithRapidProgress("bulk")).toBe(1); + }, 30_000); +}); diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 74caa0deaa7..8de45892d29 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -124,6 +124,8 @@ export interface Job { webhookLastAttemptAt?: number; recordingVerificationDeadlineAt?: number; recordingProcessingDeadlineAt?: number; + progressWatched?: boolean; + progressAt?: number; recordingVerification?: RecordingVerificationProof; manifestSha256?: string; jobId: string; @@ -150,6 +152,7 @@ const jobs = new Map(); const JOB_TTL_MS = 60 * 60 * 1000; const STALE_JOB_MS = 15 * 60 * 1000; const MAX_JOB_LIFETIME_MS = 60 * 60 * 1000; +export const JOB_PROGRESS_STALL_MS = 15 * 60 * 1000; const MAX_RECORDING_PROCESSING_BUDGET_MS = 3 * 60 * 60 * 1000; const WEBHOOK_MAX_ATTEMPTS = 3; const WEBHOOK_RETRY_BASE_MS = 500; @@ -402,6 +405,13 @@ export function updateJob( if (!job) return undefined; if (!isActivePhase(job.phase)) return undefined; + if ( + job.progressWatched && + ((updates.phase !== undefined && updates.phase !== job.phase) || + (updates.progress !== undefined && updates.progress > job.progress)) + ) { + job.progressAt = Date.now(); + } Object.assign(job, updates, { updatedAt: Date.now() }); if (job.recordingWorkerVersion) { job.recordingWorkerSequence = (job.recordingWorkerSequence ?? 0) + 1; @@ -475,6 +485,18 @@ export function beginRecordingProcessing( return true; } +export function watchJobProgress(jobId: string) { + const job = jobs.get(jobId); + if (!job) return; + job.progressWatched = true; + job.progressAt = Date.now(); +} + +export function markJobProgress(jobId: string) { + const job = jobs.get(jobId); + if (job?.progressWatched) job.progressAt = Date.now(); +} + export function deleteJob(jobId: string): boolean { const job = jobs.get(jobId); if (job) { @@ -582,6 +604,24 @@ export function cleanupExpiredJobs(): number { continue; } + if (job.progressWatched) { + const quietFor = now - (job.progressAt ?? job.createdAt); + if (isActivePhase(job.phase) && quietFor > JOB_PROGRESS_STALL_MS) { + console.warn( + `[job-manager] Marking job ${jobId} as error after ${Math.round(quietFor / 60000)}m without progress (phase=${job.phase}, age=${Math.round(age / 60000)}m)`, + ); + job.abortController?.abort(); + updateJob(jobId, { + phase: "error", + error: `Processing stopped making progress for ${Math.round(quietFor / 60000)} minutes`, + message: "Processing failed (stalled)", + }); + void sendWebhook(job); + cleaned++; + } + continue; + } + const deadline = job.recordingVerificationDeadlineAt ?? job.recordingProcessingDeadlineAt ?? diff --git a/apps/media-server/src/lib/media-common.ts b/apps/media-server/src/lib/media-common.ts index ad3c692242e..d844b880c40 100644 --- a/apps/media-server/src/lib/media-common.ts +++ b/apps/media-server/src/lib/media-common.ts @@ -61,6 +61,42 @@ export async function withTimeout( } } +export async function withIdleTimeout( + run: (touch: () => void) => Promise, + idleMs: number, + cleanup?: () => void | Promise, +): Promise { + let timeoutId: ReturnType | undefined; + let rejectIdle: ((error: Error) => void) | undefined; + let cleanupPromise: Promise | undefined; + const idlePromise = new Promise((_, reject) => { + rejectIdle = reject; + }); + const touch = () => { + if (timeoutId) clearTimeout(timeoutId); + timeoutId = setTimeout(() => { + cleanupPromise = Promise.resolve() + .then(() => cleanup?.()) + .then(() => undefined); + rejectIdle?.( + new Error( + `Stopped making progress for ${Math.round(idleMs / 1000)} seconds`, + ), + ); + }, idleMs); + }; + + touch(); + try { + return await Promise.race([run(touch), idlePromise]); + } catch (err) { + if (cleanupPromise) await cleanupPromise; + throw err; + } finally { + if (timeoutId) clearTimeout(timeoutId); + } +} + function isHttpUrl(path: string): boolean { try { const url = new URL(path); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 8f87ca0cec3..4e49a1492e6 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -11,6 +11,7 @@ import { PROCESS_TIMEOUT_MS, type ProgressCallback, UPLOAD_TIMEOUT_MS, + withIdleTimeout, withTimeout, } from "./media-common"; import { probeVideoFile } from "./media-probe"; @@ -30,6 +31,7 @@ import { const PROCESS_TIMEOUT_PER_SECOND_MS = 20_000; const MAX_PROCESS_TIMEOUT_MS = 2 * 60 * 60 * 1000; +const DOWNLOAD_IDLE_TIMEOUT_MS = 3 * 60 * 1000; // HLS/DASH sources are pulled as many sequential segment requests rather than // one streamed fetch, so per-request overhead scales with video length. A // flat 10-minute budget is enough for typical short recordings but not for a @@ -42,6 +44,7 @@ const PROBE_H264_LEVEL_TIMEOUT_MS = 10_000; const FFMPEG_HLS_CAPABILITY_TIMEOUT_MS = 10_000; const UPLOAD_MAX_RETRIES = 4; const UPLOAD_RETRY_BASE_MS = 250; +const UPLOAD_MIN_BYTES_PER_SECOND = 1024 * 1024; const MAX_STDERR_BYTES = 64 * 1024; const REPAIR_TIMEOUT_MS = 5 * 60 * 1000; const MAX_LEVEL_4_2_WIDTH = 2048; @@ -93,6 +96,7 @@ export interface VideoProcessingOptions { remuxOnly?: boolean; normalizeH264Level?: boolean; timeoutMs?: number; + idleTimeoutMs?: number; } export interface ThumbnailOptions { @@ -169,6 +173,7 @@ const DEFAULT_OPTIONS: Required = { remuxOnly: false, normalizeH264Level: false, timeoutMs: PROCESS_TIMEOUT_MS, + idleTimeoutMs: 0, }; const DEFAULT_THUMBNAIL_OPTIONS: Required = { @@ -983,14 +988,10 @@ async function storageResponseError( ); } -function parseProgressFromStderr( - stderrLine: string, - totalDurationUs: number, -): number | null { +function parseOutTimeUs(stderrLine: string): number | null { const match = stderrLine.match(/out_time_us=(\d+)/); if (!match) return null; - const currentUs = Number.parseInt(match[1] ?? "0", 10); - return Math.min(100, (currentUs / totalDurationUs) * 100); + return Number.parseInt(match[1] ?? "0", 10); } async function runFfmpegCommand( @@ -1228,6 +1229,7 @@ export async function downloadVideoToTemp( videoUrl: string, inputExtension?: string, abortSignal?: AbortSignal, + onBytes?: (bytes: number) => void, ): Promise { if (isStreamingUrl(videoUrl)) { return await downloadStreamingVideoToTemp(videoUrl, abortSignal); @@ -1240,11 +1242,26 @@ export async function downloadVideoToTemp( normalizeVideoInputExtension(inputExtension), ); + const idle = new AbortController(); + let idleTimer: ReturnType | undefined; + const stillReceiving = () => { + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout( + () => + idle.abort( + new Error( + `Download stopped receiving data for ${DOWNLOAD_IDLE_TIMEOUT_MS / 1000} seconds`, + ), + ), + DOWNLOAD_IDLE_TIMEOUT_MS, + ); + }; + try { - const timeoutSignal = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS); + stillReceiving(); const combinedSignal = abortSignal - ? AbortSignal.any([abortSignal, timeoutSignal]) - : timeoutSignal; + ? AbortSignal.any([abortSignal, idle.signal]) + : idle.signal; const response = await fetchMedia(videoUrl, { signal: combinedSignal, @@ -1263,11 +1280,15 @@ export async function downloadVideoToTemp( const reader = response.body.getReader(); const writer = file(tempFile.path).writer(); + let received = 0; try { while (true) { const { done, value } = await reader.read(); if (done) break; writer.write(value); + received += value.byteLength; + stillReceiving(); + onBytes?.(received); } await writer.end(); } finally { @@ -1281,7 +1302,11 @@ export async function downloadVideoToTemp( return tempFile; } catch (err) { await tempFile.cleanup(); - throw err; + throw idle.signal.aborted && !abortSignal?.aborted + ? idle.signal.reason + : err; + } finally { + if (idleTimer) clearTimeout(idleTimer); } } @@ -1599,56 +1624,65 @@ export async function processVideo( const stderrLines: string[] = []; const maxStderrLines = 50; - try { - await withTimeout( - (async () => { - void drainStream(proc.stdout as ReadableStream); + const encode = async (touch?: () => void) => { + void drainStream(proc.stdout as ReadableStream); - const stderrReader = ( - proc.stderr as ReadableStream - ).getReader(); - const decoder = new TextDecoder(); - let stderrBuffer = ""; - - try { - while (true) { - const { done, value } = await stderrReader.read(); - if (done) break; - - stderrBuffer += decoder.decode(value, { stream: true }); - const lines = stderrBuffer.split("\n"); - stderrBuffer = lines.pop() ?? ""; - - for (const line of lines) { - stderrLines.push(line); - if (stderrLines.length > maxStderrLines) { - stderrLines.shift(); - } - const progress = parseProgressFromStderr(line, totalDurationUs); - if (progress !== null && onProgress) { - onProgress(progress, `Encoding: ${Math.round(progress)}%`); - } - } + const stderrReader = ( + proc.stderr as ReadableStream + ).getReader(); + const decoder = new TextDecoder(); + let stderrBuffer = ""; + let encodedUs = -1; + + try { + while (true) { + const { done, value } = await stderrReader.read(); + if (done) break; + + stderrBuffer += decoder.decode(value, { stream: true }); + const lines = stderrBuffer.split("\n"); + stderrBuffer = lines.pop() ?? ""; + + for (const line of lines) { + stderrLines.push(line); + if (stderrLines.length > maxStderrLines) { + stderrLines.shift(); } - } finally { - stderrReader.releaseLock(); + const outTimeUs = parseOutTimeUs(line); + if (outTimeUs === null || outTimeUs <= encodedUs) continue; + encodedUs = outTimeUs; + touch?.(); + const progress = Math.min(100, (outTimeUs / totalDurationUs) * 100); + onProgress?.(progress, `Encoding: ${Math.round(progress)}%`); } + } + } finally { + stderrReader.releaseLock(); + } - const exitCode = await proc.exited; - if (exitCode !== 0) { - throw new Error( - `FFmpeg exited with code ${exitCode}. Last stderr: ${stderrLines.slice(-10).join(" | ")}`, - ); - } + const exitCode = await proc.exited; + if (exitCode !== 0) { + throw new Error( + `FFmpeg exited with code ${exitCode}. Last stderr: ${stderrLines.slice(-10).join(" | ")}`, + ); + } - const outputSize = await file(outputTempFile.path).size; - if (outputSize === 0) { - throw new Error("FFmpeg produced empty output file"); - } - })(), - processTimeoutMs, - () => terminateProcess(proc), - ); + const outputSize = await file(outputTempFile.path).size; + if (outputSize === 0) { + throw new Error("FFmpeg produced empty output file"); + } + }; + + try { + if (opts.idleTimeoutMs > 0) { + await withIdleTimeout(encode, opts.idleTimeoutMs, () => + terminateProcess(proc), + ); + } else { + await withTimeout(encode(), processTimeoutMs, () => + terminateProcess(proc), + ); + } return outputTempFile; } catch (err) { @@ -2106,8 +2140,15 @@ async function readUploadReceipt( return { objectIdentity: `"cap-drive-content-v1:${digest}"` }; } -function uploadSignal(abortSignal?: AbortSignal) { - const timeout = AbortSignal.timeout(UPLOAD_TIMEOUT_MS); +export function uploadTimeoutMs(contentLength = 0) { + return Math.max( + UPLOAD_TIMEOUT_MS, + Math.ceil(contentLength / UPLOAD_MIN_BYTES_PER_SECOND) * 1000, + ); +} + +function uploadSignal(abortSignal?: AbortSignal, contentLength?: number) { + const timeout = AbortSignal.timeout(uploadTimeoutMs(contentLength)); return abortSignal ? AbortSignal.any([abortSignal, timeout]) : timeout; } @@ -2152,7 +2193,7 @@ async function uploadWithRetry( method: "PUT", headers, body: bodyFactory(), - signal: uploadSignal(abortSignal), + signal: uploadSignal(abortSignal, contentLength), }); } catch (err) { abortSignal?.throwIfAborted(); diff --git a/apps/media-server/src/lib/video-input-validation.ts b/apps/media-server/src/lib/video-input-validation.ts index 41e26532384..87905b314f3 100644 --- a/apps/media-server/src/lib/video-input-validation.ts +++ b/apps/media-server/src/lib/video-input-validation.ts @@ -1,13 +1,23 @@ import { spawn } from "bun"; -import { PROCESS_TIMEOUT_MS, withTimeout } from "./media-common"; +import { + PROCESS_TIMEOUT_MS, + withIdleTimeout, + withTimeout, +} from "./media-common"; import { registerSubprocess, terminateProcess } from "./subprocess"; const MAX_DIAGNOSTIC_LENGTH = 8_192; +export interface ValidationProgress { + idleTimeoutMs: number; + onProgress: () => void; +} + export async function validateVideoInput( inputPath: string, abortSignal?: AbortSignal, timeoutMs = PROCESS_TIMEOUT_MS, + progress?: ValidationProgress, ): Promise { abortSignal?.throwIfAborted(); const proc = registerSubprocess( @@ -35,11 +45,12 @@ export async function validateVideoInput( "demux", "-abort_on", "empty_output", + ...(progress ? ["-progress", "pipe:1"] : []), "-f", "null", "-", ], - stdout: "ignore", + stdout: progress ? "pipe" : "ignore", stderr: "pipe", }), ); @@ -65,13 +76,52 @@ export async function validateVideoInput( reader.releaseLock(); } }; - const completion = Promise.all([proc.exited, readDiagnostics()]); + const readProgress = async (touch: () => void) => { + if (!progress) return; + const reader = (proc.stdout as ReadableStream).getReader(); + const decoder = new TextDecoder(); + let buffered = ""; + let decodedUs = -1; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffered += decoder.decode(value, { stream: true }); + const lines = buffered.split("\n"); + buffered = lines.pop() ?? ""; + for (const line of lines) { + const match = line.match(/^out_time_us=(\d+)/); + const outTimeUs = match ? Number.parseInt(match[1] ?? "0", 10) : -1; + if (outTimeUs <= decodedUs) continue; + decodedUs = outTimeUs; + touch(); + progress.onProgress(); + } + } + } finally { + reader.releaseLock(); + } + }; + let completion: Promise = Promise.resolve(); + const run = (touch: () => void) => { + const settled = Promise.all([ + proc.exited, + readDiagnostics(), + readProgress(touch), + ]); + completion = settled; + return settled; + }; try { - const [exitCode, diagnostics] = await withTimeout( - completion, - timeoutMs, - () => terminateProcess(proc), - ); + const [exitCode, diagnostics] = progress + ? await withIdleTimeout(run, progress.idleTimeoutMs, () => + terminateProcess(proc), + ) + : await withTimeout( + run(() => {}), + timeoutMs, + () => terminateProcess(proc), + ); abortSignal?.throwIfAborted(); if (diagnostics) { console.warn("[video/input-validation] Source decoding diagnostics", { diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index cd2b44e68b5..3080f8dbc2c 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -26,9 +26,11 @@ import { getMaxConcurrentVideoProcesses, getSystemResources, hasCriticalMemoryPressure, + markJobProgress, sendWebhook, touchJob, updateJob, + watchJobProgress, } from "../lib/job-manager"; import { PROCESS_TIMEOUT_MS } from "../lib/media-common"; import { renderEditedVideo } from "../lib/media-edit"; @@ -81,6 +83,10 @@ import { validateVideoInput } from "../lib/video-input-validation"; const video = new Hono(); const PROCESSING_HEARTBEAT_MS = 60 * 1000; +const LIVENESS_WEBHOOK_TICKS = 5; +const PROGRESS_WEBHOOK_INTERVAL_MS = 1000; +const BULK_PROGRESS_WEBHOOK_INTERVAL_MS = 5 * 1000; +const FFMPEG_IDLE_TIMEOUT_MS = 5 * 60 * 1000; const RECORDING_WORKER_INSTANCE = randomUUID(); const SEGMENTED_RECORDING_TIMEOUT_MS = 3 * PROCESS_TIMEOUT_MS + 35 * 60 * 1000; const POST_VERIFICATION_ASSET_BUDGET_MS = 5 * 60 * 1000; @@ -345,6 +351,36 @@ async function withJobHeartbeat( } } +async function withBoundedTransfer( + jobId: string, + operation: () => Promise, +): Promise { + let ticks = 0; + const interval = setInterval(() => { + const job = getJob(jobId); + if ( + !job || + job.phase === "complete" || + job.phase === "error" || + job.phase === "cancelled" + ) { + clearInterval(interval); + return; + } + touchJob(jobId); + markJobProgress(jobId); + ticks++; + if (ticks % LIVENESS_WEBHOOK_TICKS === 0) void sendWebhook(job); + }, PROCESSING_HEARTBEAT_MS); + interval.unref?.(); + + try { + return await operation(); + } finally { + clearInterval(interval); + } +} + async function withMuxMemoryGuard( abortController: AbortController, operation: () => Promise, @@ -926,11 +962,20 @@ async function processWithResilientRetry( preset: options.preset, remuxOnly: options.remuxOnly, normalizeH264Level: options.normalizeH264Level, + idleTimeoutMs: FFMPEG_IDLE_TIMEOUT_MS, }; + const progressWebhookIntervalMs = + options.priority === "bulk" + ? BULK_PROGRESS_WEBHOOK_INTERVAL_MS + : PROGRESS_WEBHOOK_INTERVAL_MS; + let progressWebhookAt = 0; const onProgress = (progress: number, message: string) => { const scaledProgress = 10 + progress * 0.7; updateJob(jobId, { progress: scaledProgress, message }); + const now = Date.now(); + if (now - progressWebhookAt < progressWebhookIntervalMs) return; + progressWebhookAt = now; const currentJob = getJob(jobId); if (currentJob) { void sendWebhook(currentJob).catch((error) => @@ -1289,6 +1334,7 @@ async function processVideoAsync( const abortController = new AbortController(); updateJob(jobId, { abortController }); + watchJobProgress(jobId); let repairedTempFile: TempFileHandle | null = null; let lastResortRepairFile: TempFileHandle | null = null; @@ -1301,11 +1347,12 @@ async function processVideoAsync( }); await sendWebhook(job); - const inputTempFile = await withJobHeartbeat(jobId, () => + const inputTempFile = await withBoundedTransfer(jobId, () => downloadVideoToTemp( videoUrl, options.inputExtension, abortController.signal, + () => markJobProgress(jobId), ), ); updateJob(jobId, { inputTempFile }); @@ -1314,7 +1361,7 @@ async function processVideoAsync( if (sourcePresignedUrl) { updateJob(jobId, { message: "Saving original video..." }); await sendWebhook(job); - await withJobHeartbeat(jobId, () => + await withBoundedTransfer(jobId, () => uploadFileToS3( inputTempFile.path, sourcePresignedUrl, @@ -1328,7 +1375,15 @@ async function processVideoAsync( if (isWebm) { updateJob(jobId, { message: "Checking the original recording..." }); await withJobHeartbeat(jobId, () => - validateVideoInput(inputTempFile.path, abortController.signal), + validateVideoInput( + inputTempFile.path, + abortController.signal, + undefined, + { + idleTimeoutMs: FFMPEG_IDLE_TIMEOUT_MS, + onProgress: () => markJobProgress(jobId), + }, + ), ); } @@ -1380,11 +1435,13 @@ async function processVideoAsync( }); await sendWebhook(job); - const uploadReceipt = await uploadFileToS3( - outputTempFile.path, - outputPresignedUrl, - "video/mp4", - abortController.signal, + const uploadReceipt = await withBoundedTransfer(jobId, () => + uploadFileToS3( + outputTempFile.path, + outputPresignedUrl, + "video/mp4", + abortController.signal, + ), ); if (thumbnailPresignedUrl || previewGifPresignedUrl) { diff --git a/apps/web/__tests__/integration/loom-import-jobs.test.ts b/apps/web/__tests__/integration/loom-import-jobs.test.ts new file mode 100644 index 00000000000..48dbf2c61be --- /dev/null +++ b/apps/web/__tests__/integration/loom-import-jobs.test.ts @@ -0,0 +1,1107 @@ +import { randomUUID } from "node:crypto"; +import { + importedVideos, + loomImportJobItems, + loomImportJobs, + organizationMembers, + organizations, + spaceMembers, + spaces, + spaceVideos, + users, + videos, + videoUploads, +} from "@cap/database/schema"; +import { Organisation, User, type Video } from "@cap/web-domain"; +import { and, asc, eq, inArray, notInArray } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { Effect, Option } from "effect"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as MySql2Database | undefined, + start: vi.fn(), + concurrency: "2", + globalConcurrency: "12", +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "test", + serverEnv: () => ({ + CAP_VIDEOS_DEFAULT_PUBLIC: true, + LOOM_IMPORT_CONCURRENCY: fixture.concurrency, + LOOM_IMPORT_GLOBAL_CONCURRENCY: fixture.globalConcurrency, + WEB_URL: "https://cap.test", + }), +})); +vi.mock("@cap/database/loops/queue", () => ({ + enqueueLoopsSync: vi.fn(async () => undefined), +})); +vi.mock("@cap/web-backend/src/Storage/index", () => ({ + Storage: { + getWritableAccessForUser: () => + Effect.succeed({ + bucketId: Option.none(), + storageIntegrationId: Option.none(), + }), + }, +})); +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); +vi.mock("workflow/api", () => ({ start: fixture.start })); +vi.mock("@/workflows/import-loom-video", () => ({ + importLoomVideoWorkflow: "importLoomVideoWorkflow", +})); +vi.mock("@/workflows/loom-import-job", () => ({ + loomImportJobWorkflow: "loomImportJobWorkflow", +})); + +import { + dispatchLoomImportForVideo, + dispatchLoomImportJob, + dispatchLoomImports, +} from "@/lib/loom-import/dispatch"; +import { + cancelLoomImportJob, + createLoomImportJob, + listLoomImportJobs, + markLoomImportJobStarting, + prepareLoomImportJob, + resetFailedLoomImportItems, + resolveLoomImportJob, +} from "@/lib/loom-import/jobs"; +import { + LOOM_IMPORT_RESTART_MESSAGE, + LOOM_IMPORT_SILENT_ERROR, + recoverLoomImportJobs, +} from "@/lib/loom-import/recovery"; +import { markLoomImportRetrying } from "@/lib/loom-import/retry"; +import { + getLoomImportSnapshot, + LOOM_IMPORT_CURSOR_OVERLAP_MS, +} from "@/lib/loom-import/snapshot"; + +const databaseUrl = process.env.CAP_LOOM_IMPORT_TEST_DATABASE_URL; + +const LOOM = { + ok1: "0dd0a01e10c742b28dbea75082c08635", + ok2: "31f430c1a1e744b8a7b6c18a26982c71", + ok3: "2bff5385f32643ea9d065227ebfadd0d", + ok4: "41f6aad19fb74048a8e9dbf3105b5a6a", + private: "43e16ce2319c4171ba9d09f7999eca16", + missing: "ffffffffffffffffffffffffffffffff", +}; + +const RECORDED = { + [LOOM.ok1]: "2022-02-15T20:02:03.132Z", + [LOOM.ok2]: "2022-06-22T19:17:16.020Z", + [LOOM.ok3]: "2022-06-22T19:17:34.669Z", + [LOOM.ok4]: "2023-08-18T20:00:46.121Z", +}; + +function database() { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; +} + +function id() { + return randomUUID().replaceAll("-", "").slice(0, 15); +} + +function share(loomId: string) { + return `https://www.loom.com/share/${loomId}`; +} + +const loomFetchMock = vi.fn( + async (_url: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as { + variables: Record; + }; + const data: Record = {}; + for (const [key, loomId] of Object.entries(body.variables)) { + const alias = key.replace("id", "v"); + if (loomId === LOOM.private) { + data[alias] = { __typename: "PrivateVideo", id: loomId }; + } else if (loomId === LOOM.missing) { + data[alias] = null; + } else { + data[alias] = { + __typename: "RegularUserVideo", + name: `Video ${loomId.slice(0, 4)}`, + createdAt: RECORDED[loomId] ?? "2024-01-01T00:00:00.000Z", + thumbnails: { + default: `https://cdn.loom.com/sessions/thumbnails/${loomId}-00001.jpg`, + }, + video_properties: { duration: 120.5, width: 1920, height: 1080 }, + }; + } + } + return Response.json({ data }); + }, +); +const loomFetch = loomFetchMock as unknown as typeof fetch; + +async function makeOrganization({ pro }: { pro: boolean }) { + const ownerId = User.UserId.make(id()); + const orgId = Organisation.OrganisationId.make(id()); + await database() + .insert(users) + .values({ + id: ownerId, + email: `${ownerId}@example.com`, + name: "Owner", + activeOrganizationId: orgId, + defaultOrgId: orgId, + stripeSubscriptionStatus: pro ? "active" : null, + }); + await database().insert(organizations).values({ + id: orgId, + name: "Acme", + ownerId, + }); + await database().insert(organizationMembers).values({ + id: id(), + organizationId: orgId, + userId: ownerId, + role: "owner", + }); + return { ownerId, orgId }; +} + +async function items(jobId: string) { + return database() + .select() + .from(loomImportJobItems) + .where(eq(loomImportJobItems.jobId, jobId)) + .orderBy(asc(loomImportJobItems.rowNumber)); +} + +async function jobStatus(jobId: string) { + const [job] = await database() + .select({ status: loomImportJobs.status }) + .from(loomImportJobs) + .where(eq(loomImportJobs.id, jobId)); + return job?.status; +} + +async function finishVideo(videoId: Video.VideoId) { + await database() + .delete(videoUploads) + .where(eq(videoUploads.videoId, videoId)); + return dispatchLoomImportForVideo(videoId); +} + +async function failVideo(videoId: Video.VideoId, error: string) { + await database() + .update(videoUploads) + .set({ phase: "error", processingError: error }) + .where(eq(videoUploads.videoId, videoId)); + return dispatchLoomImportForVideo(videoId); +} + +function loomIds(count: number) { + return Array.from({ length: count }, () => randomUUID().replaceAll("-", "")); +} + +async function readyJob( + owner: { ownerId: User.UserId; orgId: Organisation.OrganisationId }, + count: number, +) { + const { jobId } = await createLoomImportJob({ + userId: owner.ownerId, + orgId: owner.orgId, + fileName: `${count}.csv`, + rows: loomIds(count).map((loomId, index) => ({ + rowNumber: index + 2, + loomUrl: share(loomId), + })), + }); + await resolveLoomImportJob(jobId, { fetchImpl: loomFetch }); + expect(await prepareLoomImportJob(jobId)).toBe("importing"); + return jobId; +} + +async function inFlight() { + return database() + .select({ + jobId: loomImportJobItems.jobId, + videoId: loomImportJobItems.videoId, + updatedAt: loomImportJobItems.updatedAt, + }) + .from(loomImportJobItems) + .where(eq(loomImportJobItems.status, "importing")) + .orderBy(asc(loomImportJobItems.updatedAt), asc(loomImportJobItems.id)); +} + +describe.runIf(Boolean(databaseUrl))( + "Loom CSV imports with an isolated MySQL database", + () => { + let pool: Pool | undefined; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_loom_import_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error("Loom import tests require a local test database."); + } + pool = createPool(databaseUrl); + fixture.database = drizzle(pool); + await database().select().from(loomImportJobs).limit(1); + }); + + afterAll(async () => { + await pool?.end(); + fixture.database = undefined; + }); + + beforeEach(async () => { + fixture.start.mockReset().mockResolvedValue({ runId: "run" }); + fixture.concurrency = "2"; + fixture.globalConcurrency = "12"; + loomFetchMock.mockClear(); + await database().delete(loomImportJobItems); + await database().delete(loomImportJobs); + }); + + it("checks every link, waits for Pro on the free plan, then imports with original dates in a bounded window", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: false }); + const teammate = `teammate-${id()}@example.com`; + const { jobId, totalCount } = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "loom-export.csv", + rows: [ + { + rowNumber: 2, + loomUrl: share(LOOM.ok1), + ownerEmail: teammate, + spaceName: "Sales", + }, + { + rowNumber: 3, + loomUrl: `https://www.loom.com/share/My-Demo-${LOOM.ok2}?sid=1`, + }, + { rowNumber: 4, loomUrl: share(LOOM.ok3), spaceName: "sales" }, + { rowNumber: 5, loomUrl: share(LOOM.private) }, + { rowNumber: 6, loomUrl: share(LOOM.missing) }, + { rowNumber: 7, loomUrl: "https://example.com/not-loom" }, + { rowNumber: 8, loomUrl: share(LOOM.ok1) }, + { rowNumber: 9, loomUrl: share(LOOM.ok4) }, + ], + }); + expect(totalCount).toBe(8); + + await resolveLoomImportJob(jobId, { fetchImpl: loomFetch }); + expect(loomFetchMock).toHaveBeenCalledTimes(1); + const resolved = await items(jobId); + expect(resolved.map((item) => [item.rowNumber, item.status])).toEqual([ + [2, "ready"], + [3, "ready"], + [4, "ready"], + [5, "failed"], + [6, "failed"], + [7, "failed"], + [8, "skipped"], + [9, "ready"], + ]); + expect(resolved[0]).toMatchObject({ + title: `Video ${LOOM.ok1.slice(0, 4)}`, + durationSeconds: 120.5, + width: 1920, + thumbnailUrl: `https://cdn.loom.com/sessions/thumbnails/${LOOM.ok1}-00001.jpg`, + }); + expect(resolved[0]?.loomCreatedAt?.toISOString()).toBe( + RECORDED[LOOM.ok1], + ); + expect(resolved[1]?.loomVideoId).toBe(LOOM.ok2); + expect(resolved[3]?.error).toContain("private"); + expect(resolved[6]?.error).toBe("Same video as row 2."); + + expect(await prepareLoomImportJob(jobId)).toBe("awaiting_upgrade"); + const waiting = await getLoomImportSnapshot({ jobId, userId: ownerId }); + expect(waiting?.job).toMatchObject({ + status: "awaiting_upgrade", + canStart: false, + isPro: false, + }); + expect(waiting?.summary?.counts).toMatchObject({ + ready: 4, + failed: 3, + skipped: 1, + }); + expect(waiting?.summary?.totalDuration).toBeCloseTo(482); + expect(fixture.start).not.toHaveBeenCalled(); + expect( + await database().select().from(videos).where(eq(videos.orgId, orgId)), + ).toEqual([]); + + await database() + .update(users) + .set({ stripeSubscriptionStatus: "active" }) + .where(eq(users.id, ownerId)); + expect( + (await getLoomImportSnapshot({ jobId, userId: ownerId }))?.job.canStart, + ).toBe(true); + expect(await markLoomImportJobStarting(jobId)).toBe(true); + expect(await markLoomImportJobStarting(jobId)).toBe(false); + await resolveLoomImportJob(jobId, { fetchImpl: loomFetch }); + expect(await prepareLoomImportJob(jobId)).toBe("importing"); + + const [teammateUser] = await database() + .select({ id: users.id }) + .from(users) + .where(eq(users.email, teammate)); + expect(teammateUser).toBeDefined(); + const [salesSpace, ...otherSpaces] = await database() + .select({ id: spaces.id, name: spaces.name }) + .from(spaces) + .where(eq(spaces.organizationId, orgId)); + expect(otherSpaces).toEqual([]); + if (!salesSpace) throw new Error("Expected the Sales space to exist."); + expect(salesSpace.name).toBe("Sales"); + const members = await database() + .select({ userId: spaceMembers.userId, role: spaceMembers.role }) + .from(spaceMembers) + .where(eq(spaceMembers.spaceId, salesSpace.id)); + expect(members).toEqual( + expect.arrayContaining([ + { userId: ownerId, role: "admin" }, + { userId: teammateUser?.id, role: "member" }, + ]), + ); + + expect(await dispatchLoomImportJob(jobId)).toEqual({ + started: 2, + completed: false, + }); + let state = await items(jobId); + expect( + state + .filter((item) => item.status === "importing") + .map((item) => item.rowNumber), + ).toEqual([2, 3]); + expect(fixture.start).toHaveBeenCalledTimes(2); + const first = state[0]; + const firstVideoId = first?.videoId as Video.VideoId; + const [firstVideo] = await database() + .select() + .from(videos) + .where(eq(videos.id, firstVideoId)); + expect(firstVideo).toMatchObject({ + ownerId: teammateUser?.id, + orgId, + name: `Video ${LOOM.ok1.slice(0, 4)}`, + duration: 120.5, + transcriptionStatus: null, + }); + expect(firstVideo?.metadata).toEqual({ + customCreatedAt: RECORDED[LOOM.ok1], + }); + expect(firstVideo?.effectiveCreatedAt?.toISOString().slice(0, 19)).toBe( + RECORDED[LOOM.ok1]?.slice(0, 19), + ); + const [upload] = await database() + .select() + .from(videoUploads) + .where(eq(videoUploads.videoId, firstVideoId)); + expect(upload?.rawFileKey).toBe( + `${teammateUser?.id}/${firstVideoId}/raw-upload.mp4`, + ); + expect( + await database() + .select() + .from(spaceVideos) + .where(eq(spaceVideos.videoId, firstVideoId)), + ).toHaveLength(1); + expect(fixture.start.mock.calls[0]?.[1]?.[0]).toMatchObject({ + videoId: firstVideoId, + userId: teammateUser?.id, + loomVideoId: LOOM.ok1, + }); + + expect(await dispatchLoomImportJob(jobId)).toEqual({ + started: 0, + completed: false, + }); + + await finishVideo(firstVideoId); + state = await items(jobId); + expect(state[0]?.status).toBe("complete"); + expect( + state + .filter((item) => item.status === "importing") + .map((item) => item.rowNumber), + ).toEqual([3, 4]); + + await failVideo( + state[1]?.videoId as Video.VideoId, + "Media server unavailable", + ); + state = await items(jobId); + expect(state[1]).toMatchObject({ + status: "failed", + error: "Media server unavailable", + }); + expect(state[7]?.status).toBe("importing"); + + const live = await getLoomImportSnapshot({ jobId, userId: ownerId }); + expect(live?.summary?.counts).toMatchObject({ + imported: 1, + importing: 2, + failed: 4, + skipped: 1, + }); + const earlier = new Date(Date.now() - 60_000); + await database() + .update(loomImportJobItems) + .set({ updatedAt: earlier }) + .where(eq(loomImportJobItems.jobId, jobId)); + for (const item of state) { + if (!item.videoId) continue; + await database() + .update(videos) + .set({ updatedAt: earlier }) + .where(eq(videos.id, item.videoId)); + await database() + .update(videoUploads) + .set({ updatedAt: earlier }) + .where(eq(videoUploads.videoId, item.videoId)); + } + const pollStartedAt = Date.now(); + const settledView = await getLoomImportSnapshot({ + jobId, + userId: ownerId, + }); + expect(settledView?.cursor).toBeGreaterThanOrEqual(pollStartedAt); + const quiet = await getLoomImportSnapshot({ + jobId, + userId: ownerId, + since: + (settledView?.cursor ?? 0) + LOOM_IMPORT_CURSOR_OVERLAP_MS + 1_000, + }); + expect(quiet?.items.map((item) => [item.row, item.status])).toEqual([ + [4, "importing"], + [9, "importing"], + ]); + const lastPoll = Date.now() - 1_000; + await database() + .update(videoUploads) + .set({ processingProgress: 40, updatedAt: new Date() }) + .where(eq(videoUploads.videoId, state[7]?.videoId as Video.VideoId)); + await database() + .update(loomImportJobItems) + .set({ updatedAt: new Date() }) + .where(eq(loomImportJobItems.id, state[0]?.id as string)); + const delta = await getLoomImportSnapshot({ + jobId, + userId: ownerId, + since: lastPoll + LOOM_IMPORT_CURSOR_OVERLAP_MS, + }); + expect(delta?.full).toBe(false); + expect( + delta?.items.map((item) => [item.row, item.status, item.progress]), + ).toEqual([ + [2, "imported", undefined], + [4, "importing", 0], + [9, "importing", 40], + ]); + expect(delta?.summary).toBeNull(); + + await finishVideo(state[2]?.videoId as Video.VideoId); + await finishVideo(state[7]?.videoId as Video.VideoId); + expect(await jobStatus(jobId)).toBe("completed"); + + const retried = await resetFailedLoomImportItems(jobId); + expect(retried).toBe(3); + expect(await jobStatus(jobId)).toBe("checking"); + state = await items(jobId); + expect(state[1]).toMatchObject({ status: "ready", error: null }); + expect(state[4]?.status).toBe("pending"); + await resolveLoomImportJob(jobId, { fetchImpl: loomFetch }); + expect(await prepareLoomImportJob(jobId)).toBe("importing"); + fixture.start.mockClear(); + expect((await dispatchLoomImportJob(jobId)).started).toBe(1); + expect(fixture.start.mock.calls[0]?.[1]?.[0]).toMatchObject({ + videoId: state[1]?.videoId, + reuseExistingRawUpload: true, + }); + + const [summary] = await listLoomImportJobs({ userId: ownerId, orgId }); + expect(summary).toMatchObject({ id: jobId, imported: 3, totalCount: 8 }); + }); + + it("skips videos already in Cap and stops queued work when cancelled", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: true }); + fixture.concurrency = "1"; + const first = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "first.csv", + rows: [{ rowNumber: 2, loomUrl: share(LOOM.ok1) }], + }); + await resolveLoomImportJob(first.jobId, { fetchImpl: loomFetch }); + await prepareLoomImportJob(first.jobId); + await dispatchLoomImportJob(first.jobId); + + const second = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "second.csv", + rows: [ + { rowNumber: 2, loomUrl: share(LOOM.ok1) }, + { rowNumber: 3, loomUrl: share(LOOM.ok2) }, + { rowNumber: 4, loomUrl: share(LOOM.ok3) }, + ], + }); + await resolveLoomImportJob(second.jobId, { fetchImpl: loomFetch }); + const [duplicate] = await items(second.jobId); + const [existing] = await database() + .select({ id: importedVideos.id }) + .from(importedVideos) + .where( + and( + eq(importedVideos.orgId, orgId), + eq(importedVideos.sourceId, LOOM.ok1), + ), + ); + expect(duplicate).toMatchObject({ + status: "skipped", + error: "Already in Cap.", + videoId: existing?.id, + }); + + await prepareLoomImportJob(second.jobId); + await dispatchLoomImportJob(second.jobId); + await cancelLoomImportJob(second.jobId); + const state = await items(second.jobId); + expect(state.map((item) => item.status)).toEqual([ + "skipped", + "importing", + "cancelled", + ]); + expect(await jobStatus(second.jobId)).toBe("cancelled"); + expect(await dispatchLoomImportJob(second.jobId)).toEqual({ + started: 0, + completed: false, + }); + + await finishVideo(state[1]?.videoId as Video.VideoId); + expect((await items(second.jobId))[1]).toMatchObject({ + status: "complete", + error: null, + }); + expect(await jobStatus(second.jobId)).toBe("cancelled"); + }); + + it("marks a failed video imported once a retry from the video page finishes, even after the import ended", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: true }); + const { jobId } = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "retry.csv", + rows: [{ rowNumber: 2, loomUrl: share(LOOM.ok1) }], + }); + await resolveLoomImportJob(jobId, { fetchImpl: loomFetch }); + await prepareLoomImportJob(jobId); + await dispatchLoomImportJob(jobId); + const videoId = (await items(jobId))[0]?.videoId as Video.VideoId; + await failVideo(videoId, "Media server unavailable"); + expect(await jobStatus(jobId)).toBe("completed"); + expect((await items(jobId))[0]).toMatchObject({ + status: "failed", + error: "Media server unavailable", + }); + + await database() + .update(videoUploads) + .set({ + phase: "processing", + processingError: null, + processingMessage: "Retrying Loom import...", + }) + .where(eq(videoUploads.videoId, videoId)); + await dispatchLoomImportForVideo(videoId); + expect((await items(jobId))[0]?.status).toBe("failed"); + expect( + (await getLoomImportSnapshot({ jobId, userId: ownerId }))?.summary + ?.counts.importing, + ).toBe(1); + + await finishVideo(videoId); + expect((await items(jobId))[0]).toMatchObject({ + status: "complete", + error: null, + videoId, + }); + const finished = await getLoomImportSnapshot({ jobId, userId: ownerId }); + expect(finished?.summary?.counts).toMatchObject({ + imported: 1, + failed: 0, + }); + expect(finished?.items[0]).toMatchObject({ status: "imported", videoId }); + expect(await jobStatus(jobId)).toBe("completed"); + const [summary] = await listLoomImportJobs({ userId: ownerId, orgId }); + expect(summary).toMatchObject({ id: jobId, imported: 1, failed: 0 }); + + await database() + .update(loomImportJobItems) + .set({ status: "failed", error: "Media server unavailable" }) + .where(eq(loomImportJobItems.jobId, jobId)); + expect(await resetFailedLoomImportItems(jobId)).toBe(0); + expect((await items(jobId))[0]).toMatchObject({ + status: "complete", + error: null, + }); + expect(await jobStatus(jobId)).toBe("completed"); + }); + + it("restarts a stuck import only once when two recovery runs overlap", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: true }); + const stuck = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "stuck.csv", + rows: [{ rowNumber: 2, loomUrl: share(LOOM.ok1) }], + }); + await resolveLoomImportJob(stuck.jobId, { fetchImpl: loomFetch }); + expect(await prepareLoomImportJob(stuck.jobId)).toBe("importing"); + await dispatchLoomImportJob(stuck.jobId); + const videoId = (await items(stuck.jobId))[0]?.videoId as Video.VideoId; + const checking = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "checking.csv", + rows: [{ rowNumber: 2, loomUrl: share(LOOM.ok2) }], + }); + const ours = [stuck.jobId, checking.jobId]; + + await database() + .update(loomImportJobs) + .set({ status: "cancelled" }) + .where( + and( + inArray(loomImportJobs.status, ["checking", "importing"]), + notInArray(loomImportJobs.id, ours), + ), + ); + const longAgo = new Date(Date.now() - 60 * 60 * 1000); + await database() + .update(loomImportJobs) + .set({ updatedAt: longAgo }) + .where(inArray(loomImportJobs.id, ours)); + await database() + .update(videoUploads) + .set({ updatedAt: longAgo }) + .where(eq(videoUploads.videoId, videoId)); + fixture.start.mockClear(); + + const runs = await Promise.all([ + recoverLoomImportJobs(), + recoverLoomImportJobs(), + ]); + + const startsFor = (workflow: string, key: string, value: string) => + fixture.start.mock.calls.filter( + ([started, [payload]]) => + started === workflow && + (payload as Record)[key] === value, + ); + expect( + startsFor("importLoomVideoWorkflow", "videoId", videoId), + ).toHaveLength(1); + expect( + startsFor("loomImportJobWorkflow", "jobId", checking.jobId), + ).toHaveLength(1); + expect(runs.reduce((sum, run) => sum + run.restarted, 0)).toBe(1); + expect(runs.reduce((sum, run) => sum + run.resumed, 0)).toBe(1); + const [upload] = await database() + .select() + .from(videoUploads) + .where(eq(videoUploads.videoId, videoId)); + expect(upload).toMatchObject({ + phase: "uploading", + processingMessage: LOOM_IMPORT_RESTART_MESSAGE, + }); + + fixture.start.mockClear(); + await recoverLoomImportJobs(); + expect(fixture.start).not.toHaveBeenCalled(); + }); + + it("rejects files over the 2,000 video limit without creating anything", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: true }); + const rows = Array.from({ length: 2001 }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: share(`a${index.toString(16).padStart(31, "0")}`), + })); + await expect( + createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "big.csv", + rows, + }), + ).rejects.toThrow("Split your list"); + expect( + await database() + .select() + .from(loomImportJobs) + .where(eq(loomImportJobs.orgId, orgId)), + ).toEqual([]); + }); + + it("limits how many imports one person can start in an hour", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: false }); + const rows = [{ rowNumber: 2, loomUrl: share(LOOM.ok1) }]; + for (let index = 0; index < 30; index++) { + await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: `${index}.csv`, + rows, + }); + } + await expect( + createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "one-more.csv", + rows, + }), + ).rejects.toThrow("a lot of imports"); + }); + + it("creates a 2,000 video import in one request", async () => { + const { ownerId, orgId } = await makeOrganization({ pro: true }); + const rows = Array.from({ length: 2000 }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: share(`b${index.toString(16).padStart(31, "0")}`), + })); + const startedAt = performance.now(); + const { jobId, totalCount } = await createLoomImportJob({ + userId: ownerId, + orgId, + fileName: "big.csv", + rows, + }); + const elapsed = performance.now() - startedAt; + expect(totalCount).toBe(2000); + expect(await items(jobId)).toHaveLength(2000); + const snapshotStartedAt = performance.now(); + const snapshot = await getLoomImportSnapshot({ jobId, userId: ownerId }); + const snapshotElapsed = performance.now() - snapshotStartedAt; + expect(snapshot?.summary?.counts.checking).toBe(2000); + console.info( + `[loom-import benchmark] create 2000 rows: ${elapsed.toFixed(0)}ms, full snapshot: ${snapshotElapsed.toFixed(0)}ms`, + ); + }); + + it("shares a system-wide limit fairly between people and finishes every import", async () => { + fixture.globalConcurrency = "6"; + fixture.concurrency = "4"; + const [ana, ben, cy] = await Promise.all([ + makeOrganization({ pro: true }), + makeOrganization({ pro: true }), + makeOrganization({ pro: true }), + ]); + if (!ana || !ben || !cy) throw new Error("Missing organizations."); + const creatorOf = new Map(); + for (const [owner, count] of [ + [ana, 10], + [ana, 10], + [ben, 10], + [cy, 10], + ] as const) { + creatorOf.set(await readyJob(owner, count), owner.ownerId); + } + + expect((await dispatchLoomImports()).started).toBe(4); + expect((await dispatchLoomImports()).started).toBe(2); + expect((await dispatchLoomImports()).started).toBe(0); + const first = await inFlight(); + expect(first).toHaveLength(6); + const perPerson = new Map(); + for (const row of first) { + const creator = creatorOf.get(row.jobId) ?? ""; + perPerson.set(creator, (perPerson.get(creator) ?? 0) + 1); + } + expect([...perPerson.values()]).toEqual([2, 2, 2]); + + let most = 0; + let mostForOneImport = 0; + for (let step = 0; step < 100; step++) { + const running = await inFlight(); + if (running.length === 0) break; + most = Math.max(most, running.length); + const perJob = new Map(); + for (const row of running) { + perJob.set(row.jobId, (perJob.get(row.jobId) ?? 0) + 1); + } + mostForOneImport = Math.max(mostForOneImport, ...perJob.values()); + await finishVideo(running[0]?.videoId as Video.VideoId); + } + expect(most).toBe(6); + expect(mostForOneImport).toBeLessThanOrEqual(4); + for (const jobId of creatorOf.keys()) { + expect(await jobStatus(jobId)).toBe("completed"); + expect( + (await items(jobId)).every((item) => item.status === "complete"), + ).toBe(true); + } + expect(fixture.start).toHaveBeenCalledTimes(40); + }); + + it("starts nothing new while the media server is busy, then carries on", async () => { + fixture.concurrency = "4"; + const owner = await makeOrganization({ pro: true }); + const jobId = await readyJob(owner, 8); + expect((await dispatchLoomImportJob(jobId)).started).toBe(4); + const [busy, done] = await inFlight(); + await database() + .update(videoUploads) + .set({ processingMessage: "Queued for Loom import processing..." }) + .where(eq(videoUploads.videoId, busy?.videoId as Video.VideoId)); + + const paused = await finishVideo(done?.videoId as Video.VideoId); + expect(paused).toMatchObject({ started: 0, waiting: true, inFlight: 3 }); + expect(await inFlight()).toHaveLength(3); + + await database() + .update(videoUploads) + .set({ processingMessage: "Starting video processing..." }) + .where(eq(videoUploads.videoId, busy?.videoId as Video.VideoId)); + expect(await dispatchLoomImports()).toMatchObject({ + started: 1, + waiting: false, + inFlight: 4, + }); + }); + + it("keeps checking links Loom is rate limiting instead of failing them", async () => { + const owner = await makeOrganization({ pro: true }); + const ids = loomIds(30); + const { jobId } = await createLoomImportJob({ + userId: owner.ownerId, + orgId: owner.orgId, + fileName: "busy.csv", + rows: ids.map((loomId, index) => ({ + rowNumber: index + 2, + loomUrl: share(loomId), + })), + }); + let limited = true; + const rateLimited = vi.fn( + async (url: string | URL | Request, init?: RequestInit) => + limited + ? new Response("slow down", { + status: 429, + headers: { "Retry-After": "0.01" }, + }) + : loomFetchMock(url, init), + ) as unknown as typeof fetch; + + expect( + await resolveLoomImportJob(jobId, { fetchImpl: rateLimited }), + ).toEqual({ waiting: 30 }); + expect( + (await items(jobId)).every((item) => item.status === "pending"), + ).toBe(true); + expect(await jobStatus(jobId)).toBe("checking"); + + limited = false; + expect( + await resolveLoomImportJob(jobId, { + fetchImpl: rateLimited, + budgetMs: -1, + }), + ).toEqual({ waiting: 30 }); + expect( + await resolveLoomImportJob(jobId, { fetchImpl: rateLimited }), + ).toEqual({ waiting: 0 }); + expect( + (await items(jobId)).every((item) => item.status === "ready"), + ).toBe(true); + + const stuck = await createLoomImportJob({ + userId: owner.ownerId, + orgId: owner.orgId, + fileName: "down.csv", + rows: [{ rowNumber: 2, loomUrl: share(loomIds(1)[0] as string) }], + }); + limited = true; + expect( + await resolveLoomImportJob(stuck.jobId, { + fetchImpl: rateLimited, + giveUp: true, + }), + ).toEqual({ waiting: 0 }); + expect((await items(stuck.jobId))[0]).toMatchObject({ + status: "failed", + error: "Loom didn't respond. Try again in a minute.", + }); + }); + + it("queues a video again once when its workflow can't start", async () => { + fixture.concurrency = "1"; + const owner = await makeOrganization({ pro: true }); + const jobId = await readyJob(owner, 2); + fixture.start.mockRejectedValueOnce(new Error("Workflow unavailable")); + const quiet = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + + expect((await dispatchLoomImportJob(jobId)).started).toBe(1); + const [first] = await items(jobId); + const videoId = first?.videoId as Video.VideoId; + expect(first).toMatchObject({ status: "importing", error: null }); + expect(fixture.start).toHaveBeenCalledTimes(2); + expect(fixture.start.mock.calls[0]?.[1]?.[0]).toMatchObject({ + videoId, + }); + expect(fixture.start.mock.calls[1]?.[1]?.[0]).toMatchObject({ + videoId, + reuseExistingRawUpload: true, + }); + + fixture.start + .mockRejectedValueOnce(new Error("Workflow unavailable")) + .mockRejectedValueOnce(new Error("Workflow unavailable")); + await finishVideo(videoId); + const [, second] = await items(jobId); + expect(second).toMatchObject({ + status: "failed", + error: "Loom import could not start.", + }); + expect(await jobStatus(jobId)).toBe("completed"); + quiet.mockRestore(); + }); + + it("frees the slot of a video that stopped responding", async () => { + fixture.concurrency = "1"; + const owner = await makeOrganization({ pro: true }); + const jobId = await readyJob(owner, 2); + await dispatchLoomImportJob(jobId); + const [silent] = await inFlight(); + await database() + .update(videoUploads) + .set({ + phase: "processing", + updatedAt: new Date(Date.now() - 3 * 60 * 60 * 1000), + }) + .where(eq(videoUploads.videoId, silent?.videoId as Video.VideoId)); + fixture.start.mockClear(); + + const recovered = await recoverLoomImportJobs(); + expect(recovered).toMatchObject({ silent: 1, started: 1 }); + const [first, second] = await items(jobId); + expect(first).toMatchObject({ + status: "failed", + error: LOOM_IMPORT_SILENT_ERROR, + }); + expect(second?.status).toBe("importing"); + }); + + it("restarts a stalled upload once, then frees its slot if the restart never moves", async () => { + fixture.concurrency = "1"; + const owner = await makeOrganization({ pro: true }); + const jobId = await readyJob(owner, 2); + await dispatchLoomImportJob(jobId); + const [stalled] = await inFlight(); + const videoId = stalled?.videoId as Video.VideoId; + const age = async (minutes: number) => + database() + .update(videoUploads) + .set({ updatedAt: new Date(Date.now() - minutes * 60 * 1000) }) + .where(eq(videoUploads.videoId, videoId)); + + await age(35); + expect(await recoverLoomImportJobs()).toMatchObject({ + restarted: 1, + silent: 0, + }); + await age(35); + expect(await recoverLoomImportJobs()).toMatchObject({ + restarted: 0, + silent: 0, + }); + await age(60); + expect(await recoverLoomImportJobs()).toMatchObject({ + restarted: 0, + silent: 1, + started: 1, + }); + const [first, second] = await items(jobId); + expect(first).toMatchObject({ + status: "failed", + error: LOOM_IMPORT_SILENT_ERROR, + }); + expect(second?.status).toBe("importing"); + }); + + it("shows a retry from the video page as copying while it runs", async () => { + const owner = await makeOrganization({ pro: true }); + const jobId = await readyJob(owner, 1); + await dispatchLoomImportJob(jobId); + const videoId = (await items(jobId))[0]?.videoId as Video.VideoId; + await failVideo(videoId, "Media server unavailable"); + expect(await jobStatus(jobId)).toBe("completed"); + await database() + .update(loomImportJobItems) + .set({ updatedAt: new Date(Date.now() - 60_000) }) + .where(eq(loomImportJobItems.jobId, jobId)); + const before = await getLoomImportSnapshot({ + jobId, + userId: owner.ownerId, + }); + + await database() + .update(videoUploads) + .set({ + phase: "processing", + processingProgress: 35, + processingError: null, + processingMessage: "Retrying Loom import...", + }) + .where(eq(videoUploads.videoId, videoId)); + await markLoomImportRetrying(videoId); + const delta = await getLoomImportSnapshot({ + jobId, + userId: owner.ownerId, + since: before?.cursor, + }); + expect(delta?.items).toMatchObject([ + { status: "importing", progress: 35, videoId }, + ]); + expect((await dispatchLoomImports()).inFlight).toBe(1); + + await finishVideo(videoId); + expect((await items(jobId))[0]).toMatchObject({ + status: "complete", + error: null, + }); + }); + }, +); diff --git a/apps/web/__tests__/integration/loom-import-scale.test.ts b/apps/web/__tests__/integration/loom-import-scale.test.ts new file mode 100644 index 00000000000..bead4665073 --- /dev/null +++ b/apps/web/__tests__/integration/loom-import-scale.test.ts @@ -0,0 +1,979 @@ +import { randomUUID } from "node:crypto"; +import { + importedVideos, + loomImportJobItems, + loomImportJobs, + organizationMembers, + organizations, + users, + videos, + videoUploads, +} from "@cap/database/schema"; +import { Organisation, User, Video } from "@cap/web-domain"; +import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { Effect, Option } from "effect"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as MySql2Database | undefined, + start: vi.fn(), + sleepScale: 0.05, + queries: 0, +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "test", + serverEnv: () => ({ + CAP_VIDEOS_DEFAULT_PUBLIC: true, + WEB_URL: "https://cap.test", + }), +})); +vi.mock("@cap/database/loops/queue", () => ({ + enqueueLoopsSync: vi.fn(async () => undefined), +})); +vi.mock("@cap/web-backend/src/Storage/index", () => ({ + Storage: { + getWritableAccessForUser: () => + Effect.succeed({ + bucketId: Option.none(), + storageIntegrationId: Option.none(), + }), + }, +})); +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); +vi.mock("workflow", () => ({ + sleep: async (duration: string) => { + const seconds = Number.parseFloat(duration); + await new Promise((resolve) => + setTimeout(resolve, seconds * 1000 * fixture.sleepScale), + ); + }, +})); +vi.mock("workflow/api", () => ({ start: fixture.start })); +vi.mock("@/workflows/import-loom-video", () => ({ + importLoomVideoWorkflow: "importLoomVideoWorkflow", +})); + +import { + dispatchLoomImportForVideo, + dispatchLoomImports, + loomImportConcurrency, + loomImportGlobalConcurrency, +} from "@/lib/loom-import/dispatch"; +import { + cancelLoomImportJob, + createLoomImportJob, +} from "@/lib/loom-import/jobs"; +import { getLoomImportSnapshot } from "@/lib/loom-import/snapshot"; +import { loomImportJobWorkflow } from "@/workflows/loom-import-job"; + +const databaseUrl = process.env.CAP_LOOM_IMPORT_TEST_DATABASE_URL; +const enabled = + Boolean(databaseUrl) && process.env.CAP_LOOM_IMPORT_SCALE_TEST === "1"; + +const LOOM_CAPACITY = 16; +const MEDIA_CAPACITY = 9; +const FAILURE_RATE = 0.01; +const SIMULATED_MEDIA_FAILURE = "Simulated media failure"; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function database() { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; +} + +function id() { + return randomUUID().replaceAll("-", "").slice(0, 15); +} + +function percentile(values: number[], fraction: number) { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + return ( + sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))] ?? + 0 + ); +} + +function stats(values: number[]) { + return { + count: values.length, + p50: Math.round(percentile(values, 0.5)), + p95: Math.round(percentile(values, 0.95)), + max: Math.round(Math.max(0, ...values)), + }; +} + +type Simulation = { + loom: { + active: number; + most: number; + requests: number; + limited: number; + errors: number; + }; + media: { + active: number; + most: number; + turnedAway: number; + completed: number; + failed: number; + }; + dispatchMs: number[]; + running: Set>; + launched: number; + errors: number; + processing: [number, number]; + slow?: { owners: Set; share: number; factor: number }; + slowRuns: number; +}; + +function newSimulation(processing: [number, number]): Simulation { + return { + loom: { active: 0, most: 0, requests: 0, limited: 0, errors: 0 }, + media: { active: 0, most: 0, turnedAway: 0, completed: 0, failed: 0 }, + dispatchMs: [], + running: new Set(), + launched: 0, + errors: 0, + processing, + slowRuns: 0, + }; +} + +function fakeLoom(simulation: Simulation): typeof fetch { + return (async (_url: string | URL | Request, init?: RequestInit) => { + const { loom } = simulation; + loom.requests++; + if (loom.active >= LOOM_CAPACITY) { + loom.limited++; + return new Response("slow down", { + status: 429, + headers: { "Retry-After": "0.2" }, + }); + } + if (Math.random() < 0.02) { + loom.errors++; + return new Response("unavailable", { status: 503 }); + } + loom.active++; + loom.most = Math.max(loom.most, loom.active); + try { + await wait(30 + Math.random() * 40); + const { variables } = JSON.parse(String(init?.body)) as { + variables: Record; + }; + const data: Record = {}; + for (const [key, loomId] of Object.entries(variables)) { + data[key.replace("id", "v")] = { + __typename: "RegularUserVideo", + name: `Recording ${loomId.slice(0, 6)}`, + createdAt: "2023-03-14T15:09:26.535Z", + thumbnails: { + default: `https://cdn.loom.com/sessions/thumbnails/${loomId}-00001.jpg`, + }, + video_properties: { duration: 95, width: 1920, height: 1080 }, + }; + } + return Response.json({ data }); + } finally { + loom.active--; + } + }) as typeof fetch; +} + +async function timedDispatch(simulation: Simulation, videoId: string) { + const startedAt = performance.now(); + await dispatchLoomImportForVideo(videoId); + simulation.dispatchMs.push(performance.now() - startedAt); +} + +async function simulateVideo( + simulation: Simulation, + videoId: Video.VideoId, + ownerId: string, +) { + const { media } = simulation; + const upload = (values: Partial) => + database() + .update(videoUploads) + .set({ ...values, updatedAt: new Date() }) + .where(eq(videoUploads.videoId, videoId)); + + while (media.active >= MEDIA_CAPACITY) { + media.turnedAway++; + await upload({ processingMessage: "Queued for Loom import processing..." }); + await wait(100 + Math.random() * 100); + } + media.active++; + media.most = Math.max(media.most, media.active); + await upload({ + phase: "processing", + processingProgress: 0, + processingMessage: "Starting video processing...", + }); + await timedDispatch(simulation, videoId); + const [shortest, longest] = simulation.processing; + const slow = + simulation.slow?.owners.has(ownerId) && + Math.random() < simulation.slow.share; + if (slow) simulation.slowRuns++; + const duration = + (shortest + Math.random() * (longest - shortest)) * + (slow ? (simulation.slow?.factor ?? 1) : 1); + await wait(duration / 2); + await upload({ + processingProgress: 50, + processingMessage: "Processing video", + }); + await wait(duration / 2); + media.active--; + if (Math.random() < FAILURE_RATE) { + media.failed++; + await upload({ + phase: "error", + processingError: SIMULATED_MEDIA_FAILURE, + }); + } else { + media.completed++; + await database() + .delete(videoUploads) + .where(eq(videoUploads.videoId, videoId)); + } + await timedDispatch(simulation, videoId); + await timedDispatch(simulation, videoId); +} + +async function makePeople(count: number) { + const people: { ownerId: User.UserId; orgId: Organisation.OrganisationId }[] = + []; + for (let index = 0; index < count; index++) { + const ownerId = User.UserId.make(id()); + const orgId = Organisation.OrganisationId.make(id()); + await database() + .insert(users) + .values({ + id: ownerId, + email: `${ownerId}@scale.test`, + name: `Person ${index}`, + activeOrganizationId: orgId, + defaultOrgId: orgId, + stripeSubscriptionStatus: "active", + }); + await database() + .insert(organizations) + .values({ id: orgId, name: `Org ${index}`, ownerId }); + await database().insert(organizationMembers).values({ + id: id(), + organizationId: orgId, + userId: ownerId, + role: "owner", + }); + people.push({ ownerId, orgId }); + } + return people; +} + +async function cleanUp(orgIds: string[]) { + await database().delete(loomImportJobItems); + await database().delete(loomImportJobs); + if (orgIds.length === 0) return; + const created = await database() + .select({ id: videos.id }) + .from(videos) + .where(inArray(videos.orgId, orgIds as Organisation.OrganisationId[])); + for (let index = 0; index < created.length; index += 1000) { + const ids = created + .slice(index, index + 1000) + .map((row) => Video.VideoId.make(row.id)); + await database() + .delete(videoUploads) + .where(inArray(videoUploads.videoId, ids)); + await database() + .delete(importedVideos) + .where(inArray(importedVideos.id, ids)); + await database().delete(videos).where(inArray(videos.id, ids)); + } +} + +async function lookupFailureCount() { + const [row] = await database() + .select({ count: sql`COUNT(*)`.mapWith(Number) }) + .from(loomImportJobItems) + .where( + or( + eq(loomImportJobItems.status, "pending"), + and( + eq(loomImportJobItems.status, "failed"), + ne(loomImportJobItems.error, SIMULATED_MEDIA_FAILURE), + ), + ), + ); + return row?.count ?? 0; +} + +async function inFlightCount() { + const [row] = await database() + .select({ count: sql`COUNT(*)`.mapWith(Number) }) + .from(loomImportJobItems) + .where(eq(loomImportJobItems.status, "importing")); + return row?.count ?? 0; +} + +async function statusesByJob() { + return database() + .select({ + jobId: loomImportJobItems.jobId, + status: loomImportJobItems.status, + count: sql`COUNT(*)`.mapWith(Number), + }) + .from(loomImportJobItems) + .groupBy(loomImportJobItems.jobId, loomImportJobItems.status); +} + +function wireStart(simulation: Simulation) { + fixture.start + .mockReset() + .mockImplementation( + async ( + _workflow: unknown, + [payload]: [{ videoId: string; userId: string }], + ) => { + simulation.launched++; + const run = simulateVideo( + simulation, + Video.VideoId.make(payload.videoId), + payload.userId, + ).catch((error) => { + simulation.errors++; + console.error("[scale] simulated video failed", error); + }); + simulation.running.add(run); + run.finally(() => simulation.running.delete(run)); + return { runId: payload.videoId }; + }, + ); +} + +async function measure(runs: number, run: () => Promise) { + const ms: number[] = []; + const queries: number[] = []; + let result: T | undefined; + for (let index = 0; index < runs; index++) { + const before = fixture.queries; + const startedAt = performance.now(); + result = await run(); + ms.push(performance.now() - startedAt); + queries.push(fixture.queries - before); + } + return { + result: result as T, + ms: stats(ms), + queries: Math.max(...queries), + }; +} + +describe.runIf(enabled)("Loom CSV imports under load", () => { + let pool: Pool | undefined; + const orgIds: string[] = []; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_loom_import_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error("Loom import tests require a local test database."); + } + pool = createPool({ uri: databaseUrl, connectionLimit: 60 }); + fixture.database = drizzle(pool, { + logger: { + logQuery: () => { + fixture.queries++; + }, + }, + }); + await cleanUp([]); + }); + + afterAll(async () => { + await cleanUp(orgIds); + await pool?.end(); + fixture.database = undefined; + }); + + it( + "keeps each scheduling pass and each poll small with 100,000 rows queued", + async () => { + const people = await makePeople(50); + orgIds.push(...people.map((person) => person.orgId)); + fixture.start.mockReset().mockResolvedValue({ runId: "run" }); + const now = new Date(); + const jobIds: string[] = []; + for (const [index, person] of people.entries()) { + const jobId = id(); + jobIds.push(jobId); + await database() + .insert(loomImportJobs) + .values({ + id: jobId, + orgId: person.orgId, + createdById: person.ownerId, + fileName: `cost-${index}.csv`, + status: "importing", + totalCount: 2000, + startedAt: now, + }); + const rows = Array.from({ length: 2000 }, (_, row) => { + const loomVideoId = randomUUID().replaceAll("-", ""); + return { + id: id(), + jobId, + rowNumber: row + 2, + loomUrl: `https://www.loom.com/share/${loomVideoId}`, + loomVideoId, + status: "ready" as const, + ownerId: person.ownerId, + title: `Recording ${row}`, + durationSeconds: 95, + updatedAt: now, + }; + }); + for (let offset = 0; offset < rows.length; offset += 500) { + await database() + .insert(loomImportJobItems) + .values(rows.slice(offset, offset + 500)); + } + } + + const cold = await measure(1, () => dispatchLoomImports()); + await dispatchLoomImports(); + await dispatchLoomImports(); + expect(await inFlightCount()).toBe(12); + const full = await measure(20, () => dispatchLoomImports()); + expect(full.result.started).toBe(0); + + const finishOne = await measure(20, async () => { + const [running] = await database() + .select({ videoId: loomImportJobItems.videoId }) + .from(loomImportJobItems) + .where(eq(loomImportJobItems.status, "importing")) + .limit(1); + const videoId = running?.videoId as Video.VideoId; + await database() + .delete(videoUploads) + .where(eq(videoUploads.videoId, videoId)); + const before = fixture.queries; + const startedAt = performance.now(); + const outcome = await dispatchLoomImportForVideo(videoId); + const elapsed = performance.now() - startedAt; + const repeatBefore = fixture.queries; + await dispatchLoomImportForVideo(videoId); + return { + outcome, + elapsed, + queries: repeatBefore - before, + repeatQueries: fixture.queries - repeatBefore, + }; + }); + expect(finishOne.result.outcome?.started).toBe(1); + expect(await inFlightCount()).toBe(12); + + const [busy] = await database() + .select({ videoId: loomImportJobItems.videoId }) + .from(loomImportJobItems) + .where(eq(loomImportJobItems.status, "importing")) + .limit(1); + const admission = await measure(20, () => + dispatchLoomImportForVideo(busy?.videoId as Video.VideoId), + ); + + const owner = people[0] as (typeof people)[number]; + const jobId = jobIds[0] as string; + const first = await getLoomImportSnapshot({ + jobId, + userId: owner.ownerId, + }); + const deltaPoll = await measure(20, () => + getLoomImportSnapshot({ + jobId, + userId: owner.ownerId, + since: first?.cursor, + }), + ); + const fullPoll = await measure(5, () => + getLoomImportSnapshot({ jobId, userId: owner.ownerId }), + ); + + const explain = async (label: string, query: string) => { + const [rows] = (await pool?.query(`EXPLAIN ANALYZE ${query}`)) as [ + Array>, + unknown, + ]; + return { label, plan: Object.values(rows[0] ?? {})[0] }; + }; + const plans = [ + await explain( + "in-flight rows across all imports", + "SELECT i.id FROM loom_import_job_items i JOIN loom_import_jobs j ON j.id = i.job_id LEFT JOIN videos v ON v.id = i.video_id LEFT JOIN video_uploads u ON u.video_id = i.video_id WHERE i.status = 'importing'", + ), + await explain( + "next queued row of one import, without the index hint", + `SELECT id FROM loom_import_job_items WHERE job_id = '${jobId}' AND status = 'ready' ORDER BY csv_row LIMIT 1`, + ), + await explain( + "next queued row of one import, as the dispatcher asks", + `SELECT id FROM loom_import_job_items FORCE INDEX (job_status_row_idx) WHERE job_id = '${jobId}' AND status = 'ready' ORDER BY csv_row LIMIT 1`, + ), + + await explain( + "rows changed since the last poll", + `SELECT id FROM loom_import_job_items WHERE job_id = '${jobId}' AND updated_at > NOW(3) - INTERVAL 3 SECOND`, + ), + ]; + + await database() + .update(loomImportJobItems) + .set({ status: "complete" }) + .where( + and( + eq(loomImportJobItems.status, "ready"), + sql`${loomImportJobItems.rowNumber} <= 1991`, + ), + ); + await wait(15_000); + await pool?.query( + "TRUNCATE performance_schema.events_statements_summary_by_digest", + ); + const lateFull = await measure(20, () => dispatchLoomImports()); + const [slowest] = (await pool?.query( + "SELECT LEFT(DIGEST_TEXT, 160) AS statement, COUNT_STAR AS calls, ROUND(AVG_TIMER_WAIT / 1e9, 2) AS avg_ms, SUM_ROWS_EXAMINED AS rows_examined FROM performance_schema.events_statements_summary_by_digest WHERE SCHEMA_NAME = DATABASE() ORDER BY SUM_TIMER_WAIT DESC LIMIT 6", + )) as [Array>, unknown]; + console.info( + `[loom-import cost] slowest statements late in imports\n${JSON.stringify(slowest, null, 1)}`, + ); + const lateFinish = await measure(10, async () => { + const [running] = await database() + .select({ videoId: loomImportJobItems.videoId }) + .from(loomImportJobItems) + .where(eq(loomImportJobItems.status, "importing")) + .limit(1); + const videoId = running?.videoId as Video.VideoId; + await database() + .delete(videoUploads) + .where(eq(videoUploads.videoId, videoId)); + return dispatchLoomImportForVideo(videoId); + }); + expect(lateFinish.result?.started).toBe(1); + const lateDelta = await measure(20, () => + getLoomImportSnapshot({ + jobId, + userId: owner.ownerId, + since: Date.now(), + }), + ); + plans.push( + await explain( + "imports with nothing left, late in imports", + `SELECT j.id FROM loom_import_jobs j WHERE j.id IN (${jobIds + .map((value) => `'${value}'`) + .join(",")}) AND j.status = 'importing' ${[ + "pending", + "ready", + "importing", + ] + .map( + (status) => + `AND NOT EXISTS (SELECT 1 FROM loom_import_job_items i FORCE INDEX (job_status_row_idx) WHERE i.job_id = j.id AND i.status = '${status}')`, + ) + .join(" ")}`, + ), + await explain( + "next queued row late in an import, without the index hint", + `SELECT id FROM loom_import_job_items WHERE job_id = '${jobId}' AND status = 'ready' ORDER BY csv_row LIMIT 1`, + ), + await explain( + "next queued row late in an import, as the dispatcher asks", + `SELECT id FROM loom_import_job_items FORCE INDEX (job_status_row_idx) WHERE job_id = '${jobId}' AND status = 'ready' ORDER BY csv_row LIMIT 1`, + ), + ); + + const report = { + queuedRows: 100_000, + activeImports: jobIds.length, + lateInImports: { + passWhenFull: { queries: lateFull.queries, ms: lateFull.ms }, + videoFinished: { queries: lateFinish.queries, ms: lateFinish.ms }, + deltaPoll: { queries: lateDelta.queries, ms: lateDelta.ms }, + }, + coldPass: { queries: cold.queries, ms: cold.ms.max }, + passWhenFull: { queries: full.queries, ms: full.ms }, + videoFinished: { + queries: finishOne.result.queries, + repeatTriggerQueries: finishOne.result.repeatQueries, + ms: finishOne.ms, + }, + mediaServerAccepted: { queries: admission.queries, ms: admission.ms }, + deltaPoll: { + queries: deltaPoll.queries, + ms: deltaPoll.ms, + rows: deltaPoll.result?.items.length, + }, + fullPoll: { + queries: fullPoll.queries, + ms: fullPoll.ms, + rows: fullPoll.result?.items.length, + }, + }; + console.info(`[loom-import cost] ${JSON.stringify(report, null, 1)}`); + for (const plan of plans) { + console.info(`[loom-import plan] ${plan.label}\n${plan.plan}`); + } + + expect(full.queries).toBeLessThanOrEqual(8); + expect(finishOne.result.repeatQueries).toBe(1); + expect(deltaPoll.result?.items.length).toBeLessThanOrEqual(4); + await cleanUp([]); + }, + 10 * 60 * 1000, + ); + + it( + "50 people importing 2,000 videos each stay within capacity and all make progress", + async () => { + const people = await makePeople(50); + orgIds.push(...people.map((person) => person.orgId)); + const simulation = newSimulation([150, 350]); + wireStart(simulation); + vi.stubGlobal("fetch", fakeLoom(simulation)); + + const createdAt = performance.now(); + const jobIds: string[] = []; + for (const person of people) { + const { jobId } = await createLoomImportJob({ + userId: person.ownerId, + orgId: person.orgId, + fileName: "loom-export.csv", + rows: Array.from({ length: 2000 }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: `https://www.loom.com/share/${randomUUID().replaceAll("-", "")}`, + })), + }); + jobIds.push(jobId); + } + const createMs = performance.now() - createdAt; + + const globalLimit = loomImportGlobalConcurrency(); + const perJob = loomImportConcurrency(); + let most = 0; + let sampling = true; + const sampler = (async () => { + while (sampling) { + most = Math.max(most, await inFlightCount()); + await wait(100); + } + })(); + + const pollMs: number[] = []; + const pollRows: number[] = []; + let polling = true; + const pollers = people.map(async (person, index) => { + const jobId = jobIds[index] as string; + const first = await getLoomImportSnapshot({ + jobId, + userId: person.ownerId, + }); + let cursor = first?.cursor ?? Date.now(); + while (polling) { + await wait(2000); + const startedAt = performance.now(); + const delta = await getLoomImportSnapshot({ + jobId, + userId: person.ownerId, + since: cursor, + }); + pollMs.push(performance.now() - startedAt); + pollRows.push(delta?.items.length ?? 0); + cursor = delta?.cursor ?? cursor; + } + }); + + const workflowsStartedAt = performance.now(); + const results = await Promise.all( + jobIds.map((jobId) => loomImportJobWorkflow({ jobId })), + ); + const checkedMs = performance.now() - workflowsStartedAt; + expect(results.every((result) => result.status === "importing")).toBe( + true, + ); + + const lookupFailures = await lookupFailureCount(); + + await wait(90_000); + + for (const jobId of jobIds) await cancelLoomImportJob(jobId); + polling = false; + await Promise.all(pollers); + while (simulation.running.size > 0) { + await Promise.all(Array.from(simulation.running)); + } + sampling = false; + await sampler; + + const finalStatuses = await statusesByJob(); + const completedByJob = new Map(); + for (const row of finalStatuses) { + if (row.status === "complete") completedByJob.set(row.jobId, row.count); + } + const perJobCompleted = jobIds.map( + (jobId) => completedByJob.get(jobId) ?? 0, + ); + const stillImporting = finalStatuses + .filter((row) => row.status === "importing") + .reduce((total, row) => total + row.count, 0); + + const fullStartedAt = performance.now(); + const full = await getLoomImportSnapshot({ + jobId: jobIds[0] as string, + userId: people[0]?.ownerId as User.UserId, + }); + const fullMs = performance.now() - fullStartedAt; + + const report = { + people: people.length, + rows: people.length * 2000, + limits: { globalLimit, perJob, mediaCapacity: MEDIA_CAPACITY }, + createAllJobsMs: Math.round(createMs), + allLinksCheckedMs: Math.round(checkedMs), + lookupFailures, + loom: simulation.loom, + mostInFlight: most, + media: simulation.media, + videosLaunched: simulation.launched, + completedPerImport: { + min: Math.min(...perJobCompleted), + max: Math.max(...perJobCompleted), + total: perJobCompleted.reduce((total, value) => total + value, 0), + }, + stillImportingAfterDrain: stillImporting, + simulationErrors: simulation.errors, + dispatchMs: stats(simulation.dispatchMs), + deltaPollMs: stats(pollMs), + deltaPollRows: stats(pollRows), + fullSnapshotMs: Math.round(fullMs), + fullSnapshotRows: full?.items.length ?? 0, + }; + console.info(`[loom-import scale] ${JSON.stringify(report, null, 1)}`); + + expect(lookupFailures).toBe(0); + expect(most).toBeLessThanOrEqual(globalLimit); + expect(simulation.media.most).toBeLessThanOrEqual(MEDIA_CAPACITY); + expect(Math.min(...perJobCompleted)).toBeGreaterThan(0); + expect(stillImporting).toBe(0); + expect(simulation.errors).toBe(0); + }, + 20 * 60 * 1000, + ); + + it( + "50 people importing at the same time all finish, with only real failures left", + async () => { + await cleanUp([]); + const people = await makePeople(50); + orgIds.push(...people.map((person) => person.orgId)); + const simulation = newSimulation([40, 120]); + wireStart(simulation); + vi.stubGlobal("fetch", fakeLoom(simulation)); + + const jobIds: string[] = []; + for (const person of people) { + const { jobId } = await createLoomImportJob({ + userId: person.ownerId, + orgId: person.orgId, + fileName: "small.csv", + rows: Array.from({ length: 40 }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: `https://www.loom.com/share/${randomUUID().replaceAll("-", "")}`, + })), + }); + jobIds.push(jobId); + } + + const startedAt = performance.now(); + await Promise.all( + jobIds.map((jobId) => loomImportJobWorkflow({ jobId })), + ); + const deadline = Date.now() + 10 * 60 * 1000; + let most = 0; + while (Date.now() < deadline) { + most = Math.max(most, await inFlightCount()); + const [open] = await database() + .select({ count: sql`COUNT(*)`.mapWith(Number) }) + .from(loomImportJobs) + .where( + and( + inArray(loomImportJobs.id, jobIds), + ne(loomImportJobs.status, "completed"), + ), + ); + if (!open?.count && simulation.running.size === 0) break; + await wait(200); + } + const elapsed = performance.now() - startedAt; + + const finalStatuses = await statusesByJob(); + const totals = new Map(); + for (const row of finalStatuses) { + totals.set(row.status, (totals.get(row.status) ?? 0) + row.count); + } + const jobs = await database() + .select({ status: loomImportJobs.status }) + .from(loomImportJobs) + .where(inArray(loomImportJobs.id, jobIds)); + const report = { + people: people.length, + videos: people.length * 40, + finishedInMs: Math.round(elapsed), + mostInFlight: most, + media: simulation.media, + itemStatuses: Object.fromEntries(totals), + simulationErrors: simulation.errors, + jobsCompleted: jobs.filter((job) => job.status === "completed").length, + dispatchMs: stats(simulation.dispatchMs), + }; + console.info(`[loom-import scale] ${JSON.stringify(report, null, 1)}`); + + expect(report.jobsCompleted).toBe(50); + expect(simulation.errors).toBe(0); + expect(totals.get("complete")).toBe(simulation.media.completed); + expect(totals.get("failed") ?? 0).toBe(simulation.media.failed); + expect((totals.get("complete") ?? 0) + (totals.get("failed") ?? 0)).toBe( + 2000, + ); + expect(most).toBeLessThanOrEqual(loomImportGlobalConcurrency()); + }, + 20 * 60 * 1000, + ); + + it( + "29 people importing 2,000 videos each, some with 3 hour recordings, all keep moving", + async () => { + await cleanUp([]); + const people = await makePeople(29); + orgIds.push(...people.map((person) => person.orgId)); + const simulation = newSimulation([150, 350]); + const longOwners = new Set( + people.slice(0, 10).map((person) => person.ownerId), + ); + simulation.slow = { owners: longOwners, share: 0.25, factor: 30 }; + wireStart(simulation); + vi.stubGlobal("fetch", fakeLoom(simulation)); + + const jobIds: string[] = []; + for (const person of people) { + const { jobId } = await createLoomImportJob({ + userId: person.ownerId, + orgId: person.orgId, + fileName: "loom-export.csv", + rows: Array.from({ length: 2000 }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: `https://www.loom.com/share/${randomUUID().replaceAll("-", "")}`, + })), + }); + jobIds.push(jobId); + } + const jobOwner = new Map( + jobIds.map((jobId, index) => [jobId, people[index]?.ownerId ?? ""]), + ); + + let most = 0; + let sampling = true; + const sampler = (async () => { + while (sampling) { + most = Math.max(most, await inFlightCount()); + await wait(100); + } + })(); + + const checkedAt = performance.now(); + await Promise.all( + jobIds.map((jobId) => loomImportJobWorkflow({ jobId })), + ); + const checkedMs = performance.now() - checkedAt; + const lookupFailures = await lookupFailureCount(); + + await wait(90_000); + for (const jobId of jobIds) await cancelLoomImportJob(jobId); + while (simulation.running.size > 0) { + await Promise.all(Array.from(simulation.running)); + } + sampling = false; + await sampler; + + const finalStatuses = await statusesByJob(); + const completed = new Map(); + for (const row of finalStatuses) { + if (row.status === "complete") completed.set(row.jobId, row.count); + } + const withLong: number[] = []; + const shortOnly: number[] = []; + for (const jobId of jobIds) { + const count = completed.get(jobId) ?? 0; + if (longOwners.has(jobOwner.get(jobId) ?? "")) withLong.push(count); + else shortOnly.push(count); + } + const stillImporting = finalStatuses + .filter((row) => row.status === "importing") + .reduce((total, row) => total + row.count, 0); + const report = { + people: people.length, + rows: people.length * 2000, + peopleWithLongVideos: longOwners.size, + longVideoRuns: simulation.slowRuns, + allLinksCheckedMs: Math.round(checkedMs), + lookupFailures, + loom: simulation.loom, + mostInFlight: most, + media: simulation.media, + completedPerImport: { + withLongVideos: { + min: Math.min(...withLong), + max: Math.max(...withLong), + }, + shortVideosOnly: { + min: Math.min(...shortOnly), + max: Math.max(...shortOnly), + }, + }, + stillImportingAfterDrain: stillImporting, + simulationErrors: simulation.errors, + dispatchMs: stats(simulation.dispatchMs), + }; + console.info(`[loom-import scale] ${JSON.stringify(report, null, 1)}`); + + expect(lookupFailures).toBe(0); + expect(most).toBeLessThanOrEqual(loomImportGlobalConcurrency()); + expect(simulation.media.most).toBeLessThanOrEqual(MEDIA_CAPACITY); + expect(Math.min(...withLong, ...shortOnly)).toBeGreaterThan(0); + expect(stillImporting).toBe(0); + expect(simulation.errors).toBe(0); + }, + 20 * 60 * 1000, + ); +}); diff --git a/apps/web/__tests__/unit/agent-docs.test.ts b/apps/web/__tests__/unit/agent-docs.test.ts index 38d5f24f4a1..c9e88344dfa 100644 --- a/apps/web/__tests__/unit/agent-docs.test.ts +++ b/apps/web/__tests__/unit/agent-docs.test.ts @@ -102,7 +102,10 @@ describe("Cap for Agents docs", () => { expect(drive).toContain("organization manages storage"); const migration = getDocBySlug("migrating-to-cap")?.content ?? ""; - expect(migration).toContain("no more than 500 data rows"); + expect(migration).toContain("no more than 2,000 videos"); + expect(migration).toContain( + "generated the first time each video is opened", + ); expect(migration).toContain("one Cap space label"); expect(migration).toContain("cap jobs wait --json"); expect(migration).toContain("hundreds of users"); @@ -294,7 +297,7 @@ describe("Cap for Agents docs", () => { expect(llms).toContain("explicit confirmation before every mutation"); expect(llms).toContain("https://cap.so/docs/teams/google-drive"); expect(llms).toContain("https://cap.so/docs/migrating-to-cap"); - expect(llms).toContain("up to 500 rows per self-serve batch"); + expect(llms).toContain("up to 2,000 rows per self-serve batch"); }); it("keeps the reviewed documentation free of em dashes", () => { diff --git a/apps/web/__tests__/unit/loom-import-csv.test.ts b/apps/web/__tests__/unit/loom-import-csv.test.ts new file mode 100644 index 00000000000..8abd2c1dda3 --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-csv.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, it } from "vitest"; +import { + buildLoomImportPlan, + detectLoomImportMapping, + extractLoomVideoId, + LOOM_IMPORT_MAX_ROWS, + parseCsv, + parseCsvRecords, + tableFromPastedLinks, +} from "@/lib/loom-import/csv"; + +const ID_A = "0dd0a01e10c742b28dbea75082c08635"; +const ID_B = "31f430c1a1e744b8a7b6c18a26982c71"; + +describe("parseCsvRecords", () => { + it("handles quotes, escaped quotes, embedded newlines and CRLF", () => { + const text = + '\uFEFFtitle,link\r\n"Quarterly ""all hands""",https://www.loom.com/share/a\r\n"Line one\nline two",https://www.loom.com/share/b\r\n'; + expect(parseCsvRecords(text)).toEqual([ + ["title", "link"], + ['Quarterly "all hands"', "https://www.loom.com/share/a"], + ["Line one\nline two", "https://www.loom.com/share/b"], + ]); + }); + + it("keeps blank lines so row numbers match the spreadsheet, and rejects unclosed quotes", () => { + expect(parseCsvRecords("a,,c\n\n,,\nd,e,\n\n,,\n")).toEqual([ + ["a", "", "c"], + [""], + ["", "", ""], + ["d", "e", ""], + ]); + expect(() => parseCsvRecords('a,"unfinished\n')).toThrow("unclosed"); + }); +}); + +describe("parseCsv", () => { + it("detects semicolon and tab separated exports", () => { + expect( + parseCsv("Video URL;Owner\nhttps://loom.com/share/x;a@b.co").rows, + ).toEqual([["https://loom.com/share/x", "a@b.co"]]); + expect( + parseCsv("Video URL\tOwner\nhttps://loom.com/share/x\ta@b.co").delimiter, + ).toBe("\t"); + }); + + it("treats a plain list of Loom links as rows instead of a header", () => { + const table = parseCsv( + `https://www.loom.com/share/${ID_A}\nhttps://www.loom.com/share/${ID_B}\n`, + ); + expect(table.headerless).toBe(true); + expect(table.rows).toHaveLength(2); + }); + + it("keeps the first video in a plain list of Loom ids", () => { + const table = parseCsv(`${ID_A}\n${ID_B.toUpperCase()}\n`); + expect(table).toMatchObject({ + headerless: true, + rows: [[ID_A], [ID_B.toUpperCase()]], + }); + const plan = buildLoomImportPlan(table, detectLoomImportMapping(table), { + allowOwners: false, + }); + expect(plan.rows).toEqual([ + { rowNumber: 1, loomUrl: `https://www.loom.com/share/${ID_A}` }, + { rowNumber: 2, loomUrl: `https://www.loom.com/share/${ID_B}` }, + ]); + + const single = parseCsv(`${ID_A}\r\n`); + expect(single.headerless).toBe(true); + expect(single.rows).toEqual([[ID_A]]); + }); + + it("still reads a header row above a column of Loom ids", () => { + const table = parseCsv(`loom_video_id,owner\n${ID_A},a@b.co\n`); + expect(table.headerless).toBe(false); + expect(table.headers).toEqual(["loom_video_id", "owner"]); + expect(table.rows).toEqual([[ID_A, "a@b.co"]]); + }); +}); + +describe("extractLoomVideoId", () => { + it.each([ + [`https://www.loom.com/share/${ID_A}`, ID_A], + [`https://www.loom.com/share/${ID_A}?sid=123#t=10`, ID_A], + [`https://www.loom.com/embed/${ID_A}`, ID_A], + [`https://loom.com/share/Weekly-update-for-the-team-${ID_A}`, ID_A], + [`www.loom.com/share/${ID_A.toUpperCase()}`, ID_A], + [ID_B, ID_B], + ])("reads %s", (value, expected) => { + expect(extractLoomVideoId(value)).toBe(expected); + }); + + it.each([ + "", + "https://example.com/share/0dd0a01e10c742b28dbea75082c08635", + "https://www.loom.com/", + "https://notloom.com/share/0dd0a01e10c742b28dbea75082c08635", + "just some text", + ])("ignores %s", (value) => { + expect(extractLoomVideoId(value)).toBeNull(); + }); +}); + +describe("detectLoomImportMapping", () => { + it("finds columns by their values when headers are unfamiliar", () => { + const table = parseCsv( + [ + "Name,Recorder,Where,Recording", + `Kickoff,ana@acme.com,Sales,https://www.loom.com/share/${ID_A}`, + `Demo,sam@acme.com,Sales,https://www.loom.com/share/${ID_B}`, + ].join("\n"), + ); + expect(detectLoomImportMapping(table)).toEqual({ + loomUrl: 3, + ownerEmail: 1, + spaceName: undefined, + }); + }); + + it("maps the Cap template and Loom style exports", () => { + const template = parseCsv( + `loom_video_url,user_email,space_name\nhttps://www.loom.com/share/${ID_A},a@b.co,Sales`, + ); + expect(detectLoomImportMapping(template)).toEqual({ + loomUrl: 0, + ownerEmail: 1, + spaceName: 2, + }); + const loomExport = parseCsv( + `Video ID,Video Title,Creator Email,Workspace\n${ID_A},Intro,a@b.co,Acme`, + ); + expect(detectLoomImportMapping(loomExport)).toEqual({ + loomUrl: 0, + ownerEmail: 2, + spaceName: undefined, + }); + }); +}); + +describe("buildLoomImportPlan", () => { + const table = parseCsv( + [ + "loom_video_url,user_email,space_name", + `https://www.loom.com/share/${ID_A},Ana@Acme.com, Sales Team `, + "https://example.com/video,ana@acme.com,", + `https://www.loom.com/share/${ID_B},not-an-email,`, + `https://www.loom.com/share/${ID_A},sam@acme.com,`, + ",,", + `${ID_B},sam@acme.com,Design`, + ].join("\n"), + ); + + it("explains every row it leaves out and normalizes the rest", () => { + const plan = buildLoomImportPlan(table, detectLoomImportMapping(table), { + allowOwners: true, + }); + expect(plan.rows).toEqual([ + { + rowNumber: 2, + loomUrl: `https://www.loom.com/share/${ID_A}`, + ownerEmail: "ana@acme.com", + spaceName: "Sales Team", + }, + { + rowNumber: 7, + loomUrl: `https://www.loom.com/share/${ID_B}`, + ownerEmail: "sam@acme.com", + spaceName: "Design", + }, + ]); + expect(plan.issues.map((issue) => [issue.rowNumber, issue.reason])).toEqual( + [ + [3, "not_loom"], + [4, "bad_email"], + [5, "duplicate"], + ], + ); + expect(plan.issues[2]?.duplicateOf).toBe(2); + expect(plan).toMatchObject({ owners: 2, spaces: 2, overLimit: false }); + }); + + it("ignores owners and spaces for people who can only import for themselves", () => { + const plan = buildLoomImportPlan(table, detectLoomImportMapping(table), { + allowOwners: false, + }); + expect(plan.rows.map((row) => [row.rowNumber, row.ownerEmail])).toEqual([ + [2, undefined], + [4, undefined], + ]); + expect(plan.issues.map((issue) => [issue.rowNumber, issue.reason])).toEqual( + [ + [3, "not_loom"], + [5, "duplicate"], + [7, "duplicate"], + ], + ); + }); + + it("flags files over the per-import limit", () => { + const rows = Array.from( + { length: LOOM_IMPORT_MAX_ROWS + 1 }, + (_, index) => [ + `https://www.loom.com/share/${index.toString(16).padStart(32, "0")}`, + ], + ); + const plan = buildLoomImportPlan( + { headers: ["url"], rows, delimiter: ",", headerless: false }, + { loomUrl: 0 }, + { allowOwners: true }, + ); + expect(plan.overLimit).toBe(true); + }); + + it("parses and plans 2,000 rows quickly", () => { + const csv = [ + "Video Title,Video URL,Creator Email,Space", + ...Array.from( + { length: 2000 }, + (_, index) => + `"Weekly update, part ${index}",https://www.loom.com/share/${index.toString(16).padStart(32, "0")},person${index % 40}@acme.com,Team ${index % 7}`, + ), + ].join("\n"); + const startedAt = performance.now(); + const parsed = parseCsv(csv); + const plan = buildLoomImportPlan(parsed, detectLoomImportMapping(parsed), { + allowOwners: true, + }); + const elapsed = performance.now() - startedAt; + expect(plan.rows).toHaveLength(2000); + expect(plan.owners).toBe(40); + expect(plan.spaces).toBe(7); + expect(elapsed).toBeLessThan(250); + }); +}); + +describe("tableFromPastedLinks", () => { + it("pulls Loom links out of free text", () => { + const table = tableFromPastedLinks( + `Here you go: https://www.loom.com/share/${ID_A}. And (https://loom.com/share/${ID_B})\nthanks!`, + ); + expect(table.rows).toEqual([ + [`https://www.loom.com/share/${ID_A}`], + [`https://loom.com/share/${ID_B}`], + ]); + }); + + it("accepts bare Loom video ids", () => { + expect(tableFromPastedLinks(`${ID_A}\n${ID_B}`).rows).toEqual([ + [ID_A], + [ID_B], + ]); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import-job-view.test.ts b/apps/web/__tests__/unit/loom-import-job-view.test.ts new file mode 100644 index 00000000000..b3262bba60e --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-job-view.test.ts @@ -0,0 +1,360 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("next/link", () => ({ + default: ({ + children, + ...props + }: React.PropsWithChildren>) => + React.createElement("a", props, children), +})); + +import { + ROW_HEIGHT, + VirtualImportList, +} from "@/app/(org)/dashboard/import/loom/[jobId]/import-list"; +import { useLoomImportJob } from "@/app/(org)/dashboard/import/loom/[jobId]/use-loom-import-job"; +import { + type LoomImportItemView, + type LoomImportJobStatus, + type LoomImportSnapshot, + summarizeLoomImportItems, +} from "@/lib/loom-import/status"; + +const item = (index: number, overrides: Partial = {}) => + ({ + id: `item-${index}`, + row: index + 2, + url: `https://www.loom.com/share/${index}`, + loomId: String(index), + title: `Video ${index}`, + email: null, + space: null, + status: "queued", + videoId: null, + recordedAt: null, + duration: null, + thumb: null, + v: 1, + ...overrides, + }) satisfies LoomImportItemView; + +const snapshot = ({ + items, + full, + cursor, + status = "importing", + summary = full, +}: { + items: LoomImportItemView[]; + full: boolean; + cursor: number; + status?: LoomImportJobStatus; + summary?: boolean; +}): LoomImportSnapshot => ({ + job: { + id: "job-1", + fileName: "loom.csv", + status, + totalCount: 3, + createdAt: "2026-10-07T00:00:00.000Z", + startedAt: null, + completedAt: null, + createdByMe: true, + canStart: false, + isPro: true, + isAdmin: true, + }, + summary: summary ? summarizeLoomImportItems(items) : null, + items, + cursor, + full, +}); + +type Pending = { + url: string; + resolve: (body: LoomImportSnapshot) => void; +}; + +let root: Root; +let container: HTMLDivElement; +let requests: Pending[]; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + requests = []; + vi.stubGlobal( + "fetch", + vi.fn( + (url: string) => + new Promise((resolve) => { + requests.push({ + url, + resolve: (body) => + resolve({ + ok: true, + status: 200, + json: async () => body, + } as Response), + }); + }), + ), + ); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +async function flush() { + await act(async () => { + for (let index = 0; index < 10; index++) await Promise.resolve(); + }); +} + +describe("useLoomImportJob", () => { + it("waits for the full load before polling and applies answers in order", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + let latest: ReturnType | undefined; + const Harness = ({ initial }: { initial: LoomImportSnapshot }) => { + latest = useLoomImportJob(initial, { watchForUpgrade: false }); + return null; + }; + + await act(async () => { + root.render( + React.createElement(Harness, { + initial: snapshot({ + items: [item(0, { status: "ready" })], + full: false, + cursor: 1_000, + status: "checking", + summary: true, + }), + }), + ); + }); + expect(requests.map((request) => request.url)).toEqual([ + "/api/import/loom/jobs?jobId=job-1", + ]); + + await act(async () => { + vi.advanceTimersByTime(5_000); + }); + expect(requests).toHaveLength(1); + + requests[0]?.resolve( + snapshot({ + items: [ + item(0, { status: "ready" }), + item(1, { status: "ready" }), + item(2, { status: "failed" }), + ], + full: true, + cursor: 2_000, + status: "checking", + }), + ); + await flush(); + expect(latest?.items.map((row) => row.status)).toEqual([ + "ready", + "ready", + "failed", + ]); + expect(latest?.summary.counts).toMatchObject({ + ready: 2, + failed: 1, + total: 3, + }); + + await act(async () => { + vi.advanceTimersByTime(2_000); + }); + expect(requests[1]?.url).toBe( + "/api/import/loom/jobs?jobId=job-1&since=2000", + ); + const refresh = latest?.refresh(); + await flush(); + expect(requests).toHaveLength(2); + + requests[1]?.resolve( + snapshot({ + items: [item(0, { status: "importing", progress: 20, v: 2 })], + full: false, + cursor: 3_000, + }), + ); + await flush(); + expect(latest?.summary.job.status).toBe("importing"); + expect(latest?.items.map((row) => row.status)).toEqual([ + "importing", + "queued", + "failed", + ]); + expect(latest?.summary.counts).toMatchObject({ + importing: 1, + queued: 1, + failed: 1, + total: 3, + }); + expect(requests[2]?.url).toBe("/api/import/loom/jobs?jobId=job-1"); + + requests[2]?.resolve( + snapshot({ + items: [ + item(0, { status: "imported", v: 3 }), + item(1, { status: "importing", v: 3 }), + item(2, { status: "failed" }), + ], + full: true, + cursor: 4_000, + }), + ); + await act(async () => { + await refresh; + }); + expect(latest?.items.map((row) => row.status)).toEqual([ + "imported", + "importing", + "failed", + ]); + }); + + it("stops polling in a background tab and catches up when it comes back", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + let hidden = false; + Object.defineProperty(document, "hidden", { + configurable: true, + get: () => hidden, + }); + const setHidden = async (value: boolean) => { + hidden = value; + await act(async () => { + document.dispatchEvent(new Event("visibilitychange")); + }); + }; + const Harness = ({ initial }: { initial: LoomImportSnapshot }) => { + useLoomImportJob(initial, { watchForUpgrade: false }); + return null; + }; + try { + await act(async () => { + root.render( + React.createElement(Harness, { + initial: snapshot({ + items: [item(0, { status: "importing" })], + full: true, + cursor: 1_000, + }), + }), + ); + }); + await act(async () => { + vi.advanceTimersByTime(2_000); + }); + expect(requests).toHaveLength(1); + requests[0]?.resolve(snapshot({ items: [], full: false, cursor: 1_000 })); + await flush(); + + await setHidden(true); + await act(async () => { + vi.advanceTimersByTime(30 * 60_000); + }); + expect(requests).toHaveLength(1); + + await setHidden(false); + await flush(); + expect(requests.map((request) => request.url)).toEqual([ + "/api/import/loom/jobs?jobId=job-1&since=1000", + "/api/import/loom/jobs?jobId=job-1", + ]); + } finally { + Reflect.deleteProperty(document, "hidden"); + } + }); +}); + +describe("VirtualImportList", () => { + const rows = Array.from({ length: 200 }, (_, index) => item(index)); + const nextFrame = (callback: FrameRequestCallback) => { + queueMicrotask(() => callback(0)); + return 1; + }; + const render = async (items: LoomImportItemView[]) => { + await act(async () => { + root.render( + React.createElement(VirtualImportList, { + items, + viewport: 640, + showOwner: false, + empty: "No videos match", + }), + ); + }); + }; + const firstRenderedRow = () => + container.querySelector("li")?.getAttribute("aria-posinset"); + + it("shows the top of the list again after a search with no matches is cleared", async () => { + vi.stubGlobal("requestAnimationFrame", nextFrame); + await render(rows); + const list = container.querySelector( + "[data-testid=loom-import-list]", + ); + if (!list) throw new Error("Missing list"); + await act(async () => { + list.scrollTop = 120 * ROW_HEIGHT; + list.dispatchEvent(new Event("scroll")); + }); + expect(Number(firstRenderedRow())).toBeGreaterThan(100); + + await render([]); + expect(container.textContent).toContain("No videos match"); + + await render(rows); + expect(firstRenderedRow()).toBe("1"); + }); + + it("keeps rows on screen when the browser clamps the scroll after a search shrinks the list", async () => { + vi.stubGlobal("requestAnimationFrame", nextFrame); + await render(rows); + const list = container.querySelector( + "[data-testid=loom-import-list]", + ); + if (!list) throw new Error("Missing list"); + let requested = 0; + Object.defineProperty(list, "scrollTop", { + configurable: true, + get: () => { + const content = Number.parseFloat( + list.querySelector("ul")?.style.height ?? "0", + ); + return Math.min(requested, Math.max(0, content - 640)); + }, + set: (value: number) => { + requested = value; + }, + }); + await act(async () => { + list.scrollTop = 150 * ROW_HEIGHT; + list.dispatchEvent(new Event("scroll")); + }); + expect(Number(firstRenderedRow())).toBeGreaterThan(130); + + await render(rows.slice(0, 20)); + const rendered = Array.from(container.querySelectorAll("li")).map((li) => + Number(li.getAttribute("aria-posinset")), + ); + expect(rendered.length).toBeGreaterThan(0); + expect(Math.max(...rendered)).toBe(20); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import-schedule.test.ts b/apps/web/__tests__/unit/loom-import-schedule.test.ts new file mode 100644 index 00000000000..8358725b6ab --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-schedule.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from "vitest"; +import { + buildLoomImportPlan, + decodeLoomImportRows, + detectLoomImportMapping, + encodeLoomImportRows, + LOOM_IMPORT_MAX_ROWS, + parseCsv, +} from "@/lib/loom-import/csv"; +import { loomRetryDelayMs } from "@/lib/loom-import/loom-api"; +import { + addLoomImportLoad, + type LoomImportQueueJob, + type LoomImportQueueLoad, + pickLoomImportJob, +} from "@/lib/loom-import/schedule"; +import { summarizeLoomImportItems } from "@/lib/loom-import/status"; + +const job = ( + id: string, + creatorId: string, + overrides: Partial = {}, +): LoomImportQueueJob => ({ + id, + creatorId, + dispatchedAt: null, + startedAt: new Date(`2026-10-07T00:00:0${id.length}Z`), + ...overrides, +}); + +const emptyLoad = (): LoomImportQueueLoad => ({ + jobs: new Map(), + creators: new Map(), +}); + +function allocate( + jobs: LoomImportQueueJob[], + slots: number, + perJob: number, + load = emptyLoad(), +) { + const picks: string[] = []; + for (let slot = 0; slot < slots; slot++) { + const next = pickLoomImportJob(jobs, load, perJob); + if (!next) break; + picks.push(next.id); + addLoomImportLoad(load, next); + } + return picks; +} + +describe("pickLoomImportJob", () => { + it("shares slots between people before giving anyone a second one", () => { + const jobs = [ + job("a1", "ana"), + job("a2", "ana"), + job("a3", "ana"), + job("b1", "ben"), + job("c1", "cy"), + ]; + const picks = allocate(jobs, 6, 4); + expect(picks.slice(0, 3).sort()).toEqual(["a1", "b1", "c1"]); + const perPerson = new Map(); + for (const id of picks) { + const creator = jobs.find((candidate) => candidate.id === id)?.creatorId; + if (creator) perPerson.set(creator, (perPerson.get(creator) ?? 0) + 1); + } + expect([...perPerson.values()].sort()).toEqual([2, 2, 2]); + }); + + it("never gives one import more than its own limit", () => { + expect(allocate([job("a1", "ana")], 10, 4)).toEqual([ + "a1", + "a1", + "a1", + "a1", + ]); + }); + + it("serves the import that waited longest when everyone is even", () => { + const jobs = [ + job("served", "ana", { dispatchedAt: new Date("2026-10-07T10:00:00Z") }), + job("waiting", "ben", { + dispatchedAt: new Date("2026-10-07T09:00:00Z"), + }), + job("new", "cy"), + ]; + expect(allocate(jobs, 1, 4)).toEqual(["new"]); + expect(allocate(jobs.slice(0, 2), 1, 4)).toEqual(["waiting"]); + }); + + it("rotates through 50 imports when only 12 can copy at once", () => { + const jobs = Array.from({ length: 50 }, (_, index) => + job(`job-${String(index).padStart(2, "0")}`, `person-${index}`), + ); + const served = new Map(); + const load = emptyLoad(); + let clock = 0; + const inFlight: string[] = []; + for (let step = 0; step < 500; step++) { + while (inFlight.length < 12) { + const next = pickLoomImportJob(jobs, load, 4); + if (!next) break; + addLoomImportLoad(load, next); + next.dispatchedAt = new Date(++clock); + inFlight.push(next.id); + served.set(next.id, (served.get(next.id) ?? 0) + 1); + } + const finished = inFlight.shift(); + const owner = jobs.find((candidate) => candidate.id === finished); + if (owner) { + load.jobs.set(owner.id, (load.jobs.get(owner.id) ?? 1) - 1); + load.creators.set( + owner.creatorId, + (load.creators.get(owner.creatorId) ?? 1) - 1, + ); + } + } + const counts = [...served.values()]; + expect(served.size).toBe(50); + expect(Math.max(...counts) - Math.min(...counts)).toBeLessThanOrEqual(1); + }); +}); + +describe("loomRetryDelayMs", () => { + it("backs off further when Loom says it is rate limiting", () => { + expect(loomRetryDelayMs(0, null, () => 0)).toBe(400); + expect(loomRetryDelayMs(2, null, () => 0)).toBe(1600); + expect(loomRetryDelayMs(0, "", () => 0)).toBe(2000); + expect(loomRetryDelayMs(1, "", () => 0)).toBe(4000); + expect(loomRetryDelayMs(0, "7", () => 0)).toBe(7000); + expect(loomRetryDelayMs(0, "600", () => 0)).toBe(10_000); + expect(loomRetryDelayMs(0, "7", () => 1)).toBe(10_500); + }); +}); + +describe("compact import rows", () => { + it("round trips owners, spaces and canonical links", () => { + const table = parseCsv( + [ + "Video URL,Creator Email,Space", + "https://www.loom.com/share/My-Demo-0dd0a01e10c742b28dbea75082c08635?sid=1,ana@acme.com,Sales", + "https://www.loom.com/share/31f430c1a1e744b8a7b6c18a26982c71,ana@acme.com,", + "2bff5385f32643ea9d065227ebfadd0d,,Sales", + ].join("\n"), + ); + const plan = buildLoomImportPlan(table, detectLoomImportMapping(table), { + allowOwners: true, + }); + const payload = encodeLoomImportRows(plan.rows); + expect(payload).toEqual({ + owners: ["ana@acme.com"], + spaces: ["Sales"], + rows: [ + [2, "0dd0a01e10c742b28dbea75082c08635", 0, 0], + [3, "31f430c1a1e744b8a7b6c18a26982c71", 0, -1], + [4, "2bff5385f32643ea9d065227ebfadd0d", -1, 0], + ], + }); + expect(decodeLoomImportRows(JSON.parse(JSON.stringify(payload)))).toEqual([ + { + rowNumber: 2, + loomUrl: "https://www.loom.com/share/0dd0a01e10c742b28dbea75082c08635", + ownerEmail: "ana@acme.com", + spaceName: "Sales", + }, + { + rowNumber: 3, + loomUrl: "https://www.loom.com/share/31f430c1a1e744b8a7b6c18a26982c71", + ownerEmail: "ana@acme.com", + }, + { + rowNumber: 4, + loomUrl: "https://www.loom.com/share/2bff5385f32643ea9d065227ebfadd0d", + spaceName: "Sales", + }, + ]); + }); + + it("rejects payloads that are not import rows and keeps bad rows visible", () => { + expect(decodeLoomImportRows(null)).toBeNull(); + expect(decodeLoomImportRows({ rows: [] })).toBeNull(); + expect( + decodeLoomImportRows({ + owners: ["a@b.co"], + spaces: [], + rows: [["x", 5, 9, "y"], "nope"], + }), + ).toEqual([ + { rowNumber: 1, loomUrl: "" }, + { rowNumber: 2, loomUrl: "" }, + ]); + }); + + it("keeps a full 2,000 video import far below the 1 MB request limit", () => { + const rows = Array.from({ length: LOOM_IMPORT_MAX_ROWS }, (_, index) => ({ + rowNumber: index + 2, + loomUrl: `https://www.loom.com/share/A-very-long-video-title-that-loom-adds-to-links-${index.toString(16).padStart(32, "0")}?sid=0123456789abcdef0123456789abcdef`, + ownerEmail: `person-${index % 200}@a-company-with-a-long-domain.example.com`, + spaceName: `Customer success and onboarding ${index % 40}`, + })); + const bytes = new TextEncoder().encode( + JSON.stringify(encodeLoomImportRows(rows)), + ).length; + const naive = new TextEncoder().encode(JSON.stringify(rows)).length; + expect(bytes).toBeLessThan(150_000); + expect(naive).toBeGreaterThan(bytes * 3); + }); +}); + +describe("summarizeLoomImportItems", () => { + it("counts rows, owners and minutes the same way on server and client", () => { + expect( + summarizeLoomImportItems([ + { status: "imported", duration: 60, email: "a@b.co" }, + { status: "importing", duration: 30, email: "c@d.co" }, + { status: "failed", duration: 90, email: "a@b.co" }, + { status: "queued", duration: null, email: null }, + ]), + ).toEqual({ + counts: { + checking: 0, + ready: 0, + queued: 1, + importing: 1, + imported: 1, + failed: 1, + skipped: 0, + cancelled: 0, + total: 4, + }, + totalDuration: 90, + importedDuration: 60, + owners: 2, + }); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import-status.test.ts b/apps/web/__tests__/unit/loom-import-status.test.ts new file mode 100644 index 00000000000..67f318930c7 --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-status.test.ts @@ -0,0 +1,409 @@ +import { describe, expect, it, vi } from "vitest"; +import { mergeLoomImportItems } from "@/app/(org)/dashboard/import/loom/[jobId]/use-loom-import-job"; +import { + buildLoomVideosQuery, + lookupLoomVideos, + toLoomVideoLookup, +} from "@/lib/loom-import/loom-api"; +import { buildLoomImportReport } from "@/lib/loom-import/report"; +import { + countLoomImportItems, + deriveLoomImportItemState, + type LoomImportItemSource, + type LoomImportItemView, + type LoomImportJobStatus, + type LoomImportSettleSource, + type LoomImportSnapshot, + loomImportProgress, + loomImportWaitingStatus, + settleLoomImportItem, +} from "@/lib/loom-import/status"; + +const source = ( + overrides: Partial, +): LoomImportItemSource => ({ + status: "importing", + videoId: "video-1", + error: null, + videoExists: true, + uploadPhase: "processing", + uploadProgress: 30, + uploadMessage: "Processing video...", + uploadError: null, + ...overrides, +}); + +describe("deriveLoomImportItemState", () => { + it("shows checked rows as ready before an upgrade and queued once importing", () => { + expect( + deriveLoomImportItemState( + source({ status: "ready" }), + "awaiting_upgrade", + ), + ).toEqual({ + status: "ready", + }); + expect( + deriveLoomImportItemState(source({ status: "ready" }), "importing"), + ).toEqual({ + status: "queued", + }); + }); + + it("reads live media progress and friendly stages", () => { + expect(deriveLoomImportItemState(source({}), "importing")).toEqual({ + status: "importing", + stage: "processing", + progress: 30, + }); + expect( + deriveLoomImportItemState( + source({ uploadPhase: "uploading", uploadProgress: 0 }), + "importing", + ).stage, + ).toBe("starting"); + expect( + deriveLoomImportItemState( + source({ + uploadMessage: "Queued for Loom import processing...", + uploadProgress: 0, + }), + "importing", + ).stage, + ).toBe("waiting"); + expect( + deriveLoomImportItemState( + source({ uploadPhase: "generating_thumbnail", uploadProgress: 90 }), + "importing", + ).stage, + ).toBe("finishing"); + }); + + it("treats a finished upload as imported even if the job row lags behind", () => { + expect( + deriveLoomImportItemState(source({ uploadPhase: null }), "importing"), + ).toEqual({ + status: "imported", + }); + }); + + it("surfaces media errors, deleted Caps and retries started elsewhere", () => { + expect( + deriveLoomImportItemState( + source({ uploadPhase: "error", uploadError: "Source unavailable" }), + "importing", + ), + ).toEqual({ status: "failed", error: "Source unavailable" }); + expect( + deriveLoomImportItemState(source({ videoExists: false }), "completed"), + ).toEqual({ + status: "failed", + error: "The imported Cap was deleted.", + }); + expect( + deriveLoomImportItemState( + source({ status: "failed", error: "Old failure" }), + "completed", + ).status, + ).toBe("importing"); + }); + + it("shows a failed row as imported once a retry from the video page finishes", () => { + expect( + deriveLoomImportItemState( + source({ status: "failed", error: "Old failure", uploadPhase: null }), + "completed", + ), + ).toEqual({ status: "imported" }); + }); + + it("keeps finished imports imported while the Cap is edited later", () => { + expect( + deriveLoomImportItemState( + source({ status: "complete", uploadPhase: "processing" }), + "completed", + ), + ).toEqual({ status: "imported" }); + }); +}); + +describe("settleLoomImportItem", () => { + const settle = (overrides: Partial) => + settleLoomImportItem({ + status: "importing", + videoExists: true, + uploadPhase: "processing", + uploadError: null, + ...overrides, + }); + + it("completes rows whose upload finished, including ones that failed before", () => { + expect(settle({ uploadPhase: null })).toEqual({ + status: "complete", + error: null, + }); + expect(settle({ status: "failed", uploadPhase: null })).toEqual({ + status: "complete", + error: null, + }); + }); + + it("fails rows whose upload errored or whose Cap was deleted", () => { + expect( + settle({ uploadPhase: "error", uploadError: "Source unavailable" }), + ).toEqual({ status: "failed", error: "Source unavailable" }); + expect(settle({ uploadPhase: "error" })).toEqual({ + status: "failed", + error: "Loom import failed.", + }); + expect(settle({ status: "complete", videoExists: false })).toEqual({ + status: "failed", + error: "The imported Cap was deleted.", + }); + }); + + it("leaves rows alone while they are still copying or already done", () => { + expect(settle({})).toBeNull(); + expect(settle({ status: "failed", uploadPhase: "processing" })).toBeNull(); + expect(settle({ status: "complete", uploadPhase: "error" })).toBeNull(); + }); +}); + +describe("loomImportWaitingStatus", () => { + it("calls checked rows ready until the import starts", () => { + expect(loomImportWaitingStatus("checking")).toBe("ready"); + expect(loomImportWaitingStatus("awaiting_upgrade")).toBe("ready"); + expect(loomImportWaitingStatus("importing")).toBe("queued"); + expect(loomImportWaitingStatus("completed")).toBe("queued"); + }); +}); + +describe("progress", () => { + it("counts partial progress of videos that are mid-copy", () => { + const counts = countLoomImportItems([ + { status: "imported" }, + { status: "failed" }, + { status: "importing" }, + { status: "queued" }, + ]); + expect(counts).toMatchObject({ + imported: 1, + failed: 1, + importing: 1, + queued: 1, + total: 4, + }); + expect(loomImportProgress(counts, 50)).toBeCloseTo(0.625); + }); +}); + +describe("Loom lookups", () => { + it("maps Loom's video types to import outcomes", () => { + expect( + toLoomVideoLookup({ + __typename: "RegularUserVideo", + name: " Weekly update ", + createdAt: "2022-08-05T21:16:35.487Z", + thumbnails: { default: "https://cdn.loom.com/thumb.jpg" }, + video_properties: { duration: 154, width: 1152, height: 720 }, + }), + ).toEqual({ + status: "ok", + title: "Weekly update", + createdAt: "2022-08-05T21:16:35.487Z", + durationSeconds: 154, + width: 1152, + height: 720, + thumbnailUrl: "https://cdn.loom.com/thumb.jpg", + }); + expect(toLoomVideoLookup({ __typename: "PrivateVideo" })).toEqual({ + status: "private", + }); + expect( + toLoomVideoLookup({ __typename: "VideoPasswordMissingOrIncorrect" }), + ).toEqual({ + status: "password", + }); + expect(toLoomVideoLookup(null)).toEqual({ status: "not_found" }); + for (const unsafe of [ + "javascript:alert(1)", + "http://cdn.loom.com/thumb.jpg", + ]) { + expect( + toLoomVideoLookup({ + __typename: "RegularUserVideo", + thumbnails: { default: unsafe }, + }), + ).toMatchObject({ status: "ok", thumbnailUrl: null }); + } + }); + + it("asks for many videos per request with aliases", () => { + const query = buildLoomVideosQuery(["a", "b"]); + expect(query.query).toContain("v0: getVideo(id: $id0"); + expect(query.query).toContain("v1: getVideo(id: $id1"); + expect(query.variables).toEqual({ id0: "a", id1: "b" }); + }); + + it("batches lookups, retries a failed request and reports batches as they land", async () => { + const ids = Array.from({ length: 60 }, (_, index) => `id${index}`); + let calls = 0; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + calls++; + if (calls === 1) return new Response("busy", { status: 503 }); + const { variables } = JSON.parse(String(init?.body)) as { + variables: Record; + }; + const data = Object.fromEntries( + Object.keys(variables).map((key) => [ + key.replace("id", "v"), + { __typename: "RegularUserVideo", name: variables[key] }, + ]), + ); + return Response.json({ data }); + }) as unknown as typeof fetch; + const batches: number[] = []; + const results = await lookupLoomVideos([...ids, "id0"], { + fetchImpl, + concurrency: 2, + onBatch: (batch) => { + batches.push(batch.size); + }, + }); + expect(results.size).toBe(60); + expect(batches.sort()).toEqual([10, 25, 25]); + expect(fetchImpl).toHaveBeenCalledTimes(4); + expect(results.get("id42")).toMatchObject({ status: "ok", title: "id42" }); + }); + + it("gives up cleanly when Loom keeps failing", async () => { + const fetchImpl = vi.fn( + async () => new Response("no", { status: 500 }), + ) as unknown as typeof fetch; + const results = await lookupLoomVideos(["a"], { fetchImpl, attempts: 2 }); + expect(results.get("a")).toEqual({ status: "error" }); + }); +}); + +const view = (overrides: Partial): LoomImportItemView => ({ + id: "item-1", + row: 2, + url: "https://www.loom.com/share/abc", + loomId: "abc", + title: "Kickoff", + email: "ana@acme.com", + space: null, + status: "imported", + videoId: "video-1", + recordedAt: "2022-08-05T21:16:35.487Z", + duration: 154, + thumb: null, + v: 1, + ...overrides, +}); + +describe("results report", () => { + it("maps every Loom link to its new Cap link and neutralizes spreadsheet formulas", () => { + const csv = buildLoomImportReport( + [ + view({}), + view({ + id: "item-2", + row: 3, + title: '=HYPERLINK("x"), "quoted"', + status: "failed", + videoId: null, + error: "This Loom is private.", + }), + ], + "https://cap.so", + ); + expect(csv.split("\n")).toEqual([ + "row,loom_url,title,recorded_at,owner_email,space,status,cap_url,note", + "2,https://www.loom.com/share/abc,Kickoff,2022-08-05T21:16:35.487Z,ana@acme.com,,imported,https://cap.so/s/video-1,", + `3,https://www.loom.com/share/abc,"'=HYPERLINK(""x""), ""quoted""",2022-08-05T21:16:35.487Z,ana@acme.com,,failed,,This Loom is private.`, + "", + ]); + }); +}); + +describe("mergeLoomImportItems", () => { + const snapshot = ( + items: LoomImportItemView[], + full: boolean, + status: LoomImportJobStatus = "importing", + ) => ({ items, full, cursor: 0, job: { status } }) as LoomImportSnapshot; + + it("moves rows the poll skipped from ready to queued when the import starts", () => { + const map = new Map(); + const first = mergeLoomImportItems( + map, + [], + snapshot( + [ + view({ id: "a", row: 2, status: "ready" }), + view({ id: "b", row: 3, status: "ready" }), + view({ id: "c", row: 4, status: "failed" }), + ], + true, + "checking", + ), + "checking", + ); + const started = mergeLoomImportItems( + map, + first.order, + snapshot( + [view({ id: "a", row: 2, status: "importing", v: 2 })], + false, + "importing", + ), + "checking", + ); + expect(started.changed).toBe(true); + expect(first.order.map((id) => map.get(id)?.status)).toEqual([ + "importing", + "queued", + "failed", + ]); + + const steady = mergeLoomImportItems( + map, + first.order, + snapshot([], false, "importing"), + "importing", + ); + expect(steady.changed).toBe(false); + }); + + it("replaces only rows that changed and keeps CSV order", () => { + const map = new Map(); + const first = mergeLoomImportItems( + map, + [], + snapshot([view({ id: "a", row: 2 }), view({ id: "b", row: 3 })], true), + ); + expect(first).toEqual({ order: ["a", "b"], changed: true }); + const original = map.get("a"); + + const unchanged = mergeLoomImportItems( + map, + first.order, + snapshot([view({ id: "a", row: 2 })], false), + ); + expect(unchanged.changed).toBe(false); + expect(map.get("a")).toBe(original); + + const progressed = mergeLoomImportItems( + map, + first.order, + snapshot( + [view({ id: "b", row: 3, status: "importing", progress: 40, v: 2 })], + false, + ), + ); + expect(progressed).toEqual({ order: ["a", "b"], changed: true }); + expect(map.get("a")).toBe(original); + expect(map.get("b")?.progress).toBe(40); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import-ui.test.ts b/apps/web/__tests__/unit/loom-import-ui.test.ts index d92f8e90913..6aaff2fd410 100644 --- a/apps/web/__tests__/unit/loom-import-ui.test.ts +++ b/apps/web/__tests__/unit/loom-import-ui.test.ts @@ -71,6 +71,7 @@ const fireEvent = { const mocks = vi.hoisted(() => ({ folders: vi.fn(), import: vi.fn(), + createJob: vi.fn(), push: vi.fn(), refresh: vi.fn(), })); @@ -99,7 +100,9 @@ vi.mock("next/link", () => ({ vi.mock("@/actions/loom", () => ({ getLoomImportFolders: mocks.folders, importFromLoom: mocks.import, - importFromLoomCsv: vi.fn(), +})); +vi.mock("@/actions/loom-import", () => ({ + createLoomImportJobAction: mocks.createJob, })); vi.mock("@cap/utils", async () => await import("@cap/utils/helpers")); vi.mock("@cap/ui", async () => ({ @@ -110,6 +113,7 @@ vi.mock("@cap/ui", async () => ({ ...(await import("../../../../packages/ui/src/components/input/Input")), })); +import { File as NodeFile } from "node:buffer"; import { Folder, Space } from "@cap/web-domain"; import { ImportLoomPage } from "@/app/(org)/dashboard/import/loom/ImportLoomPage"; import type { LoomImportDestination } from "@/lib/loom-import-destination"; @@ -123,6 +127,16 @@ beforeEach(() => { HTMLElement.prototype.hasPointerCapture = () => false; HTMLElement.prototype.setPointerCapture = vi.fn(); HTMLElement.prototype.releasePointerCapture = vi.fn(); + window.matchMedia = vi.fn((query: string) => ({ + matches: false, + media: query, + onchange: null, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + addListener: vi.fn(), + removeListener: vi.fn(), + dispatchEvent: () => false, + })); mocks.folders.mockResolvedValue([ { id: "parent", name: "Course", parentId: null }, { id: "child", name: "Live Calls - Two", parentId: "parent" }, @@ -280,4 +294,55 @@ describe("Loom importer component", () => { "/dashboard/spaces/space/folder/child", ); }); + it("imports every video from an uploaded file of bare Loom ids", async () => { + const first = "0dd0a01e10c742b28dbea75082c08635"; + const second = "31f430c1a1e744b8a7b6c18a26982c71"; + mocks.createJob.mockResolvedValue({ ok: true, jobId: "job-1" }); + await render(); + await act(async () => { + fireEvent.click(getByRole(container, "tab", { name: "Bulk Import" })); + }); + const input = container.querySelector( + "[data-testid=loom-csv-input]", + ); + if (!input) throw new Error("Missing CSV input"); + Object.defineProperty(input, "files", { + configurable: true, + value: [ + new NodeFile([`${first}\n${second}\n`], "loom-ids.csv", { + type: "text/csv", + }), + ], + }); + await act(async () => { + input.dispatchEvent(new Event("change", { bubbles: true })); + }); + await waitFor(() => + expect( + getByRole(container, "button", { name: "Import 2 videos" }), + ).toBeTruthy(), + ); + expect(container.textContent).toContain("loom-ids.csv"); + await act(async () => { + fireEvent.click( + getByRole(container, "button", { name: "Import 2 videos" }), + ); + }); + await waitFor(() => expect(mocks.createJob).toHaveBeenCalledTimes(1)); + expect(mocks.createJob).toHaveBeenCalledWith({ + orgId: "org", + fileName: "loom-ids.csv", + rows: { + owners: [], + spaces: [], + rows: [ + [1, first, -1, -1], + [2, second, -1, -1], + ], + }, + }); + await waitFor(() => + expect(mocks.push).toHaveBeenCalledWith("/dashboard/import/loom/job-1"), + ); + }); }); diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index a8068f6e3b3..703354e9802 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -706,454 +706,4 @@ describe("importFromLoom", () => { ]) expect(conditions).toContain(required); }); - - it("rejects a CSV import when the current user is not an organization admin or owner", async () => { - getOrganizationAccessMock.mockResolvedValueOnce({ - id: "org-1", - ownerId: "owner-456", - memberId: "member-row", - role: "member", - }); - - const fetchMock = vi.mocked(fetch); - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: [ - { - rowNumber: 2, - loomUrl: "https://www.loom.com/share/loom-abc1234567", - userEmail: "member@example.com", - }, - ], - }); - - expect(result).toEqual({ - success: false, - importedCount: 0, - failedCount: 0, - results: [], - error: - "Only organization admins and owners can import Loom videos from a CSV.", - }); - expect(getOrganizationAccessMock).toHaveBeenCalledWith("user-123", "org-1"); - expect(fetchMock).not.toHaveBeenCalled(); - expect(valuesMock).not.toHaveBeenCalled(); - }); - - it("provisions missing CSV users and starts imports for them", async () => { - whereMock - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([{ ownerId: "owner-456" }])) - .mockReturnValueOnce( - withLimit([{ inviteQuota: 1, stripeSubscriptionId: null }]), - ) - .mockResolvedValueOnce([]); - - const fetchMock = vi.mocked(fetch); - fetchMock.mockImplementation(async (input) => { - const url = typeof input === "string" ? input : input.toString(); - - if (url.includes("/transcoded-url")) { - return { - ok: true, - status: 200, - text: async () => - JSON.stringify({ url: "https://cdn.loom.com/video.mp4" }), - } as Response; - } - - if (url === "https://www.loom.com/graphql") { - return { - ok: true, - json: async () => ({ - data: { getVideo: { name: "Imported video" } }, - }), - } as Response; - } - - if (url.includes("/v1/oembed")) { - return { - ok: true, - json: async () => ({ duration: 42, width: 1920, height: 1080 }), - } as Response; - } - - throw new Error(`Unexpected fetch: ${url}`); - }); - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: [ - { - rowNumber: 2, - loomUrl: "https://www.loom.com/share/loom-abc1234567", - userEmail: "not-member@example.com", - }, - ], - }); - - expect(result).toEqual({ - success: true, - importedCount: 1, - failedCount: 0, - results: [ - { - rowNumber: 2, - userEmail: "not-member@example.com", - spaceName: undefined, - success: true, - videoId: "video-123", - error: undefined, - }, - ], - error: undefined, - }); - expect(fetchMock).toHaveBeenCalled(); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - email: "not-member@example.com", - activeOrganizationId: "org-1", - defaultOrgId: "org-1", - }), - ); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - organizationId: "org-1", - invitedEmail: "not-member@example.com", - invitedByUserId: "user-123", - role: "member", - }), - ); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - organizationId: "org-1", - userId: "video-123", - role: "member", - }), - ); - expect(storageGetWritableAccessForUserMock).toHaveBeenCalledWith( - "video-123", - "org-1", - ); - }); - - it("imports a CSV row without calling the Vercel firewall", async () => { - whereMock.mockImplementation((conditions: unknown) => { - const serializedConditions = JSON.stringify(conditions); - - if (serializedConditions.includes("sourceId")) { - return Promise.resolve( - serializedConditions.includes("loom-existing123") - ? [{ videoId: "existing-video" }] - : [], - ); - } - - return withLimit([{ userId: "member-123", email: "member@example.com" }]); - }); - - const fetchMock = vi.mocked(fetch); - fetchMock.mockImplementation(async (input) => { - const url = typeof input === "string" ? input : input.toString(); - - if (url.includes("/transcoded-url")) { - return { - ok: true, - status: 200, - text: async () => - JSON.stringify({ url: "https://cdn.loom.com/video.mp4" }), - } as Response; - } - - if (url === "https://www.loom.com/graphql") { - return { - ok: true, - json: async () => ({ - data: { getVideo: { name: "Imported video" } }, - }), - } as Response; - } - - if (url.includes("/v1/oembed")) { - return { - ok: true, - json: async () => ({ duration: 42, width: 1920, height: 1080 }), - } as Response; - } - - throw new Error(`Unexpected fetch: ${url}`); - }); - - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: [ - { - rowNumber: 2, - loomUrl: "https://www.loom.com/share/loom-existing123", - userEmail: "member@example.com", - }, - { - rowNumber: 3, - loomUrl: "https://www.loom.com/share/loom-newvideo123", - userEmail: "member@example.com", - }, - ], - }); - - expect(result).toEqual({ - success: true, - importedCount: 1, - failedCount: 1, - results: [ - { - rowNumber: 2, - userEmail: "member@example.com", - spaceName: undefined, - success: false, - error: "This Loom video has already been imported.", - }, - { - rowNumber: 3, - userEmail: "member@example.com", - spaceName: undefined, - success: true, - videoId: "video-123", - error: undefined, - }, - ], - error: undefined, - }); - expect(checkRateLimitMock).not.toHaveBeenCalled(); - expect(fetchMock).toHaveBeenCalled(); - expect(startMock).toHaveBeenCalledTimes(1); - }); - - it("limits CSV imports to 500 rows", async () => { - const fetchMock = vi.mocked(fetch); - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: Array.from({ length: 501 }, (_, index) => ({ - rowNumber: index + 2, - loomUrl: `https://www.loom.com/share/loom-${String(index).padStart(10, "0")}`, - userEmail: "member@example.com", - })), - }); - - expect(result).toEqual({ - success: false, - importedCount: 0, - failedCount: 501, - results: [], - error: - "CSV imports are limited to 500 rows at a time. Contact support to raise this limit.", - }); - expect(checkRateLimitMock).not.toHaveBeenCalled(); - expect(fetchMock).not.toHaveBeenCalled(); - expect(valuesMock).not.toHaveBeenCalled(); - }); - - it("starts CSV Loom imports for matched organization members when the current user is an organization admin", async () => { - getOrganizationAccessMock.mockResolvedValueOnce({ - id: "org-1", - ownerId: "owner-456", - memberId: "member-row", - role: "admin", - }); - whereMock - .mockReturnValueOnce( - withLimit([{ userId: "member-123", email: "member@example.com" }]), - ) - .mockResolvedValueOnce([]); - - const fetchMock = vi.mocked(fetch); - fetchMock.mockImplementation(async (input) => { - const url = typeof input === "string" ? input : input.toString(); - - if (url.includes("/transcoded-url")) { - return { - ok: true, - status: 200, - text: async () => - JSON.stringify({ url: "https://cdn.loom.com/video.mp4" }), - } as Response; - } - - if (url === "https://www.loom.com/graphql") { - return { - ok: true, - json: async () => ({ - data: { getVideo: { name: "Imported video" } }, - }), - } as Response; - } - - if (url.includes("/v1/oembed")) { - return { - ok: true, - json: async () => ({ duration: 42, width: 1920, height: 1080 }), - } as Response; - } - - throw new Error(`Unexpected fetch: ${url}`); - }); - - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: [ - { - rowNumber: 2, - loomUrl: "https://www.loom.com/share/loom-abc1234567", - userEmail: "MEMBER@example.com", - }, - ], - }); - - expect(result).toEqual({ - success: true, - importedCount: 1, - failedCount: 0, - results: [ - { - rowNumber: 2, - userEmail: "member@example.com", - spaceName: undefined, - success: true, - videoId: "video-123", - error: undefined, - }, - ], - error: undefined, - }); - expect(storageGetWritableAccessForUserMock).toHaveBeenCalledWith( - "member-123", - "org-1", - ); - expect(valuesMock).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - ownerId: "member-123", - }), - ); - expect(startMock).toHaveBeenCalledWith( - expect.anything(), - expect.arrayContaining([ - expect.objectContaining({ - userId: "member-123", - rawFileKey: "member-123/video-123/raw-upload.mp4", - }), - ]), - ); - }); - - it("creates missing spaces and adds CSV Loom imports to them", async () => { - whereMock - .mockReturnValueOnce( - withLimit([{ userId: "member-123", email: "member@example.com" }]), - ) - .mockResolvedValueOnce([]) - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([])) - .mockReturnValueOnce(withLimit([])); - - const fetchMock = vi.mocked(fetch); - fetchMock.mockImplementation(async (input) => { - const url = typeof input === "string" ? input : input.toString(); - - if (url.includes("/transcoded-url")) { - return { - ok: true, - status: 200, - text: async () => - JSON.stringify({ url: "https://cdn.loom.com/video.mp4" }), - } as Response; - } - - if (url === "https://www.loom.com/graphql") { - return { - ok: true, - json: async () => ({ - data: { getVideo: { name: "Imported video" } }, - }), - } as Response; - } - - if (url.includes("/v1/oembed")) { - return { - ok: true, - json: async () => ({ duration: 42, width: 1920, height: 1080 }), - } as Response; - } - - throw new Error(`Unexpected fetch: ${url}`); - }); - - const { importFromLoomCsv } = await import("@/actions/loom"); - - const result = await importFromLoomCsv({ - orgId: "org-1" as never, - rows: [ - { - rowNumber: 2, - loomUrl: "https://www.loom.com/share/loom-abc1234567", - userEmail: "member@example.com", - spaceName: " Sales Team ", - }, - ], - }); - - expect(result).toEqual({ - success: true, - importedCount: 1, - failedCount: 0, - results: [ - { - rowNumber: 2, - userEmail: "member@example.com", - spaceName: "Sales Team", - success: true, - videoId: "video-123", - error: undefined, - }, - ], - error: undefined, - }); - expect(mockDb.transaction).toHaveBeenCalledTimes(2); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - name: "Sales Team", - organizationId: "org-1", - createdById: "user-123", - }), - ); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - spaceId: "video-123", - userId: "user-123", - role: "admin", - }), - ); - expect(valuesMock).toHaveBeenCalledWith( - expect.objectContaining({ - spaceId: "video-123", - videoId: "video-123", - addedById: "user-123", - }), - ); - expect(revalidatePathMock).toHaveBeenCalledWith( - "/dashboard/spaces/video-123", - ); - expect(revalidatePathMock).toHaveBeenCalledWith("/dashboard"); - }); }); diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index 57deff5af8c..cb8076bf87c 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -12,6 +12,12 @@ const mocks = vi.hoisted(() => ({ remove: vi.fn(), fetch: vi.fn(), sleep: vi.fn(), + continueLoomImport: vi.fn(), + observe: vi.fn(), +})); + +vi.mock("@/lib/desktop-recording-job-status", () => ({ + observeDesktopRecordingJob: mocks.observe, })); vi.mock("@cap/database", () => ({ @@ -56,12 +62,16 @@ vi.mock("@/lib/video-storage", () => ({ decodeStorageVideo: (value: unknown) => value, })); vi.mock("@/lib/transcribe", () => ({ transcribeVideo: vi.fn() })); +vi.mock("@/lib/loom-import/dispatch", () => ({ + dispatchLoomImportForVideo: mocks.continueLoomImport, +})); vi.mock("@/lib/ai-generation-entitlement", () => ({ isAiGenerationEnabledForUser: () => false, })); vi.mock("workflow", () => ({ sleep: mocks.sleep, FatalError: class FatalError extends Error {}, + RetryableError: class RetryableError extends Error {}, })); import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; @@ -111,6 +121,10 @@ describe("media processing workflows", () => { ); mocks.remove.mockImplementation(() => Effect.void); mocks.sleep.mockResolvedValue(undefined); + mocks.continueLoomImport.mockReset().mockResolvedValue(null); + mocks.observe + .mockReset() + .mockResolvedValue({ status: "unavailable", delivered: false }); vi.stubGlobal("fetch", mocks.fetch); mocks.fetch.mockReset().mockImplementation(async (url: string) => { if (url.startsWith("https://www.loom.com/")) { @@ -198,6 +212,61 @@ describe("media processing workflows", () => { priority: "bulk", }); expect(mocks.remove).not.toHaveBeenCalled(); + expect(mocks.continueLoomImport.mock.calls).toEqual([["video"], ["video"]]); + }); + + it("waits for Loom instead of failing when Loom rate limits the download link", async () => { + let loomCalls = 0; + mocks.fetch.mockImplementation(async (url: string) => { + if (url.startsWith("https://www.loom.com/")) { + loomCalls++; + return loomCalls <= 4 + ? new Response("slow down", { status: 429 }) + : Response.json({ url: "https://cdn.loom.com/original.mp4" }); + } + return Response.json({ jobId: "job-1", status: "queued" }); + }); + mocks.rows = [[video], [pending], [], [metadata]]; + await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({ + success: true, + }); + expect(loomCalls).toBe(5); + expect(mocks.sleep.mock.calls[0]).toEqual(["60s"]); + expect(mocks.write).toHaveBeenCalled(); + }); + + it("gives a Loom video up after Loom stays unavailable for a while", async () => { + mocks.fetch.mockImplementation(async (url: string) => + url.startsWith("https://www.loom.com/") + ? new Response("unavailable", { status: 503 }) + : Response.json({ jobId: "job-1" }), + ); + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow( + "Loom is not responding right now", + ); + expect(mocks.sleep.mock.calls).toEqual([ + ["60s"], + ["120s"], + ["180s"], + ["240s"], + ["300s"], + ["360s"], + ]); + expect(mocks.continueLoomImport).toHaveBeenCalledExactlyOnceWith("video"); + }); + + it("frees its import slot when a Loom video fails for good", async () => { + mocks.continueLoomImport.mockRejectedValueOnce(new Error("Database busy")); + vi.spyOn(console, "error").mockImplementation(() => undefined); + mocks.fetch.mockImplementation(async (url: string) => + url.startsWith("https://www.loom.com/") + ? new Response(null, { status: 404 }) + : Response.json({ jobId: "job-1" }), + ); + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow( + "Could not retrieve a download URL from Loom", + ); + expect(mocks.continueLoomImport).toHaveBeenCalledExactlyOnceWith("video"); }); it("reuses a preserved Loom upload when retrying", async () => { @@ -234,6 +303,76 @@ describe("media processing workflows", () => { ); }); + it("restarts a Loom video from its preserved original when the worker goes quiet", async () => { + let now = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + mocks.sleep.mockImplementation(async (delay: unknown) => { + if (typeof delay === "number") now += delay; + }); + const quiet = { ...pending, updatedAt: new Date(0) }; + try { + mocks.rows = [ + [video], + ...Array.from({ length: 43 }, () => [quiet]), + [video], + [video], + [], + [metadata], + ]; + await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({ + success: true, + }); + expect(now).toBeGreaterThan(30 * 60 * 1000); + expect(mocks.observe).toHaveBeenCalledTimes(21); + for (const [lookup] of mocks.observe.mock.calls) { + expect(lookup).toMatchObject({ videoId: "video", jobId: "job-1" }); + } + expect(mocks.fetch.mock.calls.map(([url]) => url)).toEqual([ + "https://www.loom.com/api/campaigns/sessions/loom-video/transcoded-url", + "https://worker.example.com/video/import", + "https://worker.example.com/video/process", + ]); + expect(mocks.get).toHaveBeenCalledWith( + payload.rawFileKey, + expect.anything(), + ); + } finally { + clock.mockRestore(); + } + }); + + it("does not start a second copy while the media server still has the Loom video", async () => { + let now = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + mocks.sleep.mockImplementation(async (delay: unknown) => { + if (typeof delay === "number") now += delay; + }); + mocks.observe + .mockResolvedValueOnce({ status: "active", delivered: false }) + .mockResolvedValueOnce({ status: "active", delivered: false }) + .mockResolvedValueOnce({ status: "terminal", delivered: true }); + const quiet = { ...pending, updatedAt: new Date(0) }; + try { + mocks.rows = [ + [video], + ...Array.from({ length: 25 }, () => [quiet]), + [], + [metadata], + ]; + await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({ + success: true, + metadata, + }); + expect(mocks.observe).toHaveBeenCalledTimes(3); + expect(mocks.fetch.mock.calls.map(([url]) => url)).toEqual([ + "https://www.loom.com/api/campaigns/sessions/loom-video/transcoded-url", + "https://worker.example.com/video/import", + ]); + } finally { + clock.mockRestore(); + } + }); + it("supports Loom streaming fallbacks without reading a nonexistent raw object", async () => { mocks.fetch.mockImplementation(async (url: string) => Response.json( diff --git a/apps/web/__tests__/unit/media-server-progress.test.ts b/apps/web/__tests__/unit/media-server-progress.test.ts index f4e817178a3..50d2ce7fb0d 100644 --- a/apps/web/__tests__/unit/media-server-progress.test.ts +++ b/apps/web/__tests__/unit/media-server-progress.test.ts @@ -15,6 +15,8 @@ const mocks = vi.hoisted(() => ({ transcribe: vi.fn(), invalidateQuota: vi.fn(), audio: vi.fn(), + isLoomImport: vi.fn(), + continueLoomImport: vi.fn(), tables: { videos: { id: "videos.id", @@ -68,6 +70,10 @@ vi.mock("@/lib/google-drive-storage-quota", () => ({ vi.mock("@/lib/google-drive-storage-quota-cache", () => ({ invalidateGoogleDriveStorageQuotaCache: mocks.invalidateQuota, })); +vi.mock("@/lib/loom-import/dispatch", () => ({ + isLoomImportedVideo: mocks.isLoomImport, + dispatchLoomImportForVideo: mocks.continueLoomImport, +})); vi.mock("@/lib/queue-video-transcription", () => ({ queueVideoTranscription: mocks.transcribe, shouldQueueTranscriptionAfterMediaComplete: () => true, @@ -253,6 +259,17 @@ function databaseFixture(initial: DesktopRecordingJob | null = fixture().job) { }; } +function webPayload(overrides: Record = {}) { + return { + jobId: "media-job-1", + videoId, + phase: "complete", + progress: 100, + metadata: { ...metadata, bitrate: 1_000_000 }, + ...overrides, + }; +} + function request( body: unknown = fixture().payload, secret: string | null = "media-secret", @@ -283,6 +300,8 @@ describe("media-server recording progress webhook", () => { ); mocks.queue.mockReset().mockResolvedValue("queued"); mocks.transcribe.mockReset().mockResolvedValue({ success: true }); + mocks.isLoomImport.mockReset().mockResolvedValue(false); + mocks.continueLoomImport.mockReset().mockResolvedValue(null); mocks.invalidateQuota.mockResolvedValue(undefined); mocks.retry.mockResolvedValue(true); mocks.blocked.mockResolvedValue(true); @@ -331,6 +350,37 @@ describe("media-server recording progress webhook", () => { }, ); + it("leaves AI for imported Loom videos until they are viewed and keeps the import moving", async () => { + const database = databaseFixture(null); + database.video.source = { type: "webMP4" }; + mocks.isLoomImport.mockResolvedValue(true); + expect((await request(webPayload())).status).toBe(200); + expect(mocks.isLoomImport).toHaveBeenCalledExactlyOnceWith(videoId); + expect(mocks.transcribe).not.toHaveBeenCalled(); + expect(mocks.continueLoomImport).toHaveBeenCalledExactlyOnceWith(videoId); + expect(database.mutations).toContainEqual({ + operation: "delete", + table: mocks.tables.uploads, + }); + }); + + it("still transcribes ordinary web uploads as soon as processing finishes", async () => { + const database = databaseFixture(null); + database.video.source = { type: "webMP4" }; + expect((await request(webPayload())).status).toBe(200); + expect(mocks.transcribe).toHaveBeenCalledExactlyOnceWith(videoId); + expect(mocks.continueLoomImport).not.toHaveBeenCalled(); + }); + + it("keeps a Loom import moving when a video fails for good", async () => { + const database = databaseFixture(null); + database.video.source = { type: "webMP4" }; + const failed = webPayload({ phase: "error", error: "Source unavailable" }); + expect((await request(failed)).status).toBe(200); + expect(mocks.continueLoomImport).toHaveBeenCalledExactlyOnceWith(videoId); + expect(mocks.transcribe).not.toHaveBeenCalled(); + }); + it("refuses callbacks when the webhook secret is not configured", async () => { databaseFixture(); mocks.secret = undefined; diff --git a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts index dd1de83dfdd..d362bada4e5 100644 --- a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts +++ b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts @@ -188,6 +188,37 @@ describe("recoverStalledVideoPipeline", () => { expect(result.ai.statuses).toEqual({ started: 1 }); }); + it("leaves never-watched Loom imports for their first view instead of transcribing them", async () => { + const chains: ReturnType[] = []; + mockDb.mockImplementation(() => { + const chain = makeSelectChain([]); + chains.push(chain); + return chain; + }); + + const { recoverStalledVideoPipeline } = await import( + "@/lib/video-pipeline-recovery" + ); + await recoverStalledVideoPipeline({ now, concurrency: 1 }); + + const transcriptionQuery = chains[1]; + expect(transcriptionQuery?.leftJoin).toHaveBeenCalledWith( + expect.objectContaining({ id: "importedVideos.id" }), + expect.anything(), + ); + const conditions = JSON.stringify(transcriptionQuery?.where.mock.calls[0]); + expect(conditions).toContain( + JSON.stringify(["videos.transcriptionStatus", "importedVideos.id"]), + ); + expect(conditions).toContain( + JSON.stringify({ + left: "videos.transcriptionStatus", + right: "PROCESSING", + }), + ); + expect(mockTranscribeVideo).not.toHaveBeenCalled(); + }); + it("does not start a duplicate media workflow after a lost claim", async () => { const mediaCandidate = { videoId: "video-media", diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index e55f634d71c..cb60278cc5e 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mockDb = vi.fn(); const mockStart = vi.fn(); const mockSetVideoProcessingError = vi.fn(); +const mockMarkLoomImportRetrying = vi.fn(); vi.mock("@cap/database", () => ({ db: mockDb, @@ -53,6 +54,10 @@ vi.mock("@/lib/video-processing", () => ({ setVideoProcessingError: mockSetVideoProcessingError, })); +vi.mock("@/lib/loom-import/retry", () => ({ + markLoomImportRetrying: mockMarkLoomImportRetrying, +})); + vi.mock("@/workflows/process-video", () => ({ processVideoWorkflow: vi.fn(), })); @@ -198,6 +203,7 @@ describe("recoverFailedVideoProcessing", () => { }), ]); expect(result.statuses).toEqual({ started: 1 }); + expect(mockMarkLoomImportRetrying).toHaveBeenCalledWith("video-1"); }); it("keeps a failed Loom workflow start eligible for Loom-specific recovery", async () => { diff --git a/apps/web/__tests__/unit/video-processing-status.test.ts b/apps/web/__tests__/unit/video-processing-status.test.ts index 979821b2ad1..f7fe58103f1 100644 --- a/apps/web/__tests__/unit/video-processing-status.test.ts +++ b/apps/web/__tests__/unit/video-processing-status.test.ts @@ -3,13 +3,27 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ rows: [] as unknown[][], where: vi.fn(), + write: vi.fn(), sleep: vi.fn(), + observe: vi.fn(), now: 0, })); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + MEDIA_SERVER_URL: "https://worker.example.com", + WEB_URL: "https://cap.example.com", + MEDIA_SERVER_WEBHOOK_SECRET: "test-secret", + }), +})); +vi.mock("@/lib/desktop-recording-job-status", () => ({ + observeDesktopRecordingJob: mocks.observe, +})); + vi.mock("@cap/database", () => ({ db: () => ({ select: () => ({ from: () => ({ where: mocks.where }) }), + update: () => ({ set: () => ({ where: mocks.write }) }), }), })); vi.mock("@cap/database/schema", () => ({ @@ -26,6 +40,9 @@ vi.mock("workflow", () => ({ import { readVideoProcessingStatus, + UNSEEN_JOB_GRACE_MS, + VIDEO_PROCESSING_STALL_MS, + VideoProcessingFailedError, waitForVideoProcessing, } from "@/workflows/video-processing-status"; @@ -46,6 +63,10 @@ describe("durable video processing completion", () => { mocks.sleep.mockReset().mockImplementation(async (delay: number) => { mocks.now += delay; }); + mocks.observe + .mockReset() + .mockResolvedValue({ status: "unavailable", delivered: false }); + mocks.write.mockReset().mockResolvedValue(undefined); }); afterEach(() => vi.restoreAllMocks()); @@ -96,15 +117,131 @@ describe("durable video processing completion", () => { }); }); - it("bounds waiting when a worker never completes", async () => { - mocks.where.mockResolvedValue([pending]); - await expect(waitForVideoProcessing("video")).rejects.toThrow( - "Video processing timed out while processing 25% Processing video...", + it("keeps waiting on a long recording for as long as the worker reports in", async () => { + const fourHours = 4 * 60 * 60 * 1000; + const heartbeatMs = 15 * 60 * 1000; + mocks.rows = [[], [metadata]]; + mocks.where.mockImplementation(async () => + mocks.now < fourHours + ? [ + { + ...pending, + updatedAt: new Date( + Math.floor(mocks.now / heartbeatMs) * heartbeatMs, + ), + }, + ] + : mocks.rows.shift(), + ); + await expect( + waitForVideoProcessing("video", { maxPollMs: 2 * 60 * 1000 }), + ).resolves.toEqual(metadata); + expect(mocks.now).toBeGreaterThanOrEqual(fourHours); + expect(mocks.sleep.mock.calls.length).toBeLessThan(150); + }); + + it("gives up on processing that stops changing, however short the video", async () => { + mocks.where.mockResolvedValue([{ ...pending, updatedAt: new Date(0) }]); + const waiting = waitForVideoProcessing("video"); + await expect(waiting).rejects.toBeInstanceOf(VideoProcessingFailedError); + await expect(waiting).rejects.toThrow( + "Video processing stopped making progress while processing 25% Processing video...", ); - expect(mocks.now).toBeLessThanOrEqual(60 * 60 * 1000 + 30_000); - expect(mocks.sleep.mock.calls.length).toBeLessThan(130); + expect(mocks.now).toBeGreaterThan(VIDEO_PROCESSING_STALL_MS); + expect(mocks.now).toBeLessThanOrEqual(VIDEO_PROCESSING_STALL_MS + 30_000); expect(Math.max(...mocks.sleep.mock.calls.map(([delay]) => delay))).toBe( 30_000, ); + expect(mocks.observe).not.toHaveBeenCalled(); + }); + + it("restarts only after the media server has not seen the job for ten minutes of checks", async () => { + mocks.where.mockResolvedValue([{ ...pending, updatedAt: new Date(0) }]); + await expect( + waitForVideoProcessing("video", { jobId: "job-1" }), + ).rejects.toBeInstanceOf(VideoProcessingFailedError); + expect(mocks.now).toBeGreaterThanOrEqual( + VIDEO_PROCESSING_STALL_MS + UNSEEN_JOB_GRACE_MS, + ); + expect(mocks.observe.mock.calls.length).toBeGreaterThanOrEqual(20); + for (const [lookup] of mocks.observe.mock.calls) { + expect(lookup).toEqual({ + videoId: "video", + jobId: "job-1", + mediaServerUrl: "https://worker.example.com", + webhookUrl: + "https://cap.example.com/api/webhooks/media-server/progress?retryable=true", + secret: "test-secret", + }); + } + expect(mocks.write).not.toHaveBeenCalled(); + }); + + it("does not start another copy when status checks fail for a few minutes", async () => { + const stuck = { ...pending, updatedAt: new Date(0) }; + let checks = 0; + mocks.observe.mockImplementation(async () => { + checks++; + if (checks <= 8) return { status: "unavailable", delivered: false }; + if (checks <= 12) return { status: "active", delivered: false }; + return { status: "terminal", delivered: true }; + }); + mocks.rows = [[], [metadata]]; + mocks.where.mockImplementation(async () => + checks < 13 ? [stuck] : mocks.rows.shift(), + ); + await expect( + waitForVideoProcessing("video", { jobId: "job-1" }), + ).resolves.toEqual(metadata); + expect(checks).toBe(13); + expect(mocks.write).toHaveBeenCalledTimes(4); + }); + + it("keeps waiting without a second copy while the media server still runs the job", async () => { + const stuck = { ...pending, updatedAt: new Date(0) }; + let finishedAt = Number.POSITIVE_INFINITY; + mocks.observe.mockImplementation(async () => { + if (mocks.now > 90 * 60 * 1000) { + finishedAt = mocks.now; + return { status: "terminal", delivered: true }; + } + return { status: "active", delivered: false }; + }); + mocks.rows = [[], [metadata]]; + mocks.where.mockImplementation(async () => + mocks.now <= finishedAt ? [stuck] : mocks.rows.shift(), + ); + await expect( + waitForVideoProcessing("video", { jobId: "job-1" }), + ).resolves.toEqual(metadata); + expect(finishedAt).toBeGreaterThan(90 * 60 * 1000); + const checks = mocks.observe.mock.calls.length; + expect(checks).toBeGreaterThan(100); + expect(checks).toBeLessThan(mocks.where.mock.calls.length); + expect(mocks.write).toHaveBeenCalledTimes(checks - 1); + }); + + it("goes back to quiet waiting once progress arrives again", async () => { + let progress = 25; + mocks.observe.mockImplementation(async () => { + progress++; + return { status: "active", delivered: false }; + }); + mocks.rows = [[], [metadata]]; + mocks.where.mockImplementation(async () => + mocks.now < 3 * 60 * 60 * 1000 + ? [ + { + ...pending, + processingProgress: progress, + updatedAt: new Date(0), + }, + ] + : mocks.rows.shift(), + ); + await expect( + waitForVideoProcessing("video", { jobId: "job-1" }), + ).resolves.toEqual(metadata); + expect(mocks.observe.mock.calls.length).toBeLessThanOrEqual(9); }); }); diff --git a/apps/web/__tests__/unit/workflow-runtime-boundary.test.ts b/apps/web/__tests__/unit/workflow-runtime-boundary.test.ts index cc9afdcdb00..dbcbfdddd2c 100644 --- a/apps/web/__tests__/unit/workflow-runtime-boundary.test.ts +++ b/apps/web/__tests__/unit/workflow-runtime-boundary.test.ts @@ -11,6 +11,7 @@ const workflowEntries = [ "workflows/admin-reprocess-video.ts", "workflows/import-loom-video.ts", "workflows/agent-cap-operation.ts", + "workflows/loom-import-job.ts", ]; const forbiddenModules = new Set([ diff --git a/apps/web/actions/loom-import.ts b/apps/web/actions/loom-import.ts new file mode 100644 index 00000000000..f326d6b925d --- /dev/null +++ b/apps/web/actions/loom-import.ts @@ -0,0 +1,125 @@ +"use server"; + +import { getCurrentUser } from "@cap/database/auth/session"; +import { userIsPro } from "@cap/utils"; +import type { Organisation } from "@cap/web-domain"; +import { start } from "workflow/api"; +import { + cancelLoomImportJob, + createLoomImportJob, + getLoomImportJobForUser, + LoomImportError, + markLoomImportJobStarting, + resetFailedLoomImportItems, + revertLoomImportJobStart, +} from "@/lib/loom-import/jobs"; +import { loomImportJobWorkflow } from "@/workflows/loom-import-job"; + +type ActionResult> = + | ({ ok: true } & T) + | { ok: false; error: string }; + +const GENERIC_ERROR = "Something went wrong. Please try again."; + +async function startJobWorkflow(jobId: string) { + for (let attempt = 0; attempt < 2; attempt++) { + try { + await start(loomImportJobWorkflow, [{ jobId }]); + return true; + } catch (error) { + console.error("[loom-import] Could not start import workflow", { + jobId, + attempt, + error, + }); + } + } + return false; +} + +export async function createLoomImportJobAction({ + orgId, + fileName, + rows, +}: { + orgId: Organisation.OrganisationId; + fileName: string; + rows: unknown; +}): Promise> { + const user = await getCurrentUser(); + if (!user) return { ok: false, error: "Please sign in again." }; + + let jobId: string; + try { + ({ jobId } = await createLoomImportJob({ + userId: user.id, + orgId, + fileName: typeof fileName === "string" ? fileName : "", + rows, + })); + } catch (error) { + if (error instanceof LoomImportError) + return { ok: false, error: error.message }; + console.error("[loom-import] Could not create import", error); + return { ok: false, error: GENERIC_ERROR }; + } + + await startJobWorkflow(jobId); + return { ok: true, jobId }; +} + +export async function startLoomImportJobAction( + jobId: string, +): Promise> { + const user = await getCurrentUser(); + if (!user) return { ok: false, error: "Please sign in again." }; + const found = await getLoomImportJobForUser(jobId, user.id); + if (!found) return { ok: false, error: "This import doesn't exist." }; + if (found.job.createdById !== user.id) { + return { + ok: false, + error: "Only the person who uploaded this CSV can start it.", + }; + } + if (!userIsPro(user)) { + return { ok: false, error: "Importing from Loom needs Cap Pro." }; + } + + if (!(await markLoomImportJobStarting(jobId))) { + return { ok: true, started: false }; + } + if (!(await startJobWorkflow(jobId))) { + await revertLoomImportJobStart(jobId); + return { ok: false, error: "The import couldn't start. Please try again." }; + } + return { ok: true, started: true }; +} + +export async function cancelLoomImportJobAction( + jobId: string, +): Promise { + const user = await getCurrentUser(); + if (!user) return { ok: false, error: "Please sign in again." }; + const found = await getLoomImportJobForUser(jobId, user.id); + if (!found) return { ok: false, error: "This import doesn't exist." }; + await cancelLoomImportJob(jobId); + return { ok: true }; +} + +export async function retryLoomImportJobAction( + jobId: string, +): Promise> { + const user = await getCurrentUser(); + if (!user) return { ok: false, error: "Please sign in again." }; + const found = await getLoomImportJobForUser(jobId, user.id); + if (!found) return { ok: false, error: "This import doesn't exist." }; + if ( + found.job.status === "cancelled" || + found.job.status === "awaiting_upgrade" + ) { + return { ok: false, error: "This import can't be retried." }; + } + const retried = await resetFailedLoomImportItems(jobId); + if (retried > 0) await startJobWorkflow(jobId); + return { ok: true, retried }; +} diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts index 80386e189c7..8e4e46d9d83 100644 --- a/apps/web/actions/loom.ts +++ b/apps/web/actions/loom.ts @@ -7,12 +7,8 @@ import { nanoId } from "@cap/database/helpers"; import { folders, importedVideos, - organizationMembers, sharedVideos, - spaceMembers, - spaces, spaceVideos, - users, videos, videoUploads, } from "@cap/database/schema"; @@ -22,8 +18,7 @@ import { dub, userIsPro } from "@cap/utils"; import { Storage } from "@cap/web-backend"; import { type Organisation, - Space, - SpaceMemberId, + type Space, type User, Video, } from "@cap/web-domain"; @@ -32,14 +27,11 @@ import { Option } from "effect"; import { revalidatePath } from "next/cache"; import { start } from "workflow/api"; import { - getOrganizationAccess, requireOrganizationAccess, requireOrganizationSettingsManager, } from "@/actions/organization/authorization"; import { requireSpaceManager } from "@/actions/organization/space-authorization"; import type { LoomImportDestination } from "@/lib/loom-import-destination"; -import { provisionOrganizationInvitee } from "@/lib/organization-provisioning"; -import { canManageOrganizationSettings } from "@/lib/permissions/roles"; import { runPromise } from "@/lib/server"; import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; @@ -68,36 +60,6 @@ export interface LoomImportResult { error?: string; } -export interface LoomCsvImportRow { - rowNumber: number; - loomUrl: string; - userEmail: string; - spaceName?: string; -} - -export interface LoomCsvImportRowResult { - rowNumber: number; - userEmail: string; - spaceName?: string; - success: boolean; - videoId?: Video.VideoId; - error?: string; -} - -export interface LoomCsvImportResult { - success: boolean; - importedCount: number; - failedCount: number; - results: LoomCsvImportRowResult[]; - error?: string; -} - -const MAX_LOOM_CSV_ROWS = 500; -const MAX_LOOM_SPACE_NAME_LENGTH = 255; -const LOOM_CSV_LIMIT_ERROR = `CSV imports are limited to ${MAX_LOOM_CSV_ROWS} rows at a time. Contact support to raise this limit.`; -const LOOM_CSV_PERMISSION_ERROR = - "Only organization admins and owners can import Loom videos from a CSV."; - function extractLoomVideoId(url: string): string | null { try { const parsed = new URL(url); @@ -112,6 +74,9 @@ function extractLoomVideoId(url: string): string | null { return null; } + const trailingLoomId = id.match(/([0-9a-f]{32})$/i)?.[1]; + if (trailingLoomId) return trailingLoomId.toLowerCase(); + return id.split("?")[0] ?? null; } catch { return null; @@ -159,7 +124,9 @@ async function fetchLoomEndpoint( } } -async function fetchVideoName(videoId: string): Promise { +async function fetchVideoDetails( + videoId: string, +): Promise<{ name: string | null; createdAt: string | null }> { try { const response = await fetch("https://www.loom.com/graphql", { method: "POST", @@ -173,7 +140,7 @@ async function fetchVideoName(videoId: string): Promise { variables: { videoId, password: null }, query: `query GetVideoName($videoId: ID!, $password: String) { getVideo(id: $videoId, password: $password) { - ... on RegularUserVideo { name } + ... on RegularUserVideo { name createdAt } ... on PrivateVideo { id } ... on VideoPasswordMissingOrIncorrect { id } } @@ -181,12 +148,18 @@ async function fetchVideoName(videoId: string): Promise { }), }); - if (!response.ok) return null; + if (!response.ok) return { name: null, createdAt: null }; const data = await response.json(); - return data?.data?.getVideo?.name ?? null; + const video = data?.data?.getVideo; + const createdAt = + typeof video?.createdAt === "string" && + !Number.isNaN(Date.parse(video.createdAt)) + ? new Date(video.createdAt).toISOString() + : null; + return { name: video?.name ?? null, createdAt }; } catch { - return null; + return { name: null, createdAt: null }; } } @@ -270,14 +243,14 @@ export async function downloadLoomVideo( }; } - const [videoName, oembedMeta] = await Promise.all([ - fetchVideoName(videoId), + const [details, oembedMeta] = await Promise.all([ + fetchVideoDetails(videoId), fetchLoomOEmbed(videoId), ]); return { success: true, videoId, - videoName: videoName ?? undefined, + videoName: details.name ?? undefined, downloadUrl, downloadMode: isDirectMp4Url(downloadUrl) ? "direct-download" @@ -364,10 +337,11 @@ async function importLoomVideoForOwner({ }; } - const [videoName, oembedMeta] = await Promise.all([ - fetchVideoName(loomVideoId), + const [details, oembedMeta] = await Promise.all([ + fetchVideoDetails(loomVideoId), fetchLoomOEmbed(loomVideoId), ]); + const videoName = details.name; const writableResult = await Storage.getWritableAccessForUser(ownerId, orgId) .pipe(runPromise) @@ -394,6 +368,7 @@ async function importLoomVideoForOwner({ const name = videoName || `Loom Import - ${new Date().toLocaleDateString("en-US", { day: "numeric", month: "long", year: "numeric" })}`; + const rawFileKey = `${ownerId}/${videoId}/raw-upload.mp4`; await db().transaction(async (tx) => { await tx.insert(videos).values({ @@ -409,6 +384,9 @@ async function importLoomVideoForOwner({ ...(oembedMeta?.duration ? { duration: oembedMeta.duration } : {}), ...(oembedMeta?.width ? { width: oembedMeta.width } : {}), ...(oembedMeta?.height ? { height: oembedMeta.height } : {}), + ...(details.createdAt + ? { metadata: { customCreatedAt: details.createdAt } } + : {}), }); await tx.insert(videoUploads).values({ @@ -416,6 +394,7 @@ async function importLoomVideoForOwner({ phase: "uploading", processingProgress: 0, processingMessage: "Importing from Loom...", + rawFileKey, }); await tx.insert(importedVideos).values({ @@ -444,8 +423,6 @@ async function importLoomVideoForOwner({ } }); - const rawFileKey = `${ownerId}/${videoId}/raw-upload.mp4`; - if (buildEnv.NEXT_PUBLIC_IS_CAP && NODE_ENV === "production") { await dub() .links.create({ @@ -567,381 +544,3 @@ export async function importFromLoom({ destination: { folderId, spaceId }, }); } - -function normalizeImportEmail(email: string) { - return email.trim().toLowerCase(); -} - -function normalizeImportSpaceName(spaceName: string) { - return spaceName.trim().replace(/\s+/g, " "); -} - -function getSpaceNameCacheKey(spaceName: string) { - return normalizeImportSpaceName(spaceName).toLowerCase(); -} - -function isValidImportEmail(email: string) { - return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email); -} - -function isValidImportSpaceName(spaceName: string) { - return spaceName.length <= MAX_LOOM_SPACE_NAME_LENGTH; -} - -async function getOrganizationMemberByEmail( - orgId: Organisation.OrganisationId, - email: string, -) { - const [member] = await db() - .select({ - userId: organizationMembers.userId, - email: users.email, - }) - .from(organizationMembers) - .innerJoin(users, eq(organizationMembers.userId, users.id)) - .where( - and( - eq(organizationMembers.organizationId, orgId), - eq(users.email, email), - ), - ) - .limit(1); - - return member ?? null; -} - -type ImportSpaceCacheValue = { - id: Space.SpaceIdOrOrganisationId; - name: string; -}; - -async function getOrCreateImportSpace({ - orgId, - createdById, - name, - spaceCache, -}: { - orgId: Organisation.OrganisationId; - createdById: User.UserId; - name: string; - spaceCache: Map; -}) { - const normalizedName = normalizeImportSpaceName(name); - const cacheKey = getSpaceNameCacheKey(normalizedName); - const cached = spaceCache.get(cacheKey); - if (cached) return cached; - - const [existingSpace] = await db() - .select({ - id: spaces.id, - name: spaces.name, - }) - .from(spaces) - .where( - and(eq(spaces.organizationId, orgId), eq(spaces.name, normalizedName)), - ) - .limit(1); - - if (existingSpace) { - const value = { - id: existingSpace.id, - name: existingSpace.name, - }; - spaceCache.set(cacheKey, value); - return value; - } - - const spaceId = Space.SpaceId.make(nanoId()); - - await db().transaction(async (tx) => { - await tx.insert(spaces).values({ - id: spaceId, - name: normalizedName, - organizationId: orgId, - createdById, - iconUrl: null, - }); - - await tx.insert(spaceMembers).values({ - id: SpaceMemberId.make(nanoId()), - spaceId, - userId: createdById, - role: "admin", - }); - }); - - const value = { - id: spaceId, - name: normalizedName, - }; - spaceCache.set(cacheKey, value); - return value; -} - -async function addImportedVideoToSpace({ - videoId, - spaceId, - addedById, -}: { - videoId: Video.VideoId; - spaceId: Space.SpaceIdOrOrganisationId; - addedById: User.UserId; -}) { - const [existingSpaceVideo] = await db() - .select({ id: spaceVideos.id }) - .from(spaceVideos) - .where( - and(eq(spaceVideos.spaceId, spaceId), eq(spaceVideos.videoId, videoId)), - ) - .limit(1); - - if (existingSpaceVideo) return; - - await db().insert(spaceVideos).values({ - id: nanoId(), - spaceId, - videoId, - addedById, - }); -} - -async function addImportOwnerToSpace({ - spaceId, - userId, -}: { - spaceId: Space.SpaceIdOrOrganisationId; - userId: User.UserId; -}) { - const [existingSpaceMember] = await db() - .select({ id: spaceMembers.id }) - .from(spaceMembers) - .where( - and(eq(spaceMembers.spaceId, spaceId), eq(spaceMembers.userId, userId)), - ) - .limit(1); - - if (existingSpaceMember) return; - - await db() - .insert(spaceMembers) - .values({ - id: SpaceMemberId.make(nanoId()), - spaceId, - userId, - role: "member", - }); -} - -export async function importFromLoomCsv({ - rows, - orgId, -}: { - rows: LoomCsvImportRow[]; - orgId: Organisation.OrganisationId; -}): Promise { - const user = await getCurrentUser(); - if (!user) { - return { - success: false, - importedCount: 0, - failedCount: 0, - results: [], - error: "Unauthorized", - }; - } - - if (!userIsPro(user)) { - return { - success: false, - importedCount: 0, - failedCount: 0, - results: [], - error: "Importing from Loom requires a Cap Pro subscription.", - }; - } - - const access = await getOrganizationAccess(user.id, orgId); - if (!canManageOrganizationSettings(access?.role)) { - return { - success: false, - importedCount: 0, - failedCount: 0, - results: [], - error: LOOM_CSV_PERMISSION_ERROR, - }; - } - - const inputRows = Array.isArray(rows) ? rows : []; - const normalizedRows = inputRows - .map((row, index) => ({ - rowNumber: - Number.isInteger(row.rowNumber) && row.rowNumber > 0 - ? row.rowNumber - : index + 2, - loomUrl: typeof row.loomUrl === "string" ? row.loomUrl.trim() : "", - userEmail: - typeof row.userEmail === "string" - ? normalizeImportEmail(row.userEmail) - : "", - spaceName: - typeof row.spaceName === "string" - ? normalizeImportSpaceName(row.spaceName) - : "", - })) - .filter((row) => row.loomUrl || row.userEmail || row.spaceName); - - if (normalizedRows.length === 0) { - return { - success: false, - importedCount: 0, - failedCount: 0, - results: [], - error: "No rows found to import.", - }; - } - - if (normalizedRows.length > MAX_LOOM_CSV_ROWS) { - return { - success: false, - importedCount: 0, - failedCount: normalizedRows.length, - results: [], - error: LOOM_CSV_LIMIT_ERROR, - }; - } - - const results: LoomCsvImportRowResult[] = []; - const spaceCache = new Map(); - const touchedSpaceIds = new Set(); - - for (const row of normalizedRows) { - if (!row.loomUrl) { - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName: row.spaceName || undefined, - success: false, - error: "Missing Loom video URL.", - }); - continue; - } - - if (!isValidImportEmail(row.userEmail)) { - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName: row.spaceName || undefined, - success: false, - error: "Missing or invalid user email.", - }); - continue; - } - - if (!isValidImportSpaceName(row.spaceName)) { - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName: row.spaceName, - success: false, - error: `Space name must be ${MAX_LOOM_SPACE_NAME_LENGTH} characters or fewer.`, - }); - continue; - } - - let member = await getOrganizationMemberByEmail(orgId, row.userEmail); - - if (!member) { - try { - const provisionedMember = await provisionOrganizationInvitee({ - organizationId: orgId, - email: row.userEmail, - invitedByUserId: user.id, - role: "member", - }); - member = { - userId: provisionedMember.userId, - email: row.userEmail, - }; - } catch { - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName: row.spaceName || undefined, - success: false, - error: "Could not add this email to the organization.", - }); - continue; - } - } - - try { - const result = await importLoomVideoForOwner({ - loomUrl: row.loomUrl, - orgId, - ownerId: member.userId, - }); - - let spaceName = row.spaceName || undefined; - let spaceError: string | undefined; - if (result.success && result.videoId && row.spaceName) { - try { - const space = await getOrCreateImportSpace({ - orgId, - createdById: user.id, - name: row.spaceName, - spaceCache, - }); - await addImportedVideoToSpace({ - videoId: result.videoId, - spaceId: space.id, - addedById: user.id, - }); - await addImportOwnerToSpace({ - spaceId: space.id, - userId: member.userId, - }); - touchedSpaceIds.add(space.id); - spaceName = space.name; - } catch { - spaceError = "Import started, but it could not be added to a space."; - } - } - - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName, - success: result.success, - videoId: result.videoId, - error: result.error ?? spaceError, - }); - } catch { - results.push({ - rowNumber: row.rowNumber, - userEmail: row.userEmail, - spaceName: row.spaceName || undefined, - success: false, - error: "Failed to start this import.", - }); - } - } - - const importedCount = results.filter((result) => result.success).length; - const failedCount = results.length - importedCount; - - for (const spaceId of touchedSpaceIds) { - revalidatePath(`/dashboard/spaces/${spaceId}`); - } - - if (touchedSpaceIds.size > 0) { - revalidatePath("/dashboard"); - } - - return { - success: importedCount > 0, - importedCount, - failedCount, - results, - error: importedCount > 0 ? undefined : "No Loom videos were imported.", - }; -} diff --git a/apps/web/actions/video/retry-processing.ts b/apps/web/actions/video/retry-processing.ts index fa53dabec2b..02820f6a440 100644 --- a/apps/web/actions/video/retry-processing.ts +++ b/apps/web/actions/video/retry-processing.ts @@ -6,6 +6,7 @@ import { importedVideos, videos, videoUploads } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { start } from "workflow/api"; +import { markLoomImportRetrying } from "@/lib/loom-import/retry"; import { setVideoProcessingError, startVideoProcessingWorkflow, @@ -114,6 +115,7 @@ export async function retryVideoProcessing({ throw normalizedError; } + await markLoomImportRetrying(videoId); return { success: true, status: "started" }; } diff --git a/apps/web/app/(org)/dashboard/import/loom/ImportLoomPage.tsx b/apps/web/app/(org)/dashboard/import/loom/ImportLoomPage.tsx index ebca8079829..a3fbf7b366b 100644 --- a/apps/web/app/(org)/dashboard/import/loom/ImportLoomPage.tsx +++ b/apps/web/app/(org)/dashboard/import/loom/ImportLoomPage.tsx @@ -2,34 +2,18 @@ import { Button, - Dialog, - DialogContent, - DialogFooter, - DialogHeader, - DialogTitle, Input, - Select, SelectContent, SelectItem, SelectRoot, SelectTrigger, SelectValue, - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, } from "@cap/ui"; import { Folder } from "@cap/web-domain"; import { faArrowLeft, - faCircleCheck, - faDownload, faFileCsv, faLink, - faTriangleExclamation, - faUpload, } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useQuery } from "@tanstack/react-query"; @@ -37,24 +21,12 @@ import clsx from "clsx"; import { motion } from "framer-motion"; import Link from "next/link"; import { useRouter, useSearchParams } from "next/navigation"; -import { - type ChangeEvent, - type DragEvent, - useId, - useMemo, - useRef, - useState, -} from "react"; +import { useId, useMemo, useState } from "react"; import { toast } from "sonner"; -import { - getLoomImportFolders, - importFromLoom, - importFromLoomCsv, - type LoomCsvImportResult, - type LoomCsvImportRowResult, -} from "@/actions/loom"; +import { getLoomImportFolders, importFromLoom } from "@/actions/loom"; import { useDashboardContext } from "@/app/(org)/dashboard/Contexts"; import { UpgradeModal } from "@/components/UpgradeModal"; +import type { LoomImportJobSummary } from "@/lib/loom-import/jobs"; import { type LoomImportDestination, loomImportDestinationHref, @@ -65,194 +37,20 @@ import { canManageOrganizationSettings, getEffectiveOrganizationRole, } from "@/lib/permissions/roles"; +import { BulkImport } from "./_components/bulk-import"; +import { LoomMark } from "./_components/doodles"; +import { RecentImports } from "./_components/recent-imports"; type Mode = "single" | "csv"; -type CsvData = { - fileName: string; - headers: string[]; - rows: string[][]; -}; - -type Mapping = { - loomUrl?: string; - userEmail?: string; - spaceName?: string; -}; - -type MappedRow = { - rowNumber: number; - loomUrl: string; - userEmail: string; - spaceName: string; -}; - -const LOOM_CSV_TEMPLATE = - "loom_video_url,user_email,space_name\nhttps://www.loom.com/share/0123456789abcdef,user@example.com,Sales\n"; - const ROOT_FOLDER_VALUE = "__cap_root_folder__"; -const OPTIONAL_COLUMN_VALUE = "__cap_skip_column__"; -const MAX_SPACE_NAME_LENGTH = 255; -const MAX_LOOM_CSV_IMPORT_ROWS = 500; -const LOOM_CSV_BATCH_SIZE = 10; -const LOOM_CSV_BATCH_DELAY_MS = 1500; -const LOOM_CSV_LIMIT_MESSAGE = - "CSV imports are limited to 500 videos at a time. Contact support to raise this limit."; -const LOOM_CSV_PERMISSION_MESSAGE = - "Only organization admins and owners can import Loom videos from a CSV."; - -function delay(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function chunkRows(rows: T[], size: number) { - const chunks: T[][] = []; - for (let index = 0; index < rows.length; index += size) { - chunks.push(rows.slice(index, index + size)); - } - return chunks; -} - -function buildCsvImportResult( - results: LoomCsvImportRowResult[], - error?: string, -): LoomCsvImportResult { - const importedCount = results.filter((row) => row.success).length; - const failedCount = results.length - importedCount; - - return { - success: importedCount > 0, - importedCount, - failedCount, - results, - error: - error ?? - (importedCount > 0 ? undefined : "No Loom videos were imported."), - }; -} - -function parseCsvRecords(text: string) { - const records: string[][] = []; - let field = ""; - let row: string[] = []; - let inQuotes = false; - const input = text.replace(/^\uFEFF/, ""); - - for (let index = 0; index < input.length; index += 1) { - const char = input.charAt(index); - const next = input.charAt(index + 1); - - if (char === '"') { - if (inQuotes && next === '"') { - field += '"'; - index += 1; - } else { - inQuotes = !inQuotes; - } - continue; - } - - if (char === "," && !inQuotes) { - row.push(field.trim()); - field = ""; - continue; - } - - if ((char === "\n" || char === "\r") && !inQuotes) { - if (char === "\r" && next === "\n") index += 1; - row.push(field.trim()); - if (row.some((cell) => cell.length > 0)) records.push(row); - row = []; - field = ""; - continue; - } - - field += char; - } - - if (inQuotes) throw new Error("CSV has an unclosed quoted field."); - - if (field.length > 0 || row.length > 0) { - row.push(field.trim()); - if (row.some((cell) => cell.length > 0)) records.push(row); - } - - return records; -} - -function parseCsv(text: string, fileName: string): CsvData { - const records = parseCsvRecords(text); - const headers = records[0]?.map((header) => header.trim()) ?? []; - const rows = records - .slice(1) - .filter((row) => row.some((cell) => cell.trim().length > 0)); - - if (headers.length === 0) { - throw new Error("No CSV headers found."); - } - - return { fileName, headers, rows }; -} - -function normalizeHeader(value: string) { - return value.toLowerCase().replace(/[^a-z0-9]/g, ""); -} - -function guessColumn(headers: string[], candidates: string[]) { - const normalizedHeaders = headers.map(normalizeHeader); - const directMatch = normalizedHeaders.findIndex((header) => - candidates.includes(header), - ); - if (directMatch !== -1) return String(directMatch); - - const partialMatch = normalizedHeaders.findIndex((header) => - candidates.some((candidate) => header.includes(candidate)), - ); - return partialMatch === -1 ? undefined : String(partialMatch); -} - -function isLoomUrl(value: string) { - try { - return new URL(value).hostname.includes("loom.com"); - } catch { - return false; - } -} - -function isEmail(value: string) { - return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value); -} - -function isValidSpaceName(value: string) { - return value.length <= MAX_SPACE_NAME_LENGTH; -} - -function pluralize(count: number, singular: string, plural: string) { - return count === 1 ? singular : plural; -} - -const LoomMark = ({ size = 18 }: { size?: number }) => ( - - - -); - export const ImportLoomPage = ({ initialDestination = {}, + recentJobs = [], }: { initialDestination?: LoomImportDestination; + recentJobs?: LoomImportJobSummary[]; }) => { const { user, activeOrganization, spacesData } = useDashboardContext(); const router = useRouter(); @@ -269,11 +67,10 @@ export const ImportLoomPage = ({ ownerId: activeOrganization?.organization.ownerId, memberRole: currentMember?.role, }); - const canUseCsvImport = canManageOrganizationSettings(currentRole); + const isAdmin = canManageOrganizationSettings(currentRole); const [mode, setMode] = useState(prefilledMode); - const activeMode = canUseCsvImport ? mode : "single"; - const [upgradeModalOpen, setUpgradeModalOpen] = useState(!user?.isPro); + const [upgradeModalOpen, setUpgradeModalOpen] = useState(false); const [selectedFolderId, setSelectedFolderId] = useState( initialDestination.folderId ?? ROOT_FOLDER_VALUE, @@ -285,7 +82,7 @@ export const ImportLoomPage = ({ queryKey: ["loom-import-folders", user.id, orgId, spaceId], queryFn: () => orgId ? getLoomImportFolders({ orgId, spaceId }) : Promise.resolve([]), - enabled: Boolean(orgId) && activeMode === "single", + enabled: Boolean(orgId) && mode === "single", }); const folderRows = useMemo( () => buildMoveFolderDestinationRows(foldersQuery.data ?? []), @@ -317,76 +114,6 @@ export const ImportLoomPage = ({ const [loomUrl, setLoomUrl] = useState(prefilledLoomUrl); const [isImporting, setIsImporting] = useState(false); - const inputRef = useRef(null); - const [csvData, setCsvData] = useState(null); - const [mapping, setMapping] = useState({}); - const [isDragOver, setIsDragOver] = useState(false); - const [confirmOpen, setConfirmOpen] = useState(false); - const [isCsvImporting, setIsCsvImporting] = useState(false); - const [csvImportProgress, setCsvImportProgress] = useState(0); - const [result, setResult] = useState(null); - - const selectedColumnValues = [ - mapping.loomUrl, - mapping.userEmail, - mapping.spaceName, - ].filter((value) => value !== undefined); - const selectedColumnsConflict = - new Set(selectedColumnValues).size !== selectedColumnValues.length; - - const mappedRows = useMemo(() => { - if ( - !csvData || - mapping.loomUrl === undefined || - mapping.userEmail === undefined - ) { - return []; - } - - const loomIndex = Number(mapping.loomUrl); - const emailIndex = Number(mapping.userEmail); - const spaceIndex = - mapping.spaceName === undefined ? undefined : Number(mapping.spaceName); - - return csvData.rows - .map((row, index) => ({ - rowNumber: index + 2, - loomUrl: (row[loomIndex] ?? "").trim(), - userEmail: (row[emailIndex] ?? "").trim().toLowerCase(), - spaceName: - spaceIndex === undefined ? "" : (row[spaceIndex] ?? "").trim(), - })) - .filter((row) => row.loomUrl || row.userEmail || row.spaceName); - }, [csvData, mapping.loomUrl, mapping.spaceName, mapping.userEmail]); - - const readyRows = useMemo( - () => - mappedRows.filter( - (row) => - isLoomUrl(row.loomUrl) && - isEmail(row.userEmail) && - isValidSpaceName(row.spaceName), - ), - [mappedRows], - ); - - const invalidRows = mappedRows.length - readyRows.length; - const previewRows = mappedRows.slice(0, 5); - const csvLimitExceeded = readyRows.length > MAX_LOOM_CSV_IMPORT_ROWS; - const canImport = - canUseCsvImport && - !!activeOrganization && - !selectedColumnsConflict && - readyRows.length > 0 && - !csvLimitExceeded && - !isCsvImporting; - - const columnOptions = - csvData?.headers.map((header, index) => ({ - value: String(index), - label: header || `Column ${index + 1}`, - })) ?? []; - const isValidLoomUrl = (() => { try { const parsed = new URL(loomUrl.trim()); @@ -431,153 +158,6 @@ export const ImportLoomPage = ({ } }; - const handleTemplateDownload = () => { - const blob = new Blob([LOOM_CSV_TEMPLATE], { - type: "text/csv;charset=utf-8", - }); - const url = URL.createObjectURL(blob); - const link = document.createElement("a"); - link.href = url; - link.download = "cap-loom-import-template.csv"; - link.click(); - URL.revokeObjectURL(url); - }; - - const loadCsvFile = async (file: File) => { - if (!user) return; - - if (!canUseCsvImport) { - toast.error(LOOM_CSV_PERMISSION_MESSAGE); - return; - } - - if (!user.isPro) { - setUpgradeModalOpen(true); - return; - } - - if (!file.name.toLowerCase().endsWith(".csv") && file.type !== "text/csv") { - toast.error("Please upload a CSV file."); - return; - } - - try { - const parsed = parseCsv(await file.text(), file.name); - if (parsed.rows.length > MAX_LOOM_CSV_IMPORT_ROWS) { - toast.error(LOOM_CSV_LIMIT_MESSAGE); - } - - const loomUrlGuess = guessColumn(parsed.headers, [ - "loomvideourl", - "loomurl", - "loomlink", - "videourl", - "url", - ]); - const userEmailGuess = guessColumn(parsed.headers, [ - "useremail", - "memberemail", - "owneremail", - "email", - ]); - const spaceNameGuess = guessColumn(parsed.headers, [ - "spacename", - "space", - "workspace", - "workspacename", - ]); - - setCsvData(parsed); - setMapping({ - loomUrl: loomUrlGuess, - userEmail: userEmailGuess, - spaceName: spaceNameGuess, - }); - setResult(null); - setCsvImportProgress(0); - } catch (error) { - toast.error( - error instanceof Error ? error.message : "Could not parse CSV.", - ); - } - }; - - const handleFileChange = async (event: ChangeEvent) => { - const file = event.target.files?.[0]; - if (!file) return; - await loadCsvFile(file); - if (inputRef.current) inputRef.current.value = ""; - }; - - const handleDrop = async (event: DragEvent) => { - event.preventDefault(); - setIsDragOver(false); - const file = event.dataTransfer.files[0]; - if (!file) return; - await loadCsvFile(file); - }; - - const handleCsvImport = async () => { - if (csvLimitExceeded) { - toast.error(LOOM_CSV_LIMIT_MESSAGE); - return; - } - - if (!activeOrganization || !canImport) return; - - setIsCsvImporting(true); - setResult(null); - setCsvImportProgress(0); - - try { - const batches = chunkRows(readyRows, LOOM_CSV_BATCH_SIZE); - let combinedResults: LoomCsvImportRowResult[] = []; - let blockedError: string | undefined; - - for (const [batchIndex, batch] of batches.entries()) { - const importResult = await importFromLoomCsv({ - orgId: activeOrganization.organization.id, - rows: batch, - }); - - if (importResult.results.length === 0 && importResult.error) { - blockedError = importResult.error; - break; - } - - combinedResults = [...combinedResults, ...importResult.results]; - setCsvImportProgress(combinedResults.length); - setResult(buildCsvImportResult(combinedResults)); - - if (batchIndex < batches.length - 1) { - await delay(LOOM_CSV_BATCH_DELAY_MS); - } - } - - const finalResult = buildCsvImportResult(combinedResults, blockedError); - setResult(finalResult); - - if (finalResult.importedCount > 0) { - toast.success( - `${finalResult.importedCount} ${pluralize( - finalResult.importedCount, - "Loom import", - "Loom imports", - )} started.`, - ); - router.refresh(); - } else { - toast.error(finalResult.error || "No Loom videos were imported."); - } - - setConfirmOpen(false); - } catch { - toast.error("An unexpected error occurred. Please try again."); - } finally { - setIsCsvImporting(false); - } - }; - return (
@@ -597,37 +177,34 @@ export const ImportLoomPage = ({ Import from Loom

- {canUseCsvImport - ? "Bring a single Loom video into Cap, or bulk import recordings for organization members and new users from a CSV." - : "Paste a Loom share link to bring it into Cap."} + Bring one Loom over with a link, or move a whole library from a + CSV. Titles and original recording dates come with them.

- {canUseCsvImport && ( -
- setMode("single")} - /> - setMode("csv")} - /> -
- )} +
+ setMode("single")} + /> + setMode("csv")} + /> +
- {activeMode === "single" ? ( + {mode === "single" ? (

Loom video URL

@@ -753,405 +330,13 @@ export const ImportLoomPage = ({
- ) : ( -
- {!csvData && ( - <> -
-
-
- -
-
-

- First time? Start with our template -

-

- Two columns required:{" "} - - loom_video_url - {" "} - and{" "} - - user_email - - . Add{" "} - - space_name - {" "} - to place videos in spaces. Emails that are not members - yet will be added without an email invite. -

-
-
- -
+ ) : orgId ? ( + + ) : null} -
{ - event.preventDefault(); - setIsDragOver(true); - }} - onDragLeave={() => setIsDragOver(false)} - onDrop={handleDrop} - className={clsx( - "relative flex flex-col items-center justify-center w-full rounded-xl border-2 border-dashed transition-all duration-200 py-14 px-8", - isDragOver - ? "border-blue-10 bg-blue-3" - : "border-gray-4 bg-gray-1 hover:border-gray-6 hover:bg-gray-2", - )} - > -
-
- -
-
-

- Drag and drop your CSV here -

-

- Or browse your computer to upload a file. -

-
- -
-
- - )} - - - - {csvData && ( -
-
-
-
- -
-
-

- {csvData.fileName} -

-

- {csvData.rows.length}{" "} - {pluralize(csvData.rows.length, "row", "rows")} detected -

-
-
- -
- -
-
-

- Map columns -

-
- - setMapping((current) => ({ - ...current, - loomUrl: value, - })) - } - /> - - setMapping((current) => ({ - ...current, - userEmail: value, - })) - } - /> - - setMapping((current) => ({ - ...current, - spaceName: value, - })) - } - /> -
- {selectedColumnsConflict && ( -

- Choose different columns for each mapped field. -

- )} -
- -
- - - -
- - {csvLimitExceeded && ( -
- -
-

- CSV imports are limited to {MAX_LOOM_CSV_IMPORT_ROWS}{" "} - videos at a time. -

-

- Split this file into smaller batches or{" "} - - contact support - {" "} - to raise the limit. -

-
-
- )} - - {previewRows.length > 0 && ( -
- - - - Row - Loom URL - User email - Space - Status - - - - {previewRows.map((row) => { - const valid = - isLoomUrl(row.loomUrl) && - isEmail(row.userEmail) && - isValidSpaceName(row.spaceName); - return ( - - - {row.rowNumber} - - - {row.loomUrl || "—"} - - {row.userEmail || "—"} - {row.spaceName || "—"} - - - - - ); - })} - -
- {mappedRows.length > previewRows.length && ( -
- Showing {previewRows.length} of {mappedRows.length}{" "} - mapped rows. -
- )} -
- )} - -
- - -
-
-
- )} - - {result && ( -
-
-
-

- Import results -

-

- {result.importedCount}{" "} - {pluralize(result.importedCount, "started", "started")},{" "} - {result.failedCount}{" "} - {pluralize(result.failedCount, "failed", "failed")} -

-
-
- - {result.failedCount > 0 && ( - - )} -
-
-
- - - - Row - User email - Space - Status - - - - {result.results.map((row) => ( - - - {row.rowNumber} - - {row.userEmail || "—"} - {row.spaceName || "—"} - - {row.error || (row.success ? "Started" : "Failed")} - - - ))} - -
-
-
- )} -
- )} + {mode === "csv" && } - - - } - > - Start CSV import - -
- {readyRows.length} {pluralize(readyRows.length, "video", "videos")}{" "} - will be imported for existing members or newly added users in - batches of {LOOM_CSV_BATCH_SIZE}. - {readyRows.some((row) => row.spaceName) && ( - - Rows with a space name will be added to that space. Missing - spaces will be created. - - )} - {invalidRows > 0 && ( - - {invalidRows} {pluralize(invalidRows, "row", "rows")} will be - skipped because the Loom URL, email, or space name is invalid. - - )} -
- - - - - - - {label} ); - -const MappingField = ({ - label, - value, - options, - optional = false, - onValueChange, -}: { - label: string; - value: string | undefined; - options: { value: string; label: string }[]; - optional?: boolean; - onValueChange: (value: string | undefined) => void; -}) => { - const fieldOptions = optional - ? [{ value: OPTIONAL_COLUMN_VALUE, label: "Do not import" }, ...options] - : options; - - return ( -
-

{label}

- setQuery(event.target.value)} + placeholder="Search titles, owners, links" + aria-label="Search videos" + className="h-8 w-full rounded-full border border-gray-4 bg-gray-2 px-3 text-sm text-gray-12 outline-none transition-colors placeholder:text-gray-9 focus:border-blue-9 lg:w-60" + /> +
+ + + +

+ AI titles, summaries and chapters for imported videos are made the + first time each video is opened. Need more than{" "} + {numberFormat.format(2000)} videos? Start another import with the next + CSV, they run side by side. +

+ + + + setCancelOpen(false)} + /> + + ); +}; diff --git a/apps/web/app/(org)/dashboard/import/loom/[jobId]/import-list.tsx b/apps/web/app/(org)/dashboard/import/loom/[jobId]/import-list.tsx new file mode 100644 index 00000000000..69c68f2f06a --- /dev/null +++ b/apps/web/app/(org)/dashboard/import/loom/[jobId]/import-list.tsx @@ -0,0 +1,324 @@ +"use client"; + +import clsx from "clsx"; +import Link from "next/link"; +import { + memo, + type ReactNode, + useCallback, + useEffect, + useLayoutEffect, + useRef, + useState, +} from "react"; +import { + type LoomImportDisplayStatus, + type LoomImportItemView, + loomImportStageLabel, +} from "@/lib/loom-import/status"; + +export const ROW_HEIGHT = 64; +const OVERSCAN = 8; +const SCROLL_SETTLE_MS = 140; +const loadedThumbs = new Set(); + +const dateFormat = new Intl.DateTimeFormat("en-US", { + day: "numeric", + month: "short", + year: "numeric", +}); + +export function formatDuration(seconds: number | null | undefined) { + if (!seconds || seconds <= 0) return null; + const total = Math.round(seconds); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const secs = total % 60; + return hours > 0 + ? `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}` + : `${minutes}:${String(secs).padStart(2, "0")}`; +} + +function shortLoomUrl(url: string) { + return url.replace(/^https?:\/\/(www\.)?/, ""); +} + +export function useVirtualWindow(count: number, viewport: number) { + const ref = useRef(null); + const [scrollTop, setScrollTop] = useState(0); + const [scrolling, setScrolling] = useState(false); + const frame = useRef(null); + const settle = useRef(undefined); + + const onScroll = useCallback(() => { + window.clearTimeout(settle.current); + settle.current = window.setTimeout( + () => setScrolling(false), + SCROLL_SETTLE_MS, + ); + if (frame.current !== null) return; + frame.current = window.requestAnimationFrame(() => { + frame.current = null; + setScrolling(true); + setScrollTop(ref.current?.scrollTop ?? 0); + }); + }, []); + + useEffect( + () => () => { + if (frame.current !== null) window.cancelAnimationFrame(frame.current); + window.clearTimeout(settle.current); + }, + [], + ); + + useLayoutEffect(() => { + const element = ref.current; + const max = Math.max(0, count * ROW_HEIGHT - viewport); + if (element && element.scrollTop > max) element.scrollTop = max; + setScrollTop(element?.scrollTop ?? 0); + }, [count, viewport]); + + const start = Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - OVERSCAN); + const end = Math.min( + count, + Math.ceil((scrollTop + viewport) / ROW_HEIGHT) + OVERSCAN, + ); + return { ref, onScroll, start, end, scrolling }; +} + +const STATUS_TONE: Record = { + checking: "text-gray-10", + ready: "text-gray-11", + queued: "text-gray-10", + importing: "text-[var(--li-accent)]", + imported: "text-[#218358] dark:text-[#3dd68c]", + failed: "text-red-11", + skipped: "text-gray-10", + cancelled: "text-gray-9", +}; + +const STATUS_LABEL: Record = { + checking: "Checking", + ready: "Ready", + queued: "Queued", + importing: "Importing", + imported: "In Cap", + failed: "Failed", + skipped: "Skipped", + cancelled: "Cancelled", +}; + +const Tick = () => ( + +); + +const StatusCell = ({ item }: { item: LoomImportItemView }) => { + const tone = STATUS_TONE[item.status]; + if (item.status === "importing") { + const progress = item.progress ?? 0; + return ( +
+ + {loomImportStageLabel(item.stage)} + +
+
+
+
+ ); + } + + if (item.status === "imported" && item.videoId) { + return ( + + + {STATUS_LABEL.imported} + + ); + } + + if (item.status === "skipped" && item.videoId) { + return ( + + Already in Cap + + Open it + + + ); + } + + return ( +
+ + {item.status === "checking" && ( + + )} + {STATUS_LABEL[item.status]} + + {item.error && ( + + {item.error} + + )} +
+ ); +}; + +const Thumb = ({ + src, + deferred, +}: { + src: string | null; + deferred: boolean; +}) => ( +
+ {src && (!deferred || loadedThumbs.has(src)) && ( + loadedThumbs.add(src)} + onError={(event) => { + event.currentTarget.style.visibility = "hidden"; + }} + /> + )} +
+); + +export const ImportRow = memo(function ImportRow({ + item, + showOwner, + scrolling, +}: { + item: LoomImportItemView; + showOwner: boolean; + scrolling: boolean; +}) { + const meta = [ + item.recordedAt + ? `Recorded ${dateFormat.format(new Date(item.recordedAt))}` + : null, + formatDuration(item.duration), + showOwner ? item.email : null, + item.space, + ].filter(Boolean); + return ( +
+ +
+

+ {item.title ?? shortLoomUrl(item.url)} +

+

+ Row {item.row} + {meta.length > 0 && ` · ${meta.join(" · ")}`} +

+
+
+ +
+
+ ); +}); + +export const VirtualImportList = ({ + items, + viewport, + showOwner, + empty, +}: { + items: LoomImportItemView[]; + viewport: number; + showOwner: boolean; + empty: ReactNode; +}) => { + const { ref, onScroll, start, end, scrolling } = useVirtualWindow( + items.length, + viewport, + ); + const isEmpty = items.length === 0; + + return ( +
+ {isEmpty ? ( +
+ {empty} +
+ ) : ( +
    + {items.slice(start, end).map((item, index) => ( +
  • + +
    • + ))} +
+ )} +
+ ); +}; diff --git a/apps/web/app/(org)/dashboard/import/loom/[jobId]/page.tsx b/apps/web/app/(org)/dashboard/import/loom/[jobId]/page.tsx new file mode 100644 index 00000000000..80cfe736e68 --- /dev/null +++ b/apps/web/app/(org)/dashboard/import/loom/[jobId]/page.tsx @@ -0,0 +1,42 @@ +import { getCurrentUser } from "@cap/database/auth/session"; +import type { Metadata } from "next"; +import { notFound, redirect } from "next/navigation"; +import { getLoomImportSnapshot } from "@/lib/loom-import/snapshot"; +import { LoomImportJobView } from "./LoomImportJobView"; + +export const metadata: Metadata = { + title: "Loom import — Cap", +}; + +const FIRST_SCREEN_ROWS = 60; + +export default async function Page({ + params, + searchParams, +}: { + params: Promise<{ jobId: string }>; + searchParams: Promise>; +}) { + const [{ jobId }, query, user] = await Promise.all([ + params, + searchParams, + getCurrentUser(), + ]); + if (!user) redirect("/login"); + + const snapshot = await getLoomImportSnapshot({ jobId, userId: user.id }); + if (!snapshot) notFound(); + + const firstScreen = snapshot.items.slice(0, FIRST_SCREEN_ROWS); + return ( + + ); +} diff --git a/apps/web/app/(org)/dashboard/import/loom/[jobId]/upgrade-panel.tsx b/apps/web/app/(org)/dashboard/import/loom/[jobId]/upgrade-panel.tsx new file mode 100644 index 00000000000..fab51104b83 --- /dev/null +++ b/apps/web/app/(org)/dashboard/import/loom/[jobId]/upgrade-panel.tsx @@ -0,0 +1,290 @@ +"use client"; + +import { Button } from "@cap/ui"; +import NumberFlow from "@number-flow/react"; +import { useMutation } from "@tanstack/react-query"; +import clsx from "clsx"; +import { useCurrency } from "hooks/useCurrency"; +import { useId, useState } from "react"; +import { toast } from "sonner"; +import { useStripeContext } from "@/app/Layout/StripeContext"; +import { UpgradeModal } from "@/components/UpgradeModal"; +import { PRICING } from "@/data/pricing"; +import { Boil, Doodle, delay } from "../_components/doodles"; + +const numberFormat = new Intl.NumberFormat("en-US"); +const ANNUAL_SAVINGS = Math.round( + (1 - PRICING.pro.annualPerMonth / PRICING.pro.monthly) * 100, +); + +export function formatHours(seconds: number) { + if (seconds <= 0) return "0m"; + const hours = Math.floor(seconds / 3600); + const minutes = Math.round((seconds % 3600) / 60); + if (hours === 0) return `${Math.max(minutes, 1)}m`; + return minutes > 0 + ? `${numberFormat.format(hours)}h ${minutes}m` + : `${numberFormat.format(hours)}h`; +} + +const PERKS = [ + "Every video, any length, unlimited storage", + "Original titles and recording dates kept", + "AI summaries and chapters the first time a video is watched", + "Custom domain, passwords and viewer analytics", +]; + +const InkCheck = ({ at }: { at: number }) => ( + +); + +export const UpgradePanel = ({ + jobId, + videoCount, + totalDuration, + owners, + thumbs, + confirming, + canStart, + starting, + onStart, +}: { + jobId: string; + videoCount: number; + totalDuration: number; + owners: number; + thumbs: string[]; + confirming: boolean; + canStart: boolean; + starting: boolean; + onStart: () => void; +}) => { + const stripe = useStripeContext(); + const { currency } = useCurrency(); + const [annual, setAnnual] = useState(true); + const [compareOpen, setCompareOpen] = useState(false); + const titleId = useId(); + const perMonth = annual ? PRICING.pro.annualPerMonth : PRICING.pro.monthly; + + const checkout = useMutation({ + mutationFn: async () => { + const response = await fetch("/api/settings/billing/subscribe", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + priceId: stripe.plans[annual ? "yearly" : "monthly"], + quantity: 1, + returnTo: `/dashboard/import/loom/${jobId}`, + }), + }); + const data = (await response.json().catch(() => ({}))) as { + url?: string; + subscription?: boolean; + }; + if (data.subscription) { + window.location.reload(); + return; + } + if (!data.url) throw new Error("Checkout unavailable"); + window.location.href = data.url; + }, + onError: () => { + toast.error("We couldn't open checkout. Please try again."); + }, + }); + + if (confirming || canStart) { + return ( +
+ +
+

+ {canStart ? "You're on Cap Pro" : "Confirming your upgrade"} +

+

+ {canStart + ? `${numberFormat.format(videoCount)} videos are checked and ready to copy into Cap.` + : "This usually takes a few seconds. Your import starts on its own once it's done."} +

+
+ {canStart && ( + + )} +
+ ); + } + + return ( +
+
+
+ +
+

+ Your Loom library is ready to move +

+

+ We checked every link. Upgrade to Cap Pro and we'll copy them into + Cap for you, right where your team expects them. +

+
+
+ +
+ {[ + { + value: numberFormat.format(videoCount), + label: videoCount === 1 ? "video" : "videos", + }, + { value: formatHours(totalDuration), label: "of recordings" }, + { + value: numberFormat.format(owners), + label: owners === 1 ? "owner" : "owners", + }, + ].map((stat) => ( +
+
{stat.label}
+
+ {stat.value} +
+
+ ))} +
+ + {thumbs.length > 0 && ( +
+ {thumbs.slice(0, 12).map((thumb, index) => ( +
= 8 && "hidden sm:block", + )} + style={delay(0.05 * index)} + > + { + event.currentTarget.style.visibility = "hidden"; + }} + /> +
+ ))} +
+ )} +
+ +
+
+

Cap Pro

+
+ {[ + { value: false, label: "Monthly" }, + { value: true, label: `Yearly · save ${ANNUAL_SAVINGS}%` }, + ].map((option) => ( + + ))} +
+
+ +
+ + + per user / month{annual ? ", billed yearly" : ""} + +
+ +
    + {PERKS.map((perk, index) => ( +
  • + + {perk} +
    • + ))} +
+ +
+ +

+ Cancel anytime. Your import starts as soon as you're back. +

+ +
+
+ +
+ ); +}; diff --git a/apps/web/app/(org)/dashboard/import/loom/[jobId]/use-loom-import-job.ts b/apps/web/app/(org)/dashboard/import/loom/[jobId]/use-loom-import-job.ts new file mode 100644 index 00000000000..fc31731dbd8 --- /dev/null +++ b/apps/web/app/(org)/dashboard/import/loom/[jobId]/use-loom-import-job.ts @@ -0,0 +1,247 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { + isLoomImportJobActive, + type LoomImportItemView, + type LoomImportJobStatus, + type LoomImportJobView, + type LoomImportSnapshot, + type LoomImportSummaryView, + loomImportWaitingStatus, + summarizeLoomImportItems, +} from "@/lib/loom-import/status"; + +const ACTIVE_POLL_MS = 2000; +const IDLE_POLL_MS = 10_000; +const UPGRADE_POLL_MS = 2500; +const FULL_REFRESH_MS = 180_000; +const SNAPSHOT_TIMEOUT_MS = 20_000; +const RATE_WINDOW_MS = 180_000; +const RATE_MIN_SPAN_MS = 20_000; + +export type LoomImportSummary = LoomImportSummaryView & { + job: LoomImportJobView; +}; + +type RateSample = { at: number; settled: number }; + +function settledCount(summary: LoomImportSummaryView) { + const { imported, failed, skipped, cancelled } = summary.counts; + return imported + failed + skipped + cancelled; +} + +export function mergeLoomImportItems( + map: Map, + order: string[], + snapshot: LoomImportSnapshot, + previousJobStatus: LoomImportJobStatus = snapshot.job.status, +) { + let nextOrder = order; + if (snapshot.full) { + map.clear(); + nextOrder = snapshot.items.map((item) => item.id); + } + let changed = snapshot.full; + if (!snapshot.full && previousJobStatus !== snapshot.job.status) { + const waiting = loomImportWaitingStatus(snapshot.job.status); + for (const [id, item] of map) { + if ( + (item.status === "ready" || item.status === "queued") && + item.status !== waiting + ) { + map.set(id, { ...item, status: waiting }); + changed = true; + } + } + } + for (const item of snapshot.items) { + const previous = map.get(item.id); + if ( + !previous || + previous.v !== item.v || + previous.status !== item.status || + previous.progress !== item.progress || + previous.stage !== item.stage + ) { + map.set(item.id, item); + changed = true; + } + } + return { order: nextOrder, changed }; +} + +export function useLoomImportJob( + initial: LoomImportSnapshot, + { watchForUpgrade }: { watchForUpgrade: boolean }, +) { + const itemsRef = useRef | null>(null); + const orderRef = useRef([]); + const cursorRef = useRef(initial.cursor); + const jobStatusRef = useRef(initial.job.status); + const hasAllRef = useRef(initial.full); + const fullAtRef = useRef(initial.full ? Date.now() : 0); + const summaryRef = useRef( + initial.summary ?? summarizeLoomImportItems(initial.items), + ); + const queueRef = useRef>(Promise.resolve()); + const pollDelayRef = useRef(ACTIVE_POLL_MS); + const samplesRef = useRef([]); + if (itemsRef.current === null) { + itemsRef.current = new Map(initial.items.map((item) => [item.id, item])); + orderRef.current = initial.items.map((item) => item.id); + } + + const [items, setItems] = useState(initial.items); + const [summary, setSummary] = useState(() => ({ + job: initial.job, + ...summaryRef.current, + })); + const [rate, setRate] = useState(null); + + const apply = useCallback((snapshot: LoomImportSnapshot) => { + const map = itemsRef.current; + if (!map) return; + const merged = mergeLoomImportItems( + map, + orderRef.current, + snapshot, + jobStatusRef.current, + ); + orderRef.current = merged.order; + cursorRef.current = Math.max(cursorRef.current, snapshot.cursor); + jobStatusRef.current = snapshot.job.status; + if (snapshot.full) { + hasAllRef.current = true; + fullAtRef.current = Date.now(); + } + if (snapshot.summary) { + summaryRef.current = snapshot.summary; + } else if (hasAllRef.current && merged.changed) { + summaryRef.current = summarizeLoomImportItems(map.values()); + } + pollDelayRef.current = merged.changed + ? ACTIVE_POLL_MS + : Math.min(IDLE_POLL_MS, Math.round(pollDelayRef.current * 1.5)); + + const now = Date.now(); + const samples = samplesRef.current; + samples.push({ at: now, settled: settledCount(summaryRef.current) }); + while ( + samples.length > 2 && + (samples[0]?.at ?? now) < now - RATE_WINDOW_MS + ) { + samples.shift(); + } + const first = samples[0]; + const last = samples[samples.length - 1]; + const span = first && last ? last.at - first.at : 0; + const perMinute = + first && last && span >= RATE_MIN_SPAN_MS + ? ((last.settled - first.settled) / span) * 60_000 + : 0; + + setSummary({ job: snapshot.job, ...summaryRef.current }); + setRate(perMinute > 0 ? perMinute : null); + if (merged.changed) { + const list: LoomImportItemView[] = []; + for (const id of merged.order) { + const item = map.get(id); + if (item) list.push(item); + } + setItems(list); + } + }, []); + + const fetchSnapshot = useCallback( + (full = false) => { + const run = async () => { + const params = new URLSearchParams({ jobId: initial.job.id }); + if (!full) params.set("since", String(cursorRef.current)); + const response = await fetch(`/api/import/loom/jobs?${params}`, { + cache: "no-store", + credentials: "same-origin", + signal: AbortSignal.timeout(SNAPSHOT_TIMEOUT_MS), + }); + if (!response.ok) + throw new Error(`Snapshot failed: ${response.status}`); + apply((await response.json()) as LoomImportSnapshot); + }; + const request = queueRef.current.then(run, run); + queueRef.current = request.catch(() => undefined); + return request; + }, + [apply, initial.job.id], + ); + + useEffect(() => { + if (initial.full) return; + let stopped = false; + let timer: number | undefined; + const load = (attempt: number) => { + fetchSnapshot(true).catch(() => { + if (stopped || attempt >= 4) return; + timer = window.setTimeout(() => load(attempt + 1), 1000 * 2 ** attempt); + }); + }; + load(0); + return () => { + stopped = true; + window.clearTimeout(timer); + }; + }, [initial.full, fetchSnapshot]); + + const status = summary.job.status; + const shouldPoll = + isLoomImportJobActive(status) || + (status === "awaiting_upgrade" && watchForUpgrade); + + useEffect(() => { + if (!shouldPoll) return; + let stopped = false; + let timer: number | undefined; + let hiddenAt: number | null = null; + + const schedule = () => { + window.clearTimeout(timer); + if (stopped || document.hidden) return; + const wait = + status === "awaiting_upgrade" ? UPGRADE_POLL_MS : pollDelayRef.current; + timer = window.setTimeout(tick, wait); + }; + + const tick = () => { + const full = + hasAllRef.current && Date.now() - fullAtRef.current > FULL_REFRESH_MS; + fetchSnapshot(full) + .catch(() => undefined) + .finally(schedule); + }; + + const onVisibility = () => { + if (document.hidden) { + hiddenAt = Date.now(); + window.clearTimeout(timer); + return; + } + const full = hiddenAt !== null && Date.now() - hiddenAt > 60_000; + hiddenAt = null; + window.clearTimeout(timer); + fetchSnapshot(full) + .catch(() => undefined) + .finally(schedule); + }; + + document.addEventListener("visibilitychange", onVisibility); + schedule(); + return () => { + stopped = true; + window.clearTimeout(timer); + document.removeEventListener("visibilitychange", onVisibility); + }; + }, [shouldPoll, status, fetchSnapshot]); + + const refresh = useCallback(() => fetchSnapshot(true), [fetchSnapshot]); + + return { summary, items, rate, refresh }; +} diff --git a/apps/web/app/(org)/dashboard/import/loom/_components/bulk-import.tsx b/apps/web/app/(org)/dashboard/import/loom/_components/bulk-import.tsx new file mode 100644 index 00000000000..2ab672a552a --- /dev/null +++ b/apps/web/app/(org)/dashboard/import/loom/_components/bulk-import.tsx @@ -0,0 +1,675 @@ +"use client"; + +import { Button, Select } from "@cap/ui"; +import type { Organisation } from "@cap/web-domain"; +import clsx from "clsx"; +import { useRouter } from "next/navigation"; +import { + type ChangeEvent, + type DragEvent, + useId, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { toast } from "sonner"; +import { createLoomImportJobAction } from "@/actions/loom-import"; +import { + buildLoomImportPlan, + type CsvTable, + countFilledRows, + detectLoomImportMapping, + encodeLoomImportRows, + LOOM_CSV_TEMPLATE, + LOOM_IMPORT_ISSUE_LABELS, + LOOM_IMPORT_MAX_ROWS, + type LoomImportField, + type LoomImportMapping, + parseCsv, + tableFromPastedLinks, +} from "@/lib/loom-import/csv"; +import { Doodle, InkArrow, StepMark } from "./doodles"; +import { HowImportWorks } from "./how-import-works"; +import "./loom-import.css"; + +const NOT_IN_FILE = "__not_in_file__"; +const MAX_FILE_BYTES = 8 * 1024 * 1024; +const numberFormat = new Intl.NumberFormat("en-US"); +const plural = (count: number, one: string, many: string) => + `${numberFormat.format(count)} ${count === 1 ? one : many}`; + +type Source = { + name: string; + table: CsvTable; + pasted: boolean; +}; + +const GUIDE = [ + { + doodle: "sheet" as const, + title: "Get your list from Loom", + body: "Admins can export every video from Settings, Workspace, Data, Export. Or just copy your Loom links.", + }, + { + doodle: "check" as const, + title: "Drop it here", + body: "We find the Loom links, owners and spaces on our own, and check every video before anything starts.", + }, + { + doodle: "move" as const, + title: "Follow along live", + body: "Videos keep their titles and recording dates. Close the tab any time, the import keeps going.", + }, +]; + +export const ImportGuide = ({ onHowItWorks }: { onHowItWorks: () => void }) => ( +
+
    + {GUIDE.map((step, index) => ( +
  1. +
    + +
    +
    + +
    +

    {step.title}

    +

    + {step.body} +

    +
    +
    +
    2. + ))} +
+ +
+); + +function downloadTemplate() { + const blob = new Blob([LOOM_CSV_TEMPLATE], { + type: "text/csv;charset=utf-8", + }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = "cap-loom-import-template.csv"; + link.click(); + URL.revokeObjectURL(url); +} + +const SourcePicker = ({ onSource }: { onSource: (source: Source) => void }) => { + const inputRef = useRef(null); + const [isOver, setIsOver] = useState(false); + const [pasteOpen, setPasteOpen] = useState(false); + const [pasteText, setPasteText] = useState(""); + const pasteId = useId(); + + const readFile = async (file: File) => { + if (file.size > MAX_FILE_BYTES) { + toast.error("That file is over 8 MB. Split it into smaller CSVs."); + return; + } + try { + const table = parseCsv(await file.text()); + if (countFilledRows(table) === 0) { + toast.error("That CSV has a header but no rows."); + return; + } + onSource({ name: file.name, table, pasted: false }); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "We couldn't read that CSV.", + ); + } + }; + + const onDrop = (event: DragEvent) => { + event.preventDefault(); + setIsOver(false); + const file = event.dataTransfer.files[0]; + if (file) void readFile(file); + }; + + const onFile = (event: ChangeEvent) => { + const file = event.target.files?.[0]; + if (file) void readFile(file); + event.target.value = ""; + }; + + const usePasted = () => { + const table = tableFromPastedLinks(pasteText); + if (table.rows.length === 0) { + toast.error("We couldn't find any Loom links in that text."); + return; + } + onSource({ name: "Pasted Loom links", table, pasted: true }); + }; + + return ( +
+
{ + event.preventDefault(); + setIsOver(true); + }} + onDragLeave={() => setIsOver(false)} + onDrop={onDrop} + className={clsx( + "li-dropzone relative flex flex-col items-center gap-4 rounded-2xl px-6 py-10 text-center transition-colors duration-200 sm:flex-row sm:gap-8 sm:px-10 sm:text-left", + isOver ? "is-over li-accent-surface" : "bg-gray-1", + )} + > + +
+

+ {isOver ? "Drop to read your CSV" : "Drop your Loom CSV here"} +

+

+ Any CSV with Loom links works, including Loom's own export. Up to{" "} + {numberFormat.format(LOOM_IMPORT_MAX_ROWS)} videos per file. +

+
+ + +
+
+ +
+ + {pasteOpen && ( +
+ +