diff --git a/apps/media-server/src/__tests__/lib/hls-ladder-encode.integration.test.ts b/apps/media-server/src/__tests__/lib/hls-ladder-encode.integration.test.ts index ce9237915e6..4f65407684b 100644 --- a/apps/media-server/src/__tests__/lib/hls-ladder-encode.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/hls-ladder-encode.integration.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; import { execFileSync } from "node:child_process"; import { copyFileSync, @@ -26,8 +26,12 @@ import { registerMediaOperation, unregisterMediaOperation, } from "../../lib/media-operations"; +import * as mediaProbe from "../../lib/media-probe"; import { probeVideoFile } from "../../lib/media-probe"; -import { generateThumbnail } from "../../lib/media-video"; +import { + generateThumbnail, + sourceAudioCopiesToHls, +} from "../../lib/media-video"; import { getActiveSubprocessCount } from "../../lib/subprocess"; const SOURCE_SECONDS = 26; @@ -355,6 +359,29 @@ describe("HLS ladder encode", () => { } }, 60_000); + test("fails a slice instead of guessing the frame rate when its frame timing cannot be read", async () => { + const metadata = await probeVideoFile(source); + const timing = spyOn(mediaProbe, "probeFastestFrameRate").mockRejectedValue( + new Error("frame timing could not be read"), + ); + try { + await expect( + encodeHlsLadder(source, metadata, { + range: { startSeconds: 4, durationSeconds: 4 }, + }), + ).rejects.toThrow("frame timing could not be read"); + + const whole = await encodeHlsLadder(source, metadata); + try { + expect(whole.result.renditions.length).toBeGreaterThan(0); + } finally { + await whole.cleanup(); + } + } finally { + timing.mockRestore(); + } + }, 120_000); + test("starts a slice on time when it begins inside a still stretch", async () => { const still = join(directory, "still.mp4"); writeFiltered(still, SOURCE_SECONDS, "not(between(t,13,20))"); @@ -392,6 +419,156 @@ describe("HLS ladder encode", () => { expect(bounded.at(-1)).toBeCloseTo(wholeBetween.at(-1) ?? Number.NaN, 3); }, 180_000); + function audioPackets(path: string) { + return execFileSync( + "ffprobe", + [ + "-v", + "error", + "-allowed_extensions", + "ALL", + "-select_streams", + "a:0", + "-show_entries", + "packet=size", + "-of", + "csv=p=0", + path, + ], + { encoding: "utf8" }, + ) + .split("\n") + .map((line) => line.split(",")[0] ?? "") + .filter((line) => line.length > 0); + } + + function audioChannels(path: string) { + return execFileSync( + "ffprobe", + [ + "-v", + "error", + "-allowed_extensions", + "ALL", + "-select_streams", + "a:0", + "-show_entries", + "stream=channels", + "-of", + "csv=p=0", + path, + ], + { encoding: "utf8" }, + ) + .split("\n") + .map((line) => line.trim()) + .find((line) => line.length > 0); + } + + test("copies stereo AAC-LC into a slice's audio packet for packet", async () => { + const stereo = join(directory, "stereo.mp4"); + execFileSync("ffmpeg", [ + "-v", + "error", + "-i", + source, + "-f", + "lavfi", + "-i", + `sine=frequency=330:duration=${SOURCE_SECONDS}`, + "-filter_complex", + "[0:a][1:a]join=inputs=2:channel_layout=stereo[st]", + "-map", + "0:v", + "-map", + "[st]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "160k", + "-y", + stereo, + ]); + expect(await sourceAudioCopiesToHls(stereo)).toBe(true); + + const first = await encode( + { startSeconds: 0, durationSeconds: 16 }, + true, + stereo, + ); + const audio = fileNamed(first, "audio.m3u8").path; + expect(audioPackets(audio)).toEqual(audioPackets(stereo)); + expect(audioChannels(audio)).toBe("2"); + + // The copied audio plays against the stitched video like encoded audio. + const second = await encode({ startSeconds: 16 }, false, stereo); + const output = stitch([first, second], "stitched-copied-audio"); + const audioSeconds = streamSeconds(join(output, "master.m3u8"), "a:0"); + const videoSeconds = streamSeconds(join(output, "master.m3u8"), "v:0"); + expect(audioSeconds).toBeGreaterThan(SOURCE_SECONDS - 1); + expect(Math.abs(audioSeconds - videoSeconds)).toBeLessThan(0.5); + }, 180_000); + + test("still encodes audio that is not stereo AAC-LC", async () => { + expect(await sourceAudioCopiesToHls(source)).toBe(false); + const slice = await encode({ startSeconds: 0, durationSeconds: 16 }); + const audio = fileNamed(slice, "audio.m3u8").path; + expect(audioChannels(audio)).toBe("2"); + expect(audioPackets(audio)).not.toEqual(audioPackets(source)); + }, 120_000); + + test("copies only stereo AAC-LC at a common rate and bitrate", async () => { + const make = (name: string, args: string[], extension = "mp4") => { + const path = join(directory, `${name}.${extension}`); + execFileSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=2", + "-f", + "lavfi", + "-i", + "sine=frequency=330:duration=2", + "-filter_complex", + "[0:a][1:a]join=inputs=2:channel_layout=stereo[st]", + "-map", + "[st]", + ...args, + "-y", + path, + ]); + return path; + }; + expect( + await sourceAudioCopiesToHls(make("ok", ["-c:a", "aac", "-b:a", "128k"])), + ).toBe(true); + expect( + await sourceAudioCopiesToHls( + make("loud", ["-c:a", "aac", "-b:a", "256k"]), + ), + ).toBe(false); + expect( + await sourceAudioCopiesToHls( + make("rate", ["-c:a", "aac", "-b:a", "96k", "-ar", "32000"]), + ), + ).toBe(false); + expect( + await sourceAudioCopiesToHls( + make("surround", ["-c:a", "aac", "-b:a", "128k", "-ac", "6"]), + ), + ).toBe(false); + expect( + await sourceAudioCopiesToHls(make("opus", ["-c:a", "libopus"], "webm")), + ).toBe(false); + expect(await sourceAudioCopiesToHls(join(directory, "missing.mp4"))).toBe( + false, + ); + }, 60_000); + test("keeps audio in a codec the probe cannot name", async () => { const lossless = join(directory, "lossless.mov"); execFileSync("ffmpeg", [ diff --git a/apps/media-server/src/__tests__/lib/hls-ladder-route.integration.test.ts b/apps/media-server/src/__tests__/lib/hls-ladder-route.integration.test.ts index cc0081fa747..8f3b3a357fb 100644 --- a/apps/media-server/src/__tests__/lib/hls-ladder-route.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/hls-ladder-route.integration.test.ts @@ -25,6 +25,7 @@ const LADDER_FILES = [ let app: typeof appType; let getJob: typeof import("../../lib/job-manager").getJob; +let jobManager: typeof import("../../lib/job-manager"); let server: ReturnType; let baseUrl = ""; const workDir = mkdtempSync(join(tmpdir(), "cap-ladder-route-")); @@ -67,7 +68,8 @@ beforeAll(async () => { process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET; process.env.MEDIA_SERVER_MAX_CONCURRENT_VIDEO_PROCESSES = "4"; app = (await import("../../app")).default; - getJob = (await import("../../lib/job-manager")).getJob; + jobManager = await import("../../lib/job-manager"); + getJob = jobManager.getJob; const fixture = readFileSync(FIXTURE); const largePath = join(workDir, "large.mp4"); execFileSync("ffmpeg", [ @@ -242,6 +244,54 @@ describe("POST /video/ladder", () => { expect(uploads.has("slice/master.m3u8")).toBe(true); }, 120_000); + // With two slots or fewer the slice reserve and the reserve for a whole + // Infra job are the same slot (infra-capacity.test.ts covers the rule + // itself), so this needs a machine with room for three jobs. + test("holds an Infra slice back once half the worker's slots are taken and still takes a whole Infra job", async () => { + const { effectiveMax } = jobManager.getSystemResources(); + if (effectiveMax <= 2) { + console.warn( + `[test] ${effectiveMax} slots here, the slice reserve is not separate`, + ); + return; + } + const held = Array.from( + { length: Math.floor(effectiveMax / 2) }, + (_, index) => `held-${index}`, + ); + for (const jobId of held) jobManager.createJob(jobId, "other", "user-1"); + try { + const slice = await startLadder({ + uploads: uploadTargets("held-slice"), + range: { startSeconds: 0, durationSeconds: 4 }, + infra: true, + }); + expect(slice.status).toBe(503); + expect(slice.headers.get("retry-after")).not.toBeNull(); + expect(uploads.has("held-slice/master.m3u8")).toBe(false); + + const whole = await startLadder({ + uploads: uploadTargets("held-whole"), + infra: true, + }); + expect(whole.status).toBe(200); + const { jobId } = (await whole.json()) as { jobId: string }; + expect((await waitForTerminalJob(jobId)).phase).toBe("complete"); + + jobManager.deleteJob(held[0] as string); + const admitted = await startLadder({ + uploads: uploadTargets("held-slice"), + range: { startSeconds: 0, durationSeconds: 4 }, + infra: true, + }); + expect(admitted.status).toBe(200); + const sliceJob = (await admitted.json()) as { jobId: string }; + expect((await waitForTerminalJob(sliceJob.jobId)).phase).toBe("complete"); + } finally { + for (const jobId of held) jobManager.deleteJob(jobId); + } + }, 180_000); + test("fails the job when a produced file has no upload target", async () => { const response = await startLadder({ uploads: uploadTargets("partial", ["master.m3u8"]), diff --git a/apps/media-server/src/__tests__/lib/hls-ladder.test.ts b/apps/media-server/src/__tests__/lib/hls-ladder.test.ts index 3b611a2e4e8..13ca625c8fd 100644 --- a/apps/media-server/src/__tests__/lib/hls-ladder.test.ts +++ b/apps/media-server/src/__tests__/lib/hls-ladder.test.ts @@ -758,7 +758,7 @@ describe("stitchHlsLadderSlices", () => { test("advertises the peak and the time-weighted average bitrate with audio", () => { const stitched = stitchHlsLadderSlices(slices, { hasAudio: true }); expect(stitched.master).toContain( - '#EXT-X-STREAM-INF:BANDWIDTH=6128000,AVERAGE-BANDWIDTH=2218909,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud"', + '#EXT-X-STREAM-INF:BANDWIDTH=6192000,AVERAGE-BANDWIDTH=2218909,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud"', ); expect(stitched.master).toContain('URI="audio.m3u8"'); }); diff --git a/apps/media-server/src/__tests__/lib/infra-capacity.test.ts b/apps/media-server/src/__tests__/lib/infra-capacity.test.ts index 99cc3e9ceac..ffd3b8eabf5 100644 --- a/apps/media-server/src/__tests__/lib/infra-capacity.test.ts +++ b/apps/media-server/src/__tests__/lib/infra-capacity.test.ts @@ -116,6 +116,91 @@ describe("capacity for Cap Infra jobs", () => { expect(allowsWithReserve(1, 0)).toBe(true); }); + test("a slice keeps half of the worker's slots free, and never less than the reserve", () => { + const sliceAllows = ( + effectiveMax: number, + active: number, + reservedSlots = 1, + ) => + jobManager.infraCapacityAllows({ + effectiveMax, + active, + activeHeavyLadders: 0, + heavyLadder: false, + reservedSlots: jobManager.infraSliceReservedSlots( + effectiveMax, + reservedSlots, + ), + maxHeavyLadders: 1, + }); + expect(jobManager.infraSliceReservedSlots(4, 1)).toBe(2); + expect(jobManager.infraSliceReservedSlots(5, 1)).toBe(3); + expect(jobManager.infraSliceReservedSlots(4, 3)).toBe(3); + expect(sliceAllows(4, 1)).toBe(true); + expect(sliceAllows(4, 2)).toBe(false); + expect(sliceAllows(5, 2)).toBe(false); + expect(sliceAllows(6, 2)).toBe(true); + expect(sliceAllows(6, 3)).toBe(false); + expect(sliceAllows(2, 0)).toBe(true); + expect(sliceAllows(2, 1)).toBe(false); + expect(sliceAllows(1, 0)).toBe(true); + expect(sliceAllows(1, 1)).toBe(false); + expect(sliceAllows(4, 0, 3)).toBe(true); + expect(sliceAllows(4, 1, 3)).toBe(false); + }); + + test("a heavy slice still waits for the running heavy ladder", () => { + expect( + jobManager.infraCapacityAllows({ + effectiveMax: 8, + active: 1, + activeHeavyLadders: 1, + heavyLadder: true, + reservedSlots: jobManager.infraSliceReservedSlots(8, 1), + maxHeavyLadders: 1, + }), + ).toBe(false); + }); + + test("a slice stays off a worker that is making an Infra MP4", () => { + const sliceAllows = (activeInfraMp4s: number) => + jobManager.infraCapacityAllows({ + effectiveMax: 8, + active: 1, + activeHeavyLadders: 0, + heavyLadder: true, + reservedSlots: jobManager.infraSliceReservedSlots(8, 1), + maxHeavyLadders: 1, + slice: true, + activeInfraMp4s, + }); + expect(sliceAllows(0)).toBe(true); + expect(sliceAllows(1)).toBe(false); + }); + + test("a whole-video ladder still runs beside an Infra MP4", () => { + expect( + jobManager.infraCapacityAllows({ + effectiveMax: 8, + active: 1, + activeHeavyLadders: 0, + heavyLadder: true, + maxHeavyLadders: 1, + activeInfraMp4s: 1, + }), + ).toBe(true); + }); + + test("counts only Infra MP4s that are still running", () => { + const before = jobManager.getActiveInfraMp4Count(); + const mp4 = addActiveJob(); + mp4.infraMp4 = true; + addActiveJob(true); + expect(jobManager.getActiveInfraMp4Count()).toBe(before + 1); + jobManager.updateJob(mp4.jobId, { phase: "complete", progress: 100 }); + expect(jobManager.getActiveInfraMp4Count()).toBe(before); + }); + test("counts only heavy ladders that are still running", () => { const before = jobManager.getActiveInfraHeavyLadderCount(); const job = addActiveJob(true); @@ -139,7 +224,7 @@ describe("routes", () => { expect(response.status).toBe(503); expect((await response.json()).code).toBe("SERVER_BUSY"); expect(response.headers.get("Retry-After")).not.toBeNull(); - expect(infra).toHaveBeenCalledWith(true); + expect(infra).toHaveBeenCalledWith(true, false); }); test("a 1080p Infra ladder is not counted as heavy", async () => { @@ -151,7 +236,27 @@ describe("routes", () => { await app.fetch( request("/video/ladder", ladderBody({ infra: true, maxTier: "1080p" })), ); - expect(infra).toHaveBeenCalledWith(false); + expect(infra).toHaveBeenCalledWith(false, false); + }); + + test("an Infra ladder slice is held to the slice reserve", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + const infra = spyOn( + jobManager, + "canAcceptNewInfraVideoProcess", + ).mockReturnValue(false); + const response = await app.fetch( + request( + "/video/ladder", + ladderBody({ + infra: true, + range: { startSeconds: 20, durationSeconds: 96 }, + }), + ), + ); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).not.toBeNull(); + expect(infra).toHaveBeenCalledWith(true, true); }); test("an Infra MP4 over its limit gets the busy answer", async () => { @@ -167,6 +272,25 @@ describe("routes", () => { expect(infra).toHaveBeenCalledWith(false); }); + test("an Infra MP4 is marked so slices stay off its worker", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + spyOn(jobManager, "canAcceptNewInfraVideoProcess").mockReturnValue(true); + spyOn(jobManager, "getJob").mockReturnValue(undefined); + const create = spyOn(jobManager, "createJob"); + const infra = await app.fetch( + request("/video/process", processBody({ infra: true })), + ); + const own = await app.fetch(request("/video/process", processBody())); + expect(infra.status).toBe(200); + expect(own.status).toBe(200); + const [infraJob, ownJob] = create.mock.results.map( + (result) => result.value as jobManager.Job, + ); + created.push(infraJob.jobId, ownJob.jobId); + expect(infraJob.infraMp4).toBe(true); + expect(ownJob.infraMp4).toBeUndefined(); + }); + test("Cap's own jobs never consult the Infra limits", async () => { spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(false); const infra = spyOn( diff --git a/apps/media-server/src/lib/hls-ladder-encode.ts b/apps/media-server/src/lib/hls-ladder-encode.ts index 2905a0c3ea8..f2c7d9600e6 100644 --- a/apps/media-server/src/lib/hls-ladder-encode.ts +++ b/apps/media-server/src/lib/hls-ladder-encode.ts @@ -26,6 +26,7 @@ import { getProcessTimeoutMs, requireVideoSourceTraits, runTranscodeCommand, + sourceAudioCopiesToHls, sourceHasAudio, } from "./media-video"; import { @@ -98,6 +99,10 @@ export async function encodeHlsLadder( // this one. Falling back would quietly encode a variable frame rate // source on its opening rate whenever the server is busy. if (error instanceof ProbeBusyError) throw error; + // A slice has to land on the frame grid its neighbours use. Guessing + // the rate here while another worker measured it would put the slices + // on different grids, so the slice fails and is sent again. + if (options.range) throw error; if (!probeSignal.aborted) { console.warn( "[hls-ladder] Could not read frame timing, using the probed frame rate:", @@ -216,13 +221,23 @@ export async function encodeHlsLadder( const audioPlaylistName = `${HLS_LADDER_AUDIO_NAME}.m3u8`; const separateAudio = wantsAudio && range - ? runTranscodeCommand( - buildHlsAudioArgs({ inputPath, outputDir: dir, maxOutputSeconds }), - join(dir, audioPlaylistName), - metadata.duration * 1_000_000, - getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), - undefined, - signal, + ? // The whole video's audio rides with the first slice, so encoding it + // would hold that slice to the speed of one AAC encode (about 90 s + // for an hour). Stereo AAC-LC is copied instead, in seconds. + sourceAudioCopiesToHls(inputPath, signal).then((copy) => + runTranscodeCommand( + buildHlsAudioArgs({ + inputPath, + outputDir: dir, + maxOutputSeconds, + copy, + }), + join(dir, audioPlaylistName), + metadata.duration * 1_000_000, + getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), + undefined, + signal, + ), ) : Promise.resolve(); const video = encodeVideo(); diff --git a/apps/media-server/src/lib/hls-ladder.ts b/apps/media-server/src/lib/hls-ladder.ts index 5b481666246..194c9497157 100644 --- a/apps/media-server/src/lib/hls-ladder.ts +++ b/apps/media-server/src/lib/hls-ladder.ts @@ -12,6 +12,7 @@ export const HLS_LADDER_AUDIO_NAME = "audio"; export const HLS_LADDER_SEGMENT_SECONDS = 4; export const HLS_LADDER_KEYFRAME_SECONDS = 2; export const HLS_LADDER_AUDIO_BITRATE_KBPS = 128; +export const HLS_LADDER_COPIED_AUDIO_MAX_KBPS = 192; const MAX_ENCODE_DIMENSION = 4096; const MAX_LADDER_FPS = 60; @@ -545,6 +546,8 @@ export function buildHlsAudioArgs(input: { outputDir: string; extraInputArgs?: string[]; maxOutputSeconds?: number | null; + // Stereo AAC-LC from the source, kept as it is (see sourceAudioCopiesToHls). + copy?: boolean; }): string[] { return [ "ffmpeg", @@ -555,12 +558,16 @@ export function buildHlsAudioArgs(input: { "-vn", "-map", "0:a:0", - "-c:a", - "aac", - "-b:a", - `${HLS_LADDER_AUDIO_BITRATE_KBPS}k`, - "-ac", - "2", + ...(input.copy + ? ["-c:a", "copy"] + : [ + "-c:a", + "aac", + "-b:a", + `${HLS_LADDER_AUDIO_BITRATE_KBPS}k`, + "-ac", + "2", + ]), ...outputLengthCapArgs(input.maxOutputSeconds), "-hls_segment_options", "movflags=+frag_discont", @@ -674,6 +681,11 @@ export function stitchHlsLadderSlices( const audioBandwidth = options.hasAudio ? HLS_LADDER_AUDIO_BITRATE_KBPS * 1000 : 0; + // A slice's audio may be stereo AAC-LC copied from the source, up to this + // bitrate, instead of encoded at HLS_LADDER_AUDIO_BITRATE_KBPS. + const audioPeak = options.hasAudio + ? HLS_LADDER_COPIED_AUDIO_MAX_KBPS * 1000 + : 0; const playlists: Record = {}; const media: StitchedHlsLadder["media"] = []; const master = ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-INDEPENDENT-SEGMENTS"]; @@ -719,7 +731,7 @@ export function stitchHlsLadderSlices( .filter((codec): codec is string => Boolean(codec)) .join(","); const attributes = [ - `BANDWIDTH=${peak + audioBandwidth}`, + `BANDWIDTH=${peak + audioPeak}`, `AVERAGE-BANDWIDTH=${Math.round(weightedAverage / totalSeconds) + audioBandwidth}`, ...(rendition.width && rendition.height ? [`RESOLUTION=${rendition.width}x${rendition.height}`] diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 771e3dbd661..7b5b16ae351 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -163,6 +163,7 @@ export interface Job { // A ladder for a Cap Infra video at the 1440p or 2160p tier, which keeps the // GPU encoder busy on its own. infraHeavyLadder?: boolean; + infraMp4?: boolean; } const jobs = new Map(); @@ -384,6 +385,14 @@ export function getActiveInfraHeavyLadderCount(): number { return count; } +export function getActiveInfraMp4Count(): number { + let count = 0; + for (const job of jobs.values()) { + if (job.infraMp4 && isActivePhase(job.phase)) count++; + } + return count; +} + export function infraCapacityAllows(input: { effectiveMax: number; active: number; @@ -391,7 +400,10 @@ export function infraCapacityAllows(input: { heavyLadder: boolean; reservedSlots?: number; maxHeavyLadders?: number; + slice?: boolean; + activeInfraMp4s?: number; }): boolean { + if (input.slice && (input.activeInfraMp4s ?? 0) > 0) return false; const reservedSlots = input.reservedSlots ?? INFRA_RESERVED_SLOTS; const maxHeavyLadders = input.maxHeavyLadders ?? MAX_INFRA_HEAVY_LADDERS; const reserved = Math.min(reservedSlots, Math.max(0, input.effectiveMax - 1)); @@ -399,12 +411,37 @@ export function infraCapacityAllows(input: { return !input.heavyLadder || input.activeHeavyLadders < maxHeavyLadders; } -export function canAcceptNewInfraVideoProcess(heavyLadder: boolean): boolean { +// The slices of one long Infra video arrive several at a time, so a slice +// leaves at least half of the worker's slots free instead of only the +// reserved ones, and stays off a worker that is making an Infra MP4: the MP4 +// is sent before the slices and must finish as fast as it would beside a +// whole-video ladder, which runs on a worker of its own. A slice turned away +// is sent again by its caller and reaches another worker through the load +// balancer. +export function infraSliceReservedSlots( + effectiveMax: number, + reservedSlots = INFRA_RESERVED_SLOTS, +): number { + return Math.max(reservedSlots, Math.ceil(effectiveMax / 2)); +} + +export function canAcceptNewInfraVideoProcess( + heavyLadder: boolean, + slice = false, +): boolean { + const { effectiveMax } = getSystemResources(); return infraCapacityAllows({ - effectiveMax: getSystemResources().effectiveMax, + effectiveMax, active: getActiveVideoProcessCount(), activeHeavyLadders: getActiveInfraHeavyLadderCount(), heavyLadder, + ...(slice + ? { + reservedSlots: infraSliceReservedSlots(effectiveMax), + slice, + activeInfraMp4s: getActiveInfraMp4Count(), + } + : {}), }); } diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 28c8897f779..acad0cae372 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -19,6 +19,7 @@ import { hdrTransferFrom, SDR_COLOR_ARGS, } from "./hdr"; +import { HLS_LADDER_COPIED_AUDIO_MAX_KBPS } from "./hls-ladder"; import type { VideoMetadata } from "./job-manager"; import { createGuardedMediaFetch, @@ -1594,6 +1595,7 @@ async function probeVideoStreamEntries( showEntries: string, abortSignal?: AbortSignal, keyed = false, + stream = "v:0", ): Promise { const proc = registerSubprocess( spawn({ @@ -1602,7 +1604,7 @@ async function probeVideoStreamEntries( "-v", "error", "-select_streams", - "v:0", + stream, "-show_entries", showEntries, "-of", @@ -1739,6 +1741,39 @@ async function readVideoSourceTraits( }; } +const HLS_COPY_AUDIO_SAMPLE_RATES = new Set(["44100", "48000"]); +const HLS_COPY_AUDIO_MAX_BITRATE = HLS_LADDER_COPIED_AUDIO_MAX_KBPS * 1000; + +// Whether the source's first audio track can go into an HLS audio rendition as +// it is. The playlists name that rendition AAC-LC with two channels, so only +// stereo AAC-LC qualifies, at a common rate and no more than the bitrate the +// playlists allow for. Anything else, or a probe that fails, is encoded. +export async function sourceAudioCopiesToHls( + inputPath: string, + abortSignal?: AbortSignal, +): Promise { + const stdout = await probeVideoStreamEntries( + inputPath, + "stream=codec_name,profile,channels,sample_rate,bit_rate", + abortSignal, + true, + "a:0", + ); + if (stdout === null) return false; + const value = (key: string) => + stdout.match(new RegExp(`^${key}=(.*)$`, "m"))?.[1]?.trim() ?? null; + const bitrate = Number(value("bit_rate")); + return ( + value("codec_name") === "aac" && + value("profile") === "LC" && + value("channels") === "2" && + HLS_COPY_AUDIO_SAMPLE_RATES.has(value("sample_rate") ?? "") && + Number.isFinite(bitrate) && + bitrate > 0 && + bitrate <= HLS_COPY_AUDIO_MAX_BITRATE + ); +} + // One ffprobe for the two things that decide how a source is decoded: a // rotated source cannot be decoded on the GPU, and an HDR source has to be // converted to standard range on the CPU. diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index e084849a975..74c4cdefeca 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -984,6 +984,7 @@ video.on("POST", ["/process", "/import"], async (c) => { const jobId = generateJobId(); const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret); + if (result.data.infra === true) job.infraMp4 = true; void trackBackgroundTask( withMediaTransfers(32 * 1024 ** 3, () => @@ -1062,7 +1063,10 @@ video.post("/ladder", async (c) => { if ( !canAcceptNewVideoProcess() || (result.data.infra === true && - !canAcceptNewInfraVideoProcess(infraHeavyLadder)) + !canAcceptNewInfraVideoProcess( + infraHeavyLadder, + result.data.range !== undefined, + )) ) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503); diff --git a/apps/web/__tests__/fixtures/developer-video-slices.ts b/apps/web/__tests__/fixtures/developer-video-slices.ts new file mode 100644 index 00000000000..af8234acedf --- /dev/null +++ b/apps/web/__tests__/fixtures/developer-video-slices.ts @@ -0,0 +1,137 @@ +import type { + DeveloperVideoPartState, + DeveloperVideoSlice, + DeveloperVideoSlicedHls, + DeveloperVideoSliceResult, +} from "@cap/database/types"; + +// A slice's playlist as the encoder writes it: one media file read by byte +// range behind one init segment. +export function slicePlaylist(stream: string, segments: number[]): string { + let offset = 800; + const lines = [ + "#EXTM3U", + "#EXT-X-VERSION:7", + `#EXT-X-TARGETDURATION:${Math.ceil(Math.max(...segments))}`, + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:VOD", + "#EXT-X-INDEPENDENT-SEGMENTS", + `#EXT-X-MAP:URI="${stream}.m4s",BYTERANGE="800@0"`, + ]; + for (const seconds of segments) { + lines.push( + `#EXTINF:${seconds.toFixed(6)},`, + `#EXT-X-BYTERANGE:1000@${offset}`, + `${stream}.m4s`, + ); + offset += 1000; + } + lines.push("#EXT-X-ENDLIST", ""); + return lines.join("\n"); +} + +export function sliceMaster(streams: string[]): string { + return [ + "#EXTM3U", + "#EXT-X-VERSION:7", + ...streams.flatMap((stream) => [ + "#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1920x1080", + `${stream}.m3u8`, + ]), + "", + ].join("\n"); +} + +export function sliceResult( + seconds: number, + options: { audioBytes?: number; scale?: number } = {}, +): DeveloperVideoSliceResult { + const scale = options.scale ?? 1; + return { + seconds, + audioBytes: options.audioBytes ?? 0, + renditions: [ + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 4_000_000 * scale, + averageBandwidth: 3_000_000 * scale, + bytes: 5_000 * scale, + codecs: "avc1.640028", + }, + { + name: "360p", + width: 640, + height: 360, + bandwidth: 500_000 * scale, + averageBandwidth: 400_000 * scale, + bytes: 700 * scale, + codecs: "avc1.64001e", + }, + ], + }; +} + +export function sliceLadder(result: DeveloperVideoSliceResult, audioBytes = 0) { + return { + encoder: "libx264", + renditions: result.renditions, + audio: audioBytes > 0 ? { bytes: audioBytes } : null, + }; +} + +export const runningJob = ( + token: string, + overrides: Partial< + Extract + > = {}, +): DeveloperVideoPartState => ({ + state: "running", + token, + jobId: `job-${token}`, + startedAt: 2_000, + ...overrides, +}); + +export function slice( + startSeconds: number, + durationSeconds: number | null, + overrides: Partial = {}, +): DeveloperVideoSlice { + return { + startSeconds, + durationSeconds, + job: { state: "pending" }, + attempts: 0, + lostJobs: 0, + ...overrides, + }; +} + +export function finishedSlice( + startSeconds: number, + durationSeconds: number | null, + token: string, + result: DeveloperVideoSliceResult, +): DeveloperVideoSlice { + return slice(startSeconds, durationSeconds, { + job: { state: "complete", output: token }, + attempts: 1, + result, + }); +} + +// A 300 second video: a 20 second first slice and three more. +export function slicedHls( + overrides: Partial = {}, +): DeveloperVideoSlicedHls { + return { + state: "encoding", + output: "sliced-1", + exposed: null, + hasAudio: false, + slices: [slice(0, 20), slice(20, 96), slice(116, 96), slice(212, null)], + ...overrides, + }; +} diff --git a/apps/web/__tests__/fixtures/developer-video.ts b/apps/web/__tests__/fixtures/developer-video.ts index 5aa505df2ee..b9ceed6a0a6 100644 --- a/apps/web/__tests__/fixtures/developer-video.ts +++ b/apps/web/__tests__/fixtures/developer-video.ts @@ -170,6 +170,7 @@ export async function createDeveloperVideoDatabase(store: DeveloperVideoStore) { }), }); + let queue: Promise = Promise.resolve(); const database = { select, selectDistinct, @@ -181,8 +182,13 @@ export async function createDeveloperVideoDatabase(store: DeveloperVideoStore) { }, }), }), - transaction: async (run: (tx: unknown) => Promise) => - run(database), + // A transaction that reads the video for update holds its row lock, so + // two of them never overlap. The fake runs them one after the other. + transaction: (run: (tx: unknown) => Promise) => { + const result = queue.then(() => run(database)); + queue = result.catch(() => undefined); + return result; + }, }; return database; } diff --git a/apps/web/__tests__/unit/developer-video-cancel.test.ts b/apps/web/__tests__/unit/developer-video-cancel.test.ts index 37720c55e78..5cc7afb3f09 100644 --- a/apps/web/__tests__/unit/developer-video-cancel.test.ts +++ b/apps/web/__tests__/unit/developer-video-cancel.test.ts @@ -85,6 +85,37 @@ describe("runningDeveloperVideoJobIds", () => { ).toEqual([]); expect(runningDeveloperVideoJobIds(null)).toEqual([]); }); + + it("returns the job of every running slice while the hls part waits", () => { + const job = (jobId: string | null) => ({ + startSeconds: 0, + durationSeconds: 20, + job: running(jobId), + attempts: 1, + lostJobs: 0, + }); + expect( + runningDeveloperVideoJobIds( + processingState({ + hls: { state: "pending" }, + mp4: running("job-mp4"), + sliced: { + state: "encoding", + output: "sliced-1", + exposed: true, + hasAudio: true, + slices: [ + { ...job(null), job: { state: "complete", output: "a" } }, + job("slice-job-1"), + job(null), + { ...job(null), job: { state: "pending" } }, + job("slice-job-4"), + ], + }, + }), + ).sort(), + ).toEqual(["job-mp4", "slice-job-1", "slice-job-4"]); + }); }); describe("cancelDeveloperVideoJobs", () => { @@ -163,6 +194,9 @@ describe("cancelDeveloperVideoEncodes", () => { expect(query.sql).toContain( "JSON_UNQUOTE(JSON_EXTRACT(`developer_videos`.`processing`, '$.mp4.state')) = 'running'", ); + expect(query.sql).toContain( + "JSON_SEARCH(`developer_videos`.`processing`, 'one', 'running', NULL, '$.sliced.slices[*].job.state') IS NOT NULL", + ); expect(query.sql).toContain("`developer_videos`.`deletedAt` is not null"); expect(query.sql).toContain("`developer_videos`.`deletedAt` >= ?"); expect(query.params).toEqual(["app-1", "2026-11-13 06:00:00.000"]); diff --git a/apps/web/__tests__/unit/developer-video-sliced-playback.test.ts b/apps/web/__tests__/unit/developer-video-sliced-playback.test.ts new file mode 100644 index 00000000000..c2b8ef799a5 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-sliced-playback.test.ts @@ -0,0 +1,431 @@ +import type { + DeveloperVideoPartState, + DeveloperVideoSlicedHls, +} from "@cap/database/types"; +import type { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; +import { + finishedSlice, + runningJob, + slice, + slicedHls, + slicePlaylist, + sliceResult, +} from "../fixtures/developer-video-slices"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + readText: vi.fn(), + sign: vi.fn(), + standing: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const database = await createDeveloperVideoDatabase(mocks.store); + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); +vi.mock("@/lib/infra-standing", () => ({ + getInfraAppStanding: mocks.standing, +})); +vi.mock("@/lib/developer-video-storage", () => ({ + readDeveloperVideoText: mocks.readText, + signedDeveloperVideoUrl: mocks.sign, +})); + +const { GET } = await import( + "@/app/api/developer/playback/[playbackId]/[file]/route" +); +const { serializeDeveloperVideo, serializeDeveloperVideoStatus } = await import( + "@/lib/developer-video-api" +); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const FOLDER = "developer/app-1/video-1/encode-1/sliced-1"; +const BASE = `https://web.example/api/developer/playback/${PLAYBACK_ID}`; + +const request = new Request(`${BASE}/master.m3u8`) as NextRequest; + +function play(file: string) { + return GET(request, { + params: Promise.resolve({ playbackId: PLAYBACK_ID, file }), + }); +} + +// The first two slices of a video are in, the third is encoding. +const growing = (overrides: Partial = {}) => + slicedHls({ + exposed: true, + hasAudio: true, + slices: [ + finishedSlice(0, 8, "a", sliceResult(8, { audioBytes: 300 })), + finishedSlice(8, 8, "b", sliceResult(8)), + slice(16, null, { job: runningJob("c") }), + ], + ...overrides, + }); + +const whole = () => + growing({ + slices: [ + finishedSlice(0, 8, "a", sliceResult(8, { audioBytes: 300 })), + finishedSlice(8, 8, "b", sliceResult(8)), + finishedSlice(16, null, "c", sliceResult(6)), + ], + }); + +function serve( + sliced: DeveloperVideoSlicedHls, + hls: DeveloperVideoPartState = { state: "pending" }, +) { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ hls, sliced }), + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.store.appDeleted = false; + mocks.standing.mockResolvedValue({ state: "active" }); + mocks.sign.mockImplementation( + async (key: string) => `https://cdn/${key}?sig`, + ); + mocks.readText.mockImplementation(async (key: string) => { + const stored: Record = { + [`${FOLDER}/slices/1080p-0-a.m3u8`]: slicePlaylist("1080p", [4, 4]), + [`${FOLDER}/slices/1080p-1-b.m3u8`]: slicePlaylist("1080p", [4, 4]), + [`${FOLDER}/slices/1080p-2-c.m3u8`]: slicePlaylist("1080p", [4, 2]), + [`${FOLDER}/slices/audio-0-a.m3u8`]: slicePlaylist( + "audio", + [4.01, 4.01, 4.01, 4.01, 4.01, 1.9], + ), + }; + return stored[key] ?? null; + }); + serve(growing()); +}); + +describe("a ladder that is still being encoded in slices", () => { + it("is offered through the same playback links as a finished one", () => { + const video = mocks.store.video; + if (!video) throw new Error("no video"); + + expect(serializeDeveloperVideo(video).hlsUrl).toBe(`${BASE}/master.m3u8`); + expect(serializeDeveloperVideoStatus(video)).toMatchObject({ + ready: true, + status: "ready", + hlsUrl: `${BASE}/master.m3u8`, + mp4Url: null, + }); + }); + + it("serves a master playlist built from the slices that are in", async () => { + const response = await play("master.m3u8"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe( + "application/vnd.apple.mpegurl", + ); + expect(await response.text()).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-INDEPENDENT-SEGMENTS", + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="audio",DEFAULT=YES,AUTOSELECT=YES,CHANNELS="2",URI="audio.m3u8"', + '#EXT-X-STREAM-INF:BANDWIDTH=692000,AVERAGE-BANDWIDTH=528000,RESOLUTION=640x360,CODECS="avc1.64001e,mp4a.40.2",AUDIO="aud"', + "360p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=4192000,AVERAGE-BANDWIDTH=3128000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud"', + "1080p.m3u8", + "", + ].join("\n"), + ); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("serves a rendition as an event playlist with no end, starting at zero", async () => { + const response = await play("1080p.m3u8"); + const a = `https://cdn/${FOLDER}/hls/1080p-0-a.m4s?sig`; + const b = `https://cdn/${FOLDER}/hls/1080p-1-b.m4s?sig`; + + expect(response.status).toBe(200); + expect(await response.text()).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:EVENT", + "#EXT-X-INDEPENDENT-SEGMENTS", + "#EXT-X-START:TIME-OFFSET=0", + `#EXT-X-MAP:URI="${a}",BYTERANGE="800@0"`, + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + a, + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@1800", + a, + `#EXT-X-MAP:URI="${b}",BYTERANGE="800@0"`, + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + b, + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@1800", + b, + "", + ].join("\n"), + ); + }); + + it("serves the audio only as far as the video reaches", async () => { + const text = await (await play("audio.m3u8")).text(); + + expect(text.match(/#EXTINF/g)).toHaveLength(4); + expect(text).toContain("#EXT-X-PLAYLIST-TYPE:EVENT"); + expect(text).not.toContain("#EXT-X-ENDLIST"); + expect(text).toContain(`https://cdn/${FOLDER}/hls/audio-0-a.m4s?sig`); + }); + + it("lets a growing playlist be shared for two seconds only", async () => { + for (const file of ["1080p.m3u8", "audio.m3u8", "master.m3u8"]) { + expect((await play(file)).headers.get("cache-control")).toBe( + "public, max-age=2, s-maxage=2", + ); + } + }); + + it("redirects a slice's media file to its signed address", async () => { + const response = await play("1080p-1-b.m4s"); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe( + `https://cdn/${FOLDER}/hls/1080p-1-b.m4s?sig`, + ); + }); + + it("does not list a slice that finished ahead of a missing one", async () => { + serve( + growing({ + slices: [ + finishedSlice(0, 8, "a", sliceResult(8, { audioBytes: 300 })), + slice(8, 8, { job: runningJob("b") }), + finishedSlice(16, null, "c", sliceResult(6)), + ], + }), + ); + + const text = await (await play("1080p.m3u8")).text(); + + expect(text.match(/#EXT-X-MAP/g)).toHaveLength(1); + expect(text).not.toContain("1080p-2-c.m4s"); + }); + + it("has no playlist for a rendition the ladder does not have", async () => { + expect((await play("720p.m3u8")).status).toBe(404); + }); + + it("answers 502 when a finished slice's playlist cannot be read", async () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + mocks.readText.mockResolvedValue(null); + + expect((await play("1080p.m3u8")).status).toBe(502); + }); +}); + +describe("when every slice is in but the ladder is not stored yet", () => { + beforeEach(() => serve(whole())); + + it("adds the end marker and nothing else changes above it", async () => { + const done = await (await play("1080p.m3u8")).text(); + serve(growing()); + const before = await (await play("1080p.m3u8")).text(); + + expect(done.startsWith(before)).toBe(true); + expect(done.endsWith("#EXT-X-ENDLIST\n")).toBe(true); + expect(done.match(/#EXT-X-MAP/g)).toHaveLength(3); + }); + + it("lists the whole audio and caches like a finished video", async () => { + const response = await play("audio.m3u8"); + const text = await response.text(); + + expect(text.match(/#EXTINF/g)).toHaveLength(6); + expect(text.endsWith("#EXT-X-ENDLIST\n")).toBe(true); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + }); +}); + +describe("once the ladder is whole", () => { + it("serves the stored playlists like any other ladder", async () => { + serve( + { ...whole(), state: "complete" }, + { state: "complete", output: "sliced-1" }, + ); + mocks.readText.mockImplementation(async (key: string) => + key === `${FOLDER}/hls/1080p.m3u8` ? "#EXTM3U\n#EXT-X-ENDLIST\n" : null, + ); + + const response = await play("1080p.m3u8"); + + expect(await response.text()).toBe("#EXTM3U\n#EXT-X-ENDLIST\n"); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(mocks.readText).toHaveBeenCalledTimes(1); + }); +}); + +describe("when the growing ladder is not offered", () => { + it("is not found before the first slice is in", async () => { + serve( + slicedHls({ + slices: [ + slice(0, 8, { job: runningJob("a") }), + finishedSlice(8, null, "b", sliceResult(8)), + ], + }), + ); + mocks.store.video = mocks.store.video && { + ...mocks.store.video, + status: "processing", + }; + + expect((await play("master.m3u8")).status).toBe(404); + expect((await play("1080p-1-b.m4s")).status).toBe(404); + }); + + it("is not found for a video that already played from its MP4", async () => { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ + mp4: { state: "complete", output: "mp4-out" }, + sliced: growing({ exposed: false }), + }), + }); + const video = mocks.store.video; + + expect((await play("master.m3u8")).status).toBe(404); + expect(serializeDeveloperVideo(video)).toMatchObject({ + hlsUrl: null, + mp4Url: `${BASE}/video.mp4`, + }); + }); + + it("keeps playing the finished slices while one job encodes the video instead", async () => { + serve({ ...growing(), state: "abandoned" }, runningJob("whole")); + + const response = await play("1080p.m3u8"); + + expect(response.status).toBe(200); + expect(await response.text()).not.toContain("#EXT-X-ENDLIST"); + }); + + it("still serves the slices' media to players that loaded them once that job is in", async () => { + serve( + { ...growing(), state: "abandoned" }, + { + state: "complete", + output: "whole", + }, + ); + + const slice = await play("1080p-1-b.m4s"); + const wholeMedia = await play("1080p.m4s"); + + expect(slice.headers.get("location")).toBe( + `https://cdn/${FOLDER}/hls/1080p-1-b.m4s?sig`, + ); + expect(wholeMedia.headers.get("location")).toBe( + `https://cdn/${FOLDER.replace(/[^/]+$/, "whole")}/hls/1080p.m4s?sig`, + ); + }); + + it("is not found once that job has failed too", async () => { + serve( + { ...growing(), state: "abandoned" }, + { state: "failed", reason: "x" }, + ); + + expect((await play("master.m3u8")).status).toBe(404); + expect((await play("1080p.m3u8")).status).toBe(404); + }); +}); + +describe("a growing ladder and the rest of playback", () => { + const subtitleLines = (master: string) => + master + .split("\n") + .filter( + (line) => + line.startsWith("#EXT-X-MEDIA:TYPE=SUBTITLES") || + line.startsWith("#EXT-X-STREAM-INF:"), + ) + .map((line) => line.replace(/^#EXT-X-STREAM-INF:.*(,SUBTITLES=)/, "$1")); + + it("lists captions as subtitles in the master just as a whole ladder does", async () => { + mocks.store.captionRows = [{ videoId: "video-1", language: "en" }]; + + const growingMaster = await (await play("master.m3u8")).text(); + const builtWhole = await (async () => { + serve({ ...whole(), state: "encoding" }); + return (await play("master.m3u8")).text(); + })(); + serve( + { ...whole(), state: "complete" }, + { state: "complete", output: "sliced-1" }, + ); + mocks.readText.mockImplementation(async (key: string) => + key === `${FOLDER}/hls/master.m3u8` + ? builtWhole + .replace(/\n#EXT-X-MEDIA:TYPE=SUBTITLES[^\n]*/g, "") + .replace(/,SUBTITLES="subs"/g, "") + : null, + ); + const storedMaster = await (await play("master.m3u8")).text(); + + expect(growingMaster).toContain( + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs"', + ); + expect(growingMaster).toContain('URI="captions-en.m3u8"'); + expect(subtitleLines(growingMaster)).toEqual(subtitleLines(storedMaster)); + expect((await play("captions-en.m3u8")).status).toBe(200); + mocks.store.captionRows = []; + }); + + it("sends a growing rendition playlist as text/plain and the master as HLS", async () => { + expect((await play("1080p.m3u8")).headers.get("content-type")).toBe( + "text/plain; charset=utf-8", + ); + expect((await play("audio.m3u8")).headers.get("content-type")).toBe( + "text/plain; charset=utf-8", + ); + expect((await play("master.m3u8")).headers.get("content-type")).toBe( + "application/vnd.apple.mpegurl", + ); + }); + + it("tags every growing answer with the video and app, so a purge drops it", async () => { + for (const file of [ + "master.m3u8", + "1080p.m3u8", + "audio.m3u8", + "1080p-1-b.m4s", + ]) { + expect((await play(file)).headers.get("vercel-cache-tag")).toBe( + "infra-video-video-1,infra-app-app-1", + ); + } + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-sliced-playlist.test.ts b/apps/web/__tests__/unit/developer-video-sliced-playlist.test.ts new file mode 100644 index 00000000000..452b705c571 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-sliced-playlist.test.ts @@ -0,0 +1,318 @@ +import { describe, expect, it, vi } from "vitest"; +import { + buildSlicedHlsMaster, + HLS_AUDIO_BITRATE, + HLS_AUDIO_NAME, + HLS_SEGMENT_SECONDS, + joinHlsSlicePlaylists, + planHlsLadderSlices, + readHlsSlicePlaylist, +} from "@/lib/developer-video-sliced-playlist"; +import { slicePlaylist } from "../fixtures/developer-video-slices"; + +// The media server's ladder module reaches a Bun-only import through its +// input policy. Nothing here runs it. It is loaded at run time behind the +// shape below, because a static import would bring Bun's global types into +// the web app's typecheck and change what `fetch` means everywhere. +vi.mock("bun", () => ({ spawnSync: vi.fn() })); + +interface MediaServerLadder { + HLS_LADDER_AUDIO_BITRATE_KBPS: number; + HLS_LADDER_AUDIO_NAME: string; + HLS_LADDER_SEGMENT_SECONDS: number; + planHlsLadderSlices( + durationSeconds: number, + workers: number, + minSliceSeconds?: number, + ): { startSeconds: number; durationSeconds?: number }[]; + stitchHlsLadderSlices( + slices: { master: string; playlists: Record }[], + options: { hasAudio: boolean }, + ): { + master: string; + playlists: Record; + media: { slice: number; source: string; name: string }[]; + }; +} + +const MEDIA_SERVER_LADDER = "../../../media-server/src/lib/hls-ladder"; +const { + HLS_LADDER_AUDIO_BITRATE_KBPS, + HLS_LADDER_AUDIO_NAME, + HLS_LADDER_SEGMENT_SECONDS, + planHlsLadderSlices: planOnMediaServer, + stitchHlsLadderSlices, +} = (await import(/* @vite-ignore */ MEDIA_SERVER_LADDER)) as MediaServerLadder; + +const read = (stream: string, slice: number, segments: number[]) => + readHlsSlicePlaylist( + slicePlaylist(stream, segments), + `${stream}.m4s`, + `${stream}-${slice}.m4s`, + ); + +describe("the copy of the media server's slice planner", () => { + it("uses the media server's segment length and audio settings", () => { + expect(HLS_SEGMENT_SECONDS).toBe(HLS_LADDER_SEGMENT_SECONDS); + expect(HLS_AUDIO_BITRATE).toBe(HLS_LADDER_AUDIO_BITRATE_KBPS * 1000); + expect(HLS_AUDIO_NAME).toBe(HLS_LADDER_AUDIO_NAME); + }); + + it("plans exactly the slices the media server plans", () => { + const durations = [ + 0, 0.5, 3.9, 4, 19.9, 20, 59, 60, 61.3, 100, 119.99, 180, 240.5, 314, + 1146.2, 1246, 3599.97, 14_380, + ]; + for (const duration of durations) { + for (const workers of [1, 2, 3, 4]) { + for (const minimum of [20, 60]) { + expect(planHlsLadderSlices(duration, workers, minimum)).toEqual( + planOnMediaServer(duration, workers, minimum), + ); + } + } + } + }); +}); + +describe("reading a slice's playlist", () => { + it("keeps each segment's length and byte range and renames the media file", () => { + const playlist = read("1080p", 2, [4, 4, 1.5]); + + expect(playlist.map).toBe('#EXT-X-MAP:URI="1080p-2.m4s",BYTERANGE="800@0"'); + expect(playlist.targetDuration).toBe(4); + expect(playlist.seconds).toBeCloseTo(9.5); + expect(playlist.segments).toEqual([ + { + seconds: 4, + lines: [ + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p-2.m4s", + ], + }, + { + seconds: 4, + lines: [ + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@1800", + "1080p-2.m4s", + ], + }, + { + seconds: 1.5, + lines: [ + "#EXTINF:1.500000,", + "#EXT-X-BYTERANGE:1000@2800", + "1080p-2.m4s", + ], + }, + ]); + }); + + it("refuses a playlist that names another file", () => { + expect(() => + readHlsSlicePlaylist( + slicePlaylist("720p", [4]), + "1080p.m4s", + "1080p-0.m4s", + ), + ).toThrow("unexpected init segment"); + expect(() => + readHlsSlicePlaylist( + `${slicePlaylist("1080p", [4])}https://elsewhere.example/x.m4s\n`, + "1080p.m4s", + "1080p-0.m4s", + ), + ).toThrow("unexpected file"); + }); + + it("refuses a playlist with nothing in it", () => { + expect(() => + readHlsSlicePlaylist( + '#EXTM3U\n#EXT-X-MAP:URI="1080p.m4s"\n', + "1080p.m4s", + "1080p-0.m4s", + ), + ).toThrow("no segments"); + expect(() => + readHlsSlicePlaylist( + "#EXTM3U\n#EXTINF:4.0,\n1080p.m4s\n", + "1080p.m4s", + "1080p-0.m4s", + ), + ).toThrow("no init segment"); + }); +}); + +describe("joining slices into one playlist", () => { + const first = read("1080p", 0, [4, 4]); + const second = read("1080p", 1, [4, 2.5]); + + it("lists the first slice with no end marker while more are coming", () => { + expect(joinHlsSlicePlaylists([first], { ended: false })).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:EVENT", + "#EXT-X-INDEPENDENT-SEGMENTS", + "#EXT-X-START:TIME-OFFSET=0", + '#EXT-X-MAP:URI="1080p-0.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p-0.m4s", + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@1800", + "1080p-0.m4s", + "", + ].join("\n"), + ); + }); + + it("only ever adds lines as slices arrive, then the end marker", () => { + const one = joinHlsSlicePlaylists([first], { ended: false }); + const two = joinHlsSlicePlaylists([first, second], { ended: false }); + const done = joinHlsSlicePlaylists([first, second], { ended: true }); + + expect(two.startsWith(one)).toBe(true); + expect(done).toBe(`${two}#EXT-X-ENDLIST\n`); + expect(two.slice(one.length)).toBe( + [ + '#EXT-X-MAP:URI="1080p-1.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p-1.m4s", + "#EXTINF:2.500000,", + "#EXT-X-BYTERANGE:1000@1800", + "1080p-1.m4s", + "", + ].join("\n"), + ); + }); + + it("cuts the audio where the video ends while slices are still coming", () => { + const audio = readHlsSlicePlaylist( + slicePlaylist("audio", [4.01, 4.01, 4.01, 4.01, 2]), + "audio.m4s", + "audio-0.m4s", + ); + + const growing = joinHlsSlicePlaylists([audio], { + ended: false, + untilSeconds: 8, + }); + const whole = joinHlsSlicePlaylists([audio], { ended: true }); + + expect(growing.match(/#EXTINF/g)).toHaveLength(2); + expect(growing).not.toContain("#EXT-X-ENDLIST"); + expect(whole.match(/#EXTINF/g)).toHaveLength(5); + expect(whole.startsWith(growing)).toBe(true); + }); + + it("lists the same segments, in the same order, as the media server's own joiner", () => { + const slices = [ + [4, 4, 4, 4, 4], + [4, 4, 4], + [4, 4, 3.2], + ]; + const master = [ + "#EXTM3U", + '#EXT-X-STREAM-INF:BANDWIDTH=4000000,AVERAGE-BANDWIDTH=3000000,RESOLUTION=1920x1080,CODECS="avc1.640028"', + "1080p.m3u8", + "", + ].join("\n"); + const stitched = stitchHlsLadderSlices( + slices.map((segments) => ({ + master, + playlists: { "1080p": slicePlaylist("1080p", segments) }, + })), + { hasAudio: false }, + ); + + const joined = joinHlsSlicePlaylists( + slices.map((segments, index) => read("1080p", index, segments)), + { ended: true }, + ); + + // The two differ only in the lines that make a playlist one that grows. + const withoutGrowth = joined + .replace("#EXT-X-PLAYLIST-TYPE:EVENT", "#EXT-X-PLAYLIST-TYPE:VOD") + .replace("#EXT-X-START:TIME-OFFSET=0\n", ""); + expect(withoutGrowth).toBe(stitched.playlists["1080p.m3u8"]); + }); +}); + +describe("the master playlist of a ladder made of slices", () => { + const variants = [ + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 4_000_000, + averageBandwidth: 3_000_000, + codecs: "avc1.640028", + }, + { + name: "360p", + width: 640, + height: 360, + bandwidth: 500_000, + averageBandwidth: 400_000, + codecs: "avc1.64001e", + }, + ]; + + it("lists every rendition with the audio group", () => { + expect(buildSlicedHlsMaster(variants, { hasAudio: true })).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-INDEPENDENT-SEGMENTS", + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="audio",DEFAULT=YES,AUTOSELECT=YES,CHANNELS="2",URI="audio.m3u8"', + '#EXT-X-STREAM-INF:BANDWIDTH=4192000,AVERAGE-BANDWIDTH=3128000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud"', + "1080p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=692000,AVERAGE-BANDWIDTH=528000,RESOLUTION=640x360,CODECS="avc1.64001e,mp4a.40.2",AUDIO="aud"', + "360p.m3u8", + "", + ].join("\n"), + ); + }); + + it("leaves audio out for a silent video", () => { + const master = buildSlicedHlsMaster(variants, { hasAudio: false }); + + expect(master).not.toContain("AUDIO"); + expect(master).toContain( + '#EXT-X-STREAM-INF:BANDWIDTH=4000000,AVERAGE-BANDWIDTH=3000000,RESOLUTION=1920x1080,CODECS="avc1.640028"', + ); + }); + + it("writes the same master as the media server's own joiner", () => { + const sliceMasterText = [ + "#EXTM3U", + '#EXT-X-STREAM-INF:BANDWIDTH=4000000,AVERAGE-BANDWIDTH=3000000,RESOLUTION=1920x1080,CODECS="avc1.640028"', + "1080p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=500000,AVERAGE-BANDWIDTH=400000,RESOLUTION=640x360,CODECS="avc1.64001e"', + "360p.m3u8", + "", + ].join("\n"); + const stitched = stitchHlsLadderSlices( + [ + { + master: sliceMasterText, + playlists: { + "1080p": slicePlaylist("1080p", [4, 4]), + "360p": slicePlaylist("360p", [4, 4]), + }, + }, + ], + { hasAudio: true }, + ); + + expect(buildSlicedHlsMaster(variants, { hasAudio: true })).toBe( + stitched.master, + ); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-sliced-processing.test.ts b/apps/web/__tests__/unit/developer-video-sliced-processing.test.ts new file mode 100644 index 00000000000..61990aec53b --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-sliced-processing.test.ts @@ -0,0 +1,1232 @@ +import type { + DeveloperVideoProcessingState, + DeveloperVideoSlicedHls, +} from "@cap/database/types"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; +import { + slicedHls, + sliceLadder, + sliceMaster, + slicePlaylist, + sliceResult, +} from "../fixtures/developer-video-slices"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + env: { + WEB_URL: "https://web.example", + MEDIA_SERVER_WEBHOOK_URL: undefined, + } as Record, + head: vi.fn(), + readText: vi.fn(), + writeText: vi.fn(), + probe: vi.fn(), + startJob: vi.fn(), + readJob: vi.fn(), + charge: vi.fn(), + deleteObjects: vi.fn(), + sniff: vi.fn(), + cancelJob: vi.fn(), + firstMonth: vi.fn(), + announce: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const database = await createDeveloperVideoDatabase(mocks.store); + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ serverEnv: () => mocks.env })); +vi.mock("@/lib/developer-video-storage", () => ({ + headDeveloperVideoObject: mocks.head, + readDeveloperVideoText: mocks.readText, + writeDeveloperVideoText: mocks.writeText, + deleteDeveloperVideoObjects: mocks.deleteObjects, + developerVideoPrefix: (video: { appId: string; id: string }) => + `developer/${video.appId}/${video.id}/`, + internalDeveloperVideoUrl: async (key: string) => `https://storage/${key}`, + developerVideoUploadUrl: async (key: string, contentType: string) => + `https://storage/put/${key}?type=${contentType}`, +})); +vi.mock("@/lib/developer-video-charge", () => ({ + DEVELOPER_VIDEO_MAX_DURATION_SECONDS: 4 * 60 * 60, + chargeDeveloperVideoUpload: mocks.charge, +})); +vi.mock("@/lib/media-client", () => ({ + MEDIA_SERVER_SHUTDOWN_MESSAGE: "Server shutting down", + tryProbeVideoViaMediaServer: mocks.probe, + startMediaServerVideoJob: mocks.startJob, + readMediaServerVideoJob: mocks.readJob, + cancelMediaServerVideoJob: mocks.cancelJob, +})); +vi.mock("@/lib/media-container", () => ({ sniffMediaUrl: mocks.sniff })); +vi.mock("@/lib/infra-storage-minimum", () => ({ + settleFirstMonthAtReady: mocks.firstMonth, +})); +vi.mock("@/lib/developer-webhook-dispatch", () => ({ + announceDeveloperVideoEvent: mocks.announce, +})); + +const { + applyDeveloperVideoProgress, + checkDeveloperVideoPart, + dispatchDeveloperVideoPart, + dispatchDeveloperVideoParts, + failDeveloperVideoProcessing, + prepareDeveloperVideoProcessing, + readDeveloperVideoParts, + sweepSupersededDeveloperVideoOutputs, +} = await import("@/lib/developer-video-processing"); +const { developerVideoOutputs } = await import("@/lib/developer-video-api"); + +const FOLDER = "developer/app-1/video-1/encode-1/sliced-1"; +// Lengths of the four slices of the 300 second fixture video. +const SECONDS = [20, 96, 96, 88]; +// 128 kbps over the 299.9 s of audio the fixture's audio playlist lists. +const AUDIO_BYTES = 4_798_400; + +function video() { + if (!mocks.store.video) throw new Error("no video"); + return mocks.store.video; +} + +function state(): DeveloperVideoProcessingState { + const processing = video().processing; + if (!processing) throw new Error("no processing state"); + return processing; +} + +interface SentSlice { + index: number; + token: string; + body: Record; +} + +const STARTS = [0, 20, 116, 212]; + +// Every slice job sent so far, read from the encoder requests. Slices sent in +// one pass leave together, so they are listed in playback order. +function sent(): SentSlice[] { + return sentInOrder().sort((a, b) => a.index - b.index); +} + +function sentInOrder(): SentSlice[] { + return mocks.startJob.mock.calls.flatMap(([path, body]) => { + const range = (body as { range?: { startSeconds: number } }).range; + if (path !== "/video/ladder" || !range) return []; + const token = new URL( + (body as { webhookUrl: string }).webhookUrl, + ).searchParams.get("token"); + const index = STARTS.indexOf(range.startSeconds); + return token + ? [{ index, token, body: body as Record }] + : []; + }); +} + +function latest(index: number): SentSlice { + const found = sentInOrder() + .filter((entry) => entry.index === index) + .at(-1); + if (!found) throw new Error(`slice ${index} was never sent`); + return found; +} + +const segmentsFor = (seconds: number) => + Array.from({ length: seconds / 4 }, () => 4); + +function finish(index: number, token = latest(index).token) { + const seconds = SECONDS[index] ?? 0; + return applyDeveloperVideoProgress( + { + videoId: "video-1", + phase: "complete", + ladder: sliceLadder(sliceResult(seconds), index === 0 ? AUDIO_BYTES : 0), + }, + "hls", + token, + ); +} + +function failSlice(index: number) { + return applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "ffmpeg exited 1" }, + "hls", + latest(index).token, + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + delete mocks.env.INFRA_SLICED_HLS; + mocks.store.video = developerVideo({ + duration: 300, + processing: processingState({ + probe: { duration: 300, width: 1920, height: 1080, fps: 30 }, + sliced: slicedHls(), + }), + }); + mocks.store.appDeleted = false; + mocks.store.updates = []; + mocks.head.mockImplementation(async (key: string) => { + if (key.endsWith("/poster.jpg")) { + return { bytes: 50, contentType: "image/jpeg" }; + } + if (key.endsWith("/video.mp4")) { + return { bytes: 2_000, contentType: "video/mp4" }; + } + return { bytes: 1_000, contentType: "video/quicktime" }; + }); + mocks.readText.mockImplementation(async (key: string) => { + const file = key.match(/\/slices\/([A-Za-z0-9]+)-(\d)-[^/]+\.m3u8$/); + if (!file?.[1] || !file[2]) return null; + if (file[1] === "master") return sliceMaster(["1080p", "360p"]); + if (file[1] === "audio") { + return slicePlaylist("audio", [...segmentsFor(296), 3.9]); + } + return slicePlaylist(file[1], segmentsFor(SECONDS[Number(file[2])] ?? 4)); + }); + mocks.writeText.mockResolvedValue(undefined); + mocks.charge.mockResolvedValue("charged"); + mocks.deleteObjects.mockResolvedValue(0); + mocks.sniff.mockResolvedValue("media"); + mocks.startJob.mockImplementation(async (_path, body) => { + const range = (body as { range?: { startSeconds: number } }).range; + return { + status: "started", + jobId: range ? `slice-job-${STARTS.indexOf(range.startSeconds)}` : "job", + }; + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("deciding how a video's adaptive stream is encoded", () => { + const probed = ( + duration: number, + source: Partial[0]> = {}, + size = { width: 1920, height: 1080 }, + ) => { + mocks.store.video = developerVideo({ + duration, + processing: processingState({ probe: null }), + ...source, + }); + mocks.probe.mockResolvedValue({ + status: "probed", + metadata: { duration, ...size, fps: 30 }, + }); + }; + + it("plans slices for a video of twenty minutes or more", async () => { + probed(1266); + + expect(await prepareDeveloperVideoProcessing("video-1", "encode-1")).toBe( + "ready", + ); + + expect(state().sliced).toMatchObject({ + state: "encoding", + exposed: null, + slices: [ + { startSeconds: 0, durationSeconds: 20, job: { state: "pending" } }, + { startSeconds: 20, durationSeconds: 120 }, + { startSeconds: 140, durationSeconds: 376 }, + { startSeconds: 516, durationSeconds: 376 }, + { startSeconds: 892, durationSeconds: null }, + ], + }); + expect(state().sliced?.output).toMatch(/^[A-Za-z0-9_-]{16}$/); + expect(video().status).toBe("processing"); + }); + + it("leaves a ladder that tops out above 1080p on the single job", async () => { + probed(3797, { maxResolutionTier: "2160p" }, { width: 3840, height: 2160 }); + + await prepareDeveloperVideoProcessing("video-1", "encode-1"); + + expect(state().sliced).toBeUndefined(); + }); + + it("slices a 4K source whose ladder is held to 1080p", async () => { + probed(3797, { maxResolutionTier: "1080p" }, { width: 3840, height: 2160 }); + + await prepareDeveloperVideoProcessing("video-1", "encode-1"); + + expect(state().sliced?.state).toBe("encoding"); + }); + + it("leaves a shorter video on the single job", async () => { + probed(1199); + + await prepareDeveloperVideoProcessing("video-1", "encode-1"); + + expect(state().sliced).toBeUndefined(); + }); + + it("leaves every video on the single job when slicing is switched off", async () => { + mocks.env.INFRA_SLICED_HLS = "off"; + probed(1266); + + await prepareDeveloperVideoProcessing("video-1", "encode-1"); + + expect(state().sliced).toBeUndefined(); + }); + + it("sends a short video's ladder job exactly as before", async () => { + mocks.store.video = developerVideo(); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + + expect(mocks.startJob).toHaveBeenCalledTimes(1); + const [path, body] = mocks.startJob.mock.calls[0] ?? []; + const token = state().hls.state === "running" ? state().hls : null; + if (token?.state !== "running") throw new Error("hls is not running"); + const prefix = `https://storage/put/developer/app-1/video-1/encode-1/${token.token}`; + expect(path).toBe("/video/ladder"); + expect(body).toEqual({ + videoId: "video-1", + userId: "app-1", + videoUrl: "https://storage/developer/app-1/video-1/video", + thumbnailPresignedUrl: `${prefix}/poster.jpg?type=image/jpeg`, + webhookUrl: `https://web.example/api/webhooks/media-server/progress?developerVideo=hls&token=${token.token}`, + inputExtension: ".mov", + maxTier: "1080p", + includeAudio: true, + infra: true, + uploads: Object.fromEntries( + [ + ["master.m3u8", "application/vnd.apple.mpegurl"], + ...["1080p", "720p", "360p", "audio"].flatMap((name) => [ + [`${name}.m3u8`, "application/vnd.apple.mpegurl"], + [`${name}.m4s`, "video/mp4"], + ]), + ].map(([name, type]) => [name, `${prefix}/hls/${name}?type=${type}`]), + ), + }); + }); +}); + +describe("sending slices", () => { + it("asks the workflow to send them through the hls part", async () => { + expect(await readDeveloperVideoParts("video-1", "encode-1")).toEqual({ + hls: "dispatch", + mp4: "dispatch", + }); + }); + + it("sends the first three as their own ladder jobs and never the whole job", async () => { + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + + expect(mocks.startJob).toHaveBeenCalledTimes(3); + expect(sent().map((entry) => entry.body.range)).toEqual([ + { startSeconds: 0, durationSeconds: 20 }, + { startSeconds: 20, durationSeconds: 96 }, + { startSeconds: 116, durationSeconds: 96 }, + ]); + expect(state().hls).toEqual({ state: "pending" }); + expect(state().sliced?.slices.map((entry) => entry.job)).toMatchObject([ + { state: "running", jobId: "slice-job-0" }, + { state: "running", jobId: "slice-job-1" }, + { state: "running", jobId: "slice-job-2" }, + { state: "pending" }, + ]); + expect(await readDeveloperVideoParts("video-1", "encode-1")).toMatchObject({ + hls: "wait", + }); + }); + + it("sends the MP4 before any slice when the video starts", async () => { + let answerMp4: (value: { status: "started"; jobId: string }) => void = () => + undefined; + const startJob = mocks.startJob.getMockImplementation(); + mocks.startJob.mockImplementation(async (path, body) => + path === "/video/process" + ? new Promise((resolve) => { + answerMp4 = resolve; + }) + : startJob?.(path, body), + ); + + const dispatched = dispatchDeveloperVideoParts("video-1", "encode-1"); + await vi.waitFor(() => expect(mocks.startJob).toHaveBeenCalledTimes(1)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(mocks.startJob.mock.calls.map(([path]) => path)).toEqual([ + "/video/process", + ]); + + answerMp4({ status: "started", jobId: "mp4-job" }); + const results = await dispatched; + expect(results.map((result) => result.status)).toEqual([ + "fulfilled", + "fulfilled", + ]); + expect(mocks.startJob.mock.calls.map(([path]) => path)).toEqual([ + "/video/process", + "/video/ladder", + "/video/ladder", + "/video/ladder", + ]); + expect(state().mp4).toMatchObject({ state: "running", jobId: "mp4-job" }); + }); + + it("gives the first slice the audio and the poster, and the others neither", async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + const first = latest(0); + const second = latest(1); + + expect(first.body.includeAudio).toBe(true); + expect(first.body.thumbnailPresignedUrl).toBe( + `https://storage/put/developer/app-1/video-1/encode-1/${first.token}/poster.jpg?type=image/jpeg`, + ); + expect(Object.keys(first.body.uploads as object)).toContain("audio.m4s"); + + expect(second.body.includeAudio).toBe(false); + expect(second.body).not.toHaveProperty("thumbnailPresignedUrl"); + expect(Object.keys(second.body.uploads as object)).toEqual([ + "master.m3u8", + "1080p.m3u8", + "1080p.m4s", + "720p.m3u8", + "720p.m4s", + "360p.m3u8", + "360p.m4s", + ]); + }); + + it("stores each slice under its own names in the one sliced folder", async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + const second = latest(1); + const uploads = second.body.uploads as Record; + + expect(uploads["1080p.m4s"]).toBe( + `https://storage/put/${FOLDER}/hls/1080p-1-${second.token}.m4s?type=video/mp4`, + ); + expect(uploads["1080p.m3u8"]).toBe( + `https://storage/put/${FOLDER}/slices/1080p-1-${second.token}.m3u8?type=application/vnd.apple.mpegurl`, + ); + expect(second.body).toMatchObject({ + videoId: "video-1", + userId: "app-1", + videoUrl: "https://storage/developer/app-1/video-1/video", + maxTier: "1080p", + webhookUrl: `https://web.example/api/webhooks/media-server/progress?developerVideo=hls&token=${second.token}`, + }); + expect(new Set(sent().map((entry) => entry.token)).size).toBe(3); + }); + + it("sends the first slice on its own before the others", async () => { + let answerFirst: (value: { status: "started"; jobId: string }) => void = + () => undefined; + mocks.startJob.mockImplementation(async (_path, body) => + (body as { range: { startSeconds: number } }).range.startSeconds === 0 + ? new Promise((resolve) => { + answerFirst = resolve; + }) + : { status: "started", jobId: "job" }, + ); + + const dispatched = dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + await vi.waitFor(() => expect(mocks.startJob).toHaveBeenCalledTimes(1)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(sentInOrder().map((entry) => entry.index)).toEqual([0]); + + answerFirst({ status: "started", jobId: "slice-job-0" }); + expect(await dispatched).toBe("dispatched"); + expect(sentInOrder().map((entry) => entry.index)).toEqual([0, 1, 2]); + }); + + it("asks four workers for the first slice before it waits", async () => { + mocks.startJob.mockResolvedValue({ status: "busy" }); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "busy", + ); + + expect(sent().filter((entry) => entry.index === 0)).toHaveLength(4); + expect(sent().filter((entry) => entry.index === 1)).toHaveLength(2); + expect(sent().filter((entry) => entry.index === 2)).toHaveLength(2); + expect(state().sliced?.slices[0]).toMatchObject({ + job: { state: "pending" }, + attempts: 0, + }); + }); + + it("asks again at once when a worker is full, then waits for the next pass", async () => { + mocks.startJob.mockImplementation(async (_path, body) => + (body as { range: { startSeconds: number } }).range.startSeconds === 20 + ? { status: "busy" } + : { status: "started", jobId: "job" }, + ); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + + expect(sent().filter((entry) => entry.index === 1)).toHaveLength(2); + expect(state().sliced?.slices[1]).toMatchObject({ + job: { state: "pending" }, + attempts: 0, + }); + expect(state().sliced?.slices[0]?.job.state).toBe("running"); + }); + + it("falls back to the single job when the encoder has no ladder route", async () => { + mocks.startJob.mockResolvedValue({ status: "unavailable" }); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(state().sliced?.state).toBe("abandoned"); + expect(state().hls).toEqual({ state: "pending" }); + expect(sent().map((entry) => entry.index)).toEqual([0]); + }); + + it("does not start a slice for a video that was deleted", async () => { + mocks.store.video = { ...video(), deletedAt: new Date() }; + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "skipped", + ); + expect(mocks.startJob).not.toHaveBeenCalled(); + }); +}); + +describe("slice results", () => { + beforeEach(async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + mocks.store.updates = []; + }); + + it("makes the video ready and playable from the first slice", async () => { + expect(await finish(0)).toEqual({ success: true, applied: true }); + + expect(video().status).toBe("ready"); + expect(developerVideoOutputs(video())).toEqual({ + hls: true, + mp4: false, + poster: true, + }); + expect(state().sliced).toMatchObject({ exposed: true, hasAudio: true }); + expect(state().poster).toEqual({ output: latest(0).token }); + expect(video().storedBytes).toBe(1_000 + 5_700 + AUDIO_BYTES + 50); + expect(video().renditions).toEqual([ + { + name: "1080p", + width: 1920, + height: 1080, + bitrate: 3_128_000, + bytes: 5_000, + codecs: "avc1.640028,mp4a.40.2", + }, + { + name: "360p", + width: 640, + height: 360, + bitrate: 528_000, + bytes: 700, + codecs: "avc1.64001e,mp4a.40.2", + }, + ]); + }); + + it("sends the next slice as soon as one is in", async () => { + await finish(0); + + expect(sent().map((entry) => entry.index)).toEqual([0, 1, 2, 3]); + expect(sentInOrder().at(-1)?.index).toBe(3); + expect(latest(3).body.range).toEqual({ startSeconds: 212 }); + expect(state().sliced?.slices[3]?.job.state).toBe("running"); + }); + + it("applies a result once however often it is delivered", async () => { + const { token } = latest(0); + await finish(0, token); + const stored = video().storedBytes; + mocks.store.updates = []; + + expect(await finish(0, token)).toEqual({ success: true, applied: false }); + expect(mocks.store.updates).toHaveLength(0); + expect(video().storedBytes).toBe(stored); + }); + + it("keeps the video processing while only later slices are in", async () => { + await finish(2); + await finish(1); + + expect(video().status).toBe("processing"); + expect(developerVideoOutputs(video()).hls).toBe(false); + expect(video().renditions).toBeNull(); + expect(video().storedBytes).toBe(1_000 + 5_700 * 2); + + await finish(0); + + expect(video().status).toBe("ready"); + expect(video().renditions?.[0]?.bytes).toBe(15_000); + }); + + it("keeps both results when two slices finish at the same moment", async () => { + const results = await Promise.all([finish(1), finish(2), finish(0)]); + + expect(results.every((result) => result.applied)).toBe(true); + expect(state().sliced?.slices.map((entry) => entry.job.state)).toEqual([ + "complete", + "complete", + "complete", + "running", + ]); + expect(video().storedBytes).toBe(1_000 + 5_700 * 3 + AUDIO_BYTES + 50); + expect(sent().filter((entry) => entry.index === 3)).toHaveLength(1); + }); + + it("records the job id a progress report carries", async () => { + mocks.startJob.mockResolvedValue({ status: "unconfirmed" }); + mocks.store.video = developerVideo({ + processing: processingState({ sliced: slicedHls() }), + }); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + const first = latest(0); + + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "processing", jobId: "job-late" }, + "hls", + first.token, + ); + + expect(state().sliced?.slices[0]?.job).toMatchObject({ + jobId: "job-late", + }); + }); + + it("refuses a slice that does not cover its part of the video", async () => { + mocks.readText.mockImplementation(async (key: string) => + key.includes("/master-") + ? sliceMaster(["1080p", "360p"]) + : slicePlaylist(key.includes("360p") ? "360p" : "1080p", [4, 4]), + ); + + await finish(1); + + expect(state().sliced?.slices[1]).toMatchObject({ + job: { state: "running" }, + attempts: 2, + }); + expect(state().sliced?.slices[1]?.result).toBeUndefined(); + expect(sent().filter((entry) => entry.index === 1)).toHaveLength(2); + }); + + it("refuses a slice whose playlists were not stored", async () => { + mocks.readText.mockResolvedValue(null); + + await finish(0); + + expect(state().sliced?.slices[0]?.result).toBeUndefined(); + expect(video().status).toBe("processing"); + }); + + it("ignores a result for a video that was deleted mid-encode", async () => { + mocks.store.video = { ...video(), deletedAt: new Date() }; + + expect(await finish(0)).toEqual({ success: true, applied: false }); + expect(mocks.store.updates).toHaveLength(0); + expect(sent()).toHaveLength(3); + }); +}); + +describe("copied audio", () => { + it("lists audio copied from the source at its own rate", async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + await applyDeveloperVideoProgress( + { + videoId: "video-1", + phase: "complete", + ladder: sliceLadder(sliceResult(SECONDS[0] ?? 20), 5_998_000), + }, + "hls", + latest(0).token, + ); + await finish(1); + await finish(2); + await finish(3); + const written = Object.fromEntries( + mocks.writeText.mock.calls.map(([key, text]) => [ + (key as string).split("/").at(-1), + text as string, + ]), + ); + + expect(state().sliced?.slices[0]?.result?.audioBitrate).toBe(160_000); + expect(written["master.m3u8"]).toContain( + "BANDWIDTH=4192000,AVERAGE-BANDWIDTH=3160000,", + ); + expect(video().renditions?.map((rendition) => rendition.bitrate)).toEqual([ + 3_160_000, 560_000, + ]); + }); +}); + +describe("the last slice", () => { + beforeEach(async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + await finish(0); + await finish(1); + await finish(2); + }); + + it("writes the whole ladder into the sliced folder and completes the hls part", async () => { + expect(mocks.writeText).not.toHaveBeenCalled(); + + await finish(3); + + expect( + mocks.writeText.mock.calls.map(([key, , type]) => [key, type]).sort(), + ).toEqual( + ["1080p.m3u8", "360p.m3u8", "audio.m3u8", "master.m3u8"].map((file) => [ + `${FOLDER}/hls/${file}`, + "application/vnd.apple.mpegurl", + ]), + ); + expect(state().hls).toEqual({ state: "complete", output: "sliced-1" }); + expect(state().sliced?.state).toBe("complete"); + expect(video().status).toBe("ready"); + expect(await readDeveloperVideoParts("video-1", "encode-1")).toMatchObject({ + hls: "done", + }); + }); + + it("stores playlists that list every slice's media in order and end", async () => { + await finish(3); + const written = Object.fromEntries( + mocks.writeText.mock.calls.map(([key, text]) => [ + (key as string).split("/").at(-1), + text as string, + ]), + ); + const tokens = [0, 1, 2, 3].map((index) => latest(index).token); + + expect(written["1080p.m3u8"]?.match(/#EXT-X-MAP:URI="([^"]+)"/g)).toEqual( + tokens.map( + (token, index) => `#EXT-X-MAP:URI="1080p-${index}-${token}.m4s"`, + ), + ); + expect(written["1080p.m3u8"]?.match(/#EXTINF/g)).toHaveLength(75); + expect(written["1080p.m3u8"]?.endsWith("#EXT-X-ENDLIST\n")).toBe(true); + expect(written["audio.m3u8"]).toContain(`audio-0-${tokens[0]}.m4s`); + expect(written["audio.m3u8"]?.match(/#EXTINF/g)).toHaveLength(75); + expect(written["master.m3u8"]).toContain( + '#EXT-X-STREAM-INF:BANDWIDTH=4192000,AVERAGE-BANDWIDTH=3128000,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="aud"\n1080p.m3u8', + ); + }); + + it("accounts for every slice once, the audio once and one poster", async () => { + await finish(3); + + expect(video().storedBytes).toBe(1_000 + 5_700 * 4 + AUDIO_BYTES + 50); + expect(state().bytes).toEqual({ + source: 1_000, + hls: 5_700 * 4 + AUDIO_BYTES, + mp4: 0, + poster: 50, + }); + expect(video().renditions).toEqual([ + { + name: "1080p", + width: 1920, + height: 1080, + bitrate: 3_128_000, + bytes: 20_000, + codecs: "avc1.640028,mp4a.40.2", + }, + { + name: "360p", + width: 640, + height: 360, + bitrate: 528_000, + bytes: 2_800, + codecs: "avc1.64001e,mp4a.40.2", + }, + ]); + }); + + it("is retried by the workflow when the ladder could not be written", async () => { + mocks.writeText.mockRejectedValueOnce(new Error("storage is down")); + + await finish(3); + + expect(state().hls).toEqual({ state: "pending" }); + expect(await readDeveloperVideoParts("video-1", "encode-1")).toMatchObject({ + hls: "dispatch", + }); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + expect(state().hls).toEqual({ state: "complete", output: "sliced-1" }); + expect(sent()).toHaveLength(4); + }); + + it("keeps the ladder, each slice's media, the poster and nothing else", async () => { + await finish(3); + const tokens = [0, 1, 2, 3].map((index) => latest(index).token); + expect( + await sweepSupersededDeveloperVideoOutputs("video-1", "encode-1"), + ).toBe(0); + expect(mocks.deleteObjects).not.toHaveBeenCalled(); + mocks.store.video = { + ...video(), + processing: { ...state(), mp4: { state: "complete", output: "mp4-1" } }, + }; + + await sweepSupersededDeveloperVideoOutputs("video-1", "encode-1"); + + const [prefix, keep] = mocks.deleteObjects.mock.calls[0] ?? []; + expect(prefix).toBe("developer/app-1/video-1/encode-1/"); + const kept = (key: string) => (keep as (key: string) => boolean)(key); + expect(kept(`${FOLDER}/hls/master.m3u8`)).toBe(true); + expect(kept(`${FOLDER}/hls/1080p.m3u8`)).toBe(true); + expect(kept(`${FOLDER}/hls/audio.m3u8`)).toBe(true); + expect(kept(`${FOLDER}/hls/1080p-2-${tokens[2]}.m4s`)).toBe(true); + expect(kept(`${FOLDER}/hls/audio-0-${tokens[0]}.m4s`)).toBe(true); + expect( + kept(`developer/app-1/video-1/encode-1/${tokens[0]}/poster.jpg`), + ).toBe(true); + expect(kept(`${FOLDER}/hls/1080p-2-someOtherAttempt.m4s`)).toBe(false); + expect(kept(`${FOLDER}/hls/720p-2-${tokens[2]}.m4s`)).toBe(false); + expect(kept(`${FOLDER}/slices/1080p-2-${tokens[2]}.m3u8`)).toBe(false); + expect(kept("developer/app-1/video-1/encode-1/mp4-1/video.mp4")).toBe(true); + }); +}); + +describe("a lost slice", () => { + const START = new Date("2026-11-01T00:00:00Z").getTime(); + + beforeEach(async () => { + vi.useFakeTimers(); + vi.setSystemTime(START); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + }); + + it("is asked about after three quiet minutes and left alone while it runs", async () => { + mocks.readJob.mockResolvedValue({ status: "running" }); + vi.setSystemTime(START + 3 * 60_000); + + expect(await readDeveloperVideoParts("video-1", "encode-1")).toMatchObject({ + hls: "check", + }); + expect(await checkDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "waiting", + ); + + expect(mocks.readJob).toHaveBeenCalledTimes(3); + expect(state().sliced?.slices[0]?.job).toMatchObject({ + state: "running", + checkedAt: START + 3 * 60_000, + }); + expect(sent()).toHaveLength(3); + }); + + it("is sent again once the encoder has not known its job for a minute", async () => { + mocks.readJob.mockImplementation(async (jobId: string) => + jobId === "slice-job-1" ? { status: "missing" } : { status: "running" }, + ); + vi.setSystemTime(START + 3 * 60_000); + await checkDeveloperVideoPart("video-1", "encode-1", "hls"); + expect(state().sliced?.slices[1]?.job).toMatchObject({ + state: "running", + notFoundAt: START + 3 * 60_000, + }); + + vi.setSystemTime(START + 6 * 60_000 + 1_000); + await checkDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(state().sliced?.slices[1]).toMatchObject({ + job: { state: "pending" }, + attempts: 0, + lostJobs: 1, + }); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + expect(sent().filter((entry) => entry.index === 1)).toHaveLength(2); + expect(state().sliced?.state).toBe("encoding"); + }); + + it("takes the result from the encoder when the webhook never arrived", async () => { + mocks.readJob.mockImplementation(async (jobId: string) => + jobId === "slice-job-0" + ? { + status: "finished", + progress: { + videoId: "video-1", + phase: "complete", + ladder: sliceLadder(sliceResult(20), AUDIO_BYTES), + }, + } + : { status: "running" }, + ); + vi.setSystemTime(START + 3 * 60_000); + + await checkDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(state().sliced?.slices[0]?.job.state).toBe("complete"); + expect(video().status).toBe("ready"); + }); + + it("is sent again without an attempt when its server shut down", async () => { + await applyDeveloperVideoProgress( + { + videoId: "video-1", + phase: "cancelled", + message: "Server shutting down", + }, + "hls", + latest(2).token, + ); + + expect(state().sliced?.slices[2]).toMatchObject({ + job: { state: "running" }, + attempts: 1, + lostJobs: 1, + }); + expect(sent().filter((entry) => entry.index === 2)).toHaveLength(2); + }); +}); + +describe("a slice that fails twice", () => { + beforeEach(async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + await finish(0); + }); + + it("is sent a second time after its first failure", async () => { + await failSlice(1); + + expect(sent().filter((entry) => entry.index === 1)).toHaveLength(2); + expect(state().sliced?.state).toBe("encoding"); + expect(video().status).toBe("ready"); + }); + + it("falls back to today's single job, which runs once", async () => { + await failSlice(1); + await failSlice(1); + + expect(state().sliced?.state).toBe("abandoned"); + expect(video().status).toBe("ready"); + expect(developerVideoOutputs(video()).hls).toBe(true); + expect(await readDeveloperVideoParts("video-1", "encode-1")).toMatchObject({ + hls: "dispatch", + }); + const before = mocks.startJob.mock.calls.length; + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + const [path, body] = mocks.startJob.mock.calls[before] ?? []; + expect(path).toBe("/video/ladder"); + expect(body).not.toHaveProperty("range"); + expect(body).toMatchObject({ includeAudio: true, maxTier: "1080p" }); + expect(state().hls).toMatchObject({ state: "running" }); + expect(state().attempts.hls).toBe(2); + }); + + it("drops late results from the slices that were still running", async () => { + await failSlice(1); + await failSlice(1); + mocks.store.updates = []; + + expect(await finish(2)).toEqual({ success: true, applied: false }); + expect(mocks.store.updates).toHaveLength(0); + }); + + const fallBackAndFail = async () => { + await failSlice(1); + await failSlice(1); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + const hls = state().hls; + if (hls.state !== "running") throw new Error("single job is not running"); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "ffmpeg exited 1" }, + "hls", + hls.token, + ); + }; + + it("ends errored when the single job fails too and there is no MP4", async () => { + mocks.store.video = { + ...video(), + processing: { ...state(), mp4: { state: "failed", reason: "no mp4" } }, + }; + + await fallBackAndFail(); + + expect(state().hls.state).toBe("failed"); + expect(video().status).toBe("errored"); + expect(video().errorReason).toBe("no mp4"); + expect(video().renditions).toBeNull(); + expect(developerVideoOutputs(video()).hls).toBe(false); + expect(video().storedBytes).toBe(1_000 + 50); + }); + + it("plays from the MP4 with no adaptive stream when the single job fails too", async () => { + mocks.store.video = { + ...video(), + processing: { + ...state(), + mp4: { state: "complete", output: "mp4-1" }, + bytes: { ...state().bytes, mp4: 2_000 }, + }, + }; + + await fallBackAndFail(); + + expect(video().status).toBe("ready"); + expect(developerVideoOutputs(video())).toEqual({ + hls: false, + mp4: true, + poster: true, + }); + expect(video().renditions).toBeNull(); + }); + + it("keeps the media viewers were shown once the single job has finished", async () => { + await failSlice(1); + await failSlice(1); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + const hls = state().hls; + if (hls.state !== "running") throw new Error("single job is not running"); + mocks.readText.mockResolvedValue( + [ + "#EXTM3U", + "#EXT-X-STREAM-INF:BANDWIDTH=4000000", + "1080p.m3u8", + "#EXT-X-STREAM-INF:BANDWIDTH=500000", + "360p.m3u8", + ].join("\n"), + ); + await applyDeveloperVideoProgress( + { + videoId: "video-1", + phase: "complete", + ladder: sliceLadder(sliceResult(300), AUDIO_BYTES), + }, + "hls", + hls.token, + ); + expect(state().hls).toEqual({ state: "complete", output: hls.token }); + expect(video().storedBytes).toBe(1_000 + 5_700 + AUDIO_BYTES + 50 * 2); + mocks.store.video = { + ...video(), + processing: { ...state(), mp4: { state: "failed", reason: "x" } }, + }; + + await sweepSupersededDeveloperVideoOutputs("video-1", "encode-1"); + + const keep = mocks.deleteObjects.mock.calls[0]?.[1] as ( + key: string, + ) => boolean; + expect(keep(`${FOLDER}/hls/1080p-0-${latest(0).token}.m4s`)).toBe(true); + expect(keep(`${FOLDER}/hls/audio-0-${latest(0).token}.m4s`)).toBe(true); + expect(keep(`${FOLDER}/slices/1080p-0-${latest(0).token}.m3u8`)).toBe(true); + expect(keep(`${FOLDER}/slices/audio-0-${latest(0).token}.m3u8`)).toBe(true); + expect(keep(`${FOLDER}/slices/master-0-${latest(0).token}.m3u8`)).toBe( + false, + ); + expect( + keep(`developer/app-1/video-1/encode-1/${hls.token}/hls/1080p.m4s`), + ).toBe(true); + }); + + it("stops the slice jobs still running when the slices are given up", async () => { + await failSlice(1); + await failSlice(1); + + expect(state().sliced?.state).toBe("abandoned"); + expect(mocks.cancelJob.mock.calls.map(([jobId]) => jobId).sort()).toEqual([ + "slice-job-2", + "slice-job-3", + ]); + }); + + it("ends errored when the workflow runs out of time before anything else finished", async () => { + mocks.store.video = { + ...video(), + processing: { ...state(), mp4: { state: "failed", reason: "no mp4" } }, + }; + + await failDeveloperVideoProcessing( + "video-1", + "encode-1", + "Processing took too long and was stopped. Upload the video again.", + ); + + expect(state().sliced?.state).toBe("abandoned"); + expect(video().status).toBe("errored"); + expect(video().renditions).toBeNull(); + }); +}); + +describe("slices alongside the rest of Infra processing", () => { + const announced = () => + mocks.announce.mock.calls + .map(([, step]) => (step as { type: string | null }).type) + .filter((type) => type !== null); + + it("marks every slice job, and the single job it falls back to, as Infra", async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + expect(sent()).toHaveLength(3); + for (const entry of sent()) expect(entry.body.infra).toBe(true); + + await failSlice(1); + await failSlice(1); + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + const [path, body] = mocks.startJob.mock.calls.at(-1) ?? []; + expect(path).toBe("/video/ladder"); + expect(body).not.toHaveProperty("range"); + expect((body as { infra?: boolean }).infra).toBe(true); + }); + + it("stamps the encode busy when a slice finds every worker full", async () => { + vi.useFakeTimers({ now: 1_000_000 }); + mocks.startJob.mockResolvedValue({ status: "busy" }); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "busy", + ); + + expect(state().busyAt).toBe(1_000_000); + expect( + state().sliced?.slices.every((slice) => slice.job.state === "pending"), + ).toBe(true); + }); + + it("is not reported busy while other slices of the video are encoding", async () => { + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + mocks.startJob.mockResolvedValue({ status: "busy" }); + await finish(0); + expect(state().sliced?.slices.map((slice) => slice.job.state)).toEqual([ + "complete", + "running", + "running", + "pending", + ]); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + expect(state().sliced?.slices[3]?.job.state).toBe("pending"); + }); + + it("starts the MP4 and the slices on the first pass, never a single HLS job", async () => { + const results = await dispatchDeveloperVideoParts("video-1", "encode-1"); + + expect(results.map((result) => result.status)).toEqual([ + "fulfilled", + "fulfilled", + ]); + const paths = mocks.startJob.mock.calls.map(([path, body]) => [ + path, + (body as { range?: unknown }).range !== undefined, + ]); + expect(paths.filter(([path]) => path === "/video/process")).toHaveLength(1); + expect( + paths.filter(([path, ranged]) => path === "/video/ladder" && !ranged), + ).toHaveLength(0); + expect(sent()).toHaveLength(3); + expect(state().hls).toEqual({ state: "pending" }); + expect(state().mp4.state).toBe("running"); + }); + + it("cancels a slice job whose start lands after the slices were given up", async () => { + mocks.startJob.mockImplementation(async (_path, body) => { + const range = (body as { range?: { startSeconds: number } }).range; + if (range?.startSeconds === 20) { + mocks.store.video = { + ...video(), + processing: { + ...state(), + sliced: { + ...(state().sliced as DeveloperVideoSlicedHls), + state: "abandoned", + reason: "x", + }, + }, + }; + } + return { + status: "started", + jobId: `slice-job-${STARTS.indexOf(range?.startSeconds ?? -1)}`, + }; + }); + mocks.cancelJob.mockResolvedValue("cancelled"); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(mocks.cancelJob).toHaveBeenCalledWith("slice-job-1"); + expect(mocks.cancelJob).not.toHaveBeenCalledWith("slice-job-0"); + }); + + it("cancels a slice job whose start lands after the video was deleted", async () => { + mocks.startJob.mockImplementation(async (_path, body) => { + const range = (body as { range?: { startSeconds: number } }).range; + if (range?.startSeconds === 20) { + mocks.store.video = { ...video(), deletedAt: new Date() }; + } + return { + status: "started", + jobId: `slice-job-${STARTS.indexOf(range?.startSeconds ?? -1)}`, + }; + }); + mocks.cancelJob.mockResolvedValue("cancelled"); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(mocks.cancelJob).toHaveBeenCalledWith("slice-job-1"); + }); + + it("becomes ready, settles the first month and sends video.ready exactly once", async () => { + await dispatchDeveloperVideoParts("video-1", "encode-1"); + const mp4 = state().mp4; + if (mp4.state !== "running") throw new Error("mp4 is not running"); + + await finish(0); + + expect(video().status).toBe("ready"); + expect(mocks.firstMonth).toHaveBeenCalledTimes(1); + expect(announced()).toEqual([]); + expect(state().readyEventDueAt).toEqual(expect.any(Number)); + + await finish(1); + await finish(2); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete" }, + "mp4", + mp4.token, + ); + expect(state().mp4.state).toBe("complete"); + expect(announced()).toEqual([]); + + await finish(3); + + expect(state().hls).toEqual({ state: "complete", output: "sliced-1" }); + expect(announced()).toEqual(["video.ready"]); + expect(state().readyEventDueAt).toBeUndefined(); + expect(mocks.firstMonth).toHaveBeenCalledTimes(1); + + await finish(3, latest(3).token); + await readDeveloperVideoParts("video-1", "encode-1"); + expect(announced()).toEqual(["video.ready"]); + expect(mocks.firstMonth).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-sliced-state.test.ts b/apps/web/__tests__/unit/developer-video-sliced-state.test.ts new file mode 100644 index 00000000000..56397fe72da --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-sliced-state.test.ts @@ -0,0 +1,614 @@ +import type { DeveloperVideoProcessingState } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { + abandonSlicedHls, + beginPart, + beginSlices, + completeSlicedHls, + DEVELOPER_VIDEO_MAX_LOST_JOBS, + developerVideoRedrive, + developerVideoStoredBytes, + failUnfinishedParts, + findRunningSlice, + LOST_JOB_CONFIRM_MS, + markSliceChecked, + markSliceDispatched, + markSliceNotFound, + releaseSlice, + requeueLostSlice, + resolveDeveloperVideoStatus, + retryUnavailableHls, + SILENT_JOB_CHECK_MS, + settlePart, + settleSlice, + slicedHlsAction, + slicedHlsEncoding, + slicedHlsStep, +} from "@/lib/developer-video-state"; +import { processingState } from "../fixtures/developer-video"; +import { + finishedSlice, + runningJob, + slice, + slicedHls, + sliceResult, +} from "../fixtures/developer-video-slices"; + +const NOW = 10_000; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +function begin( + state: DeveloperVideoProcessingState, + tokens: string[], + now = NOW, +) { + const started = beginSlices(state, { tokens, now }); + if (!started) throw new Error("no slice was due"); + return started; +} + +function confirm(state: DeveloperVideoProcessingState, token: string) { + const next = markSliceDispatched(state, { token, jobId: `job-${token}` }); + if (!next) throw new Error(`no running slice ${token}`); + return next; +} + +function complete( + state: DeveloperVideoProcessingState, + token: string, + seconds: number, + extra: { audioBytes?: number; posterBytes?: number } = {}, +) { + const next = settleSlice(state, token, { + type: "complete", + result: sliceResult(seconds, { audioBytes: extra.audioBytes }), + posterBytes: extra.posterBytes, + }); + if (!next) throw new Error(`slice ${token} did not settle`); + return next; +} + +function fail(state: DeveloperVideoProcessingState, token: string) { + const next = settleSlice(state, token, { type: "failed", reason: "boom" }); + if (!next) throw new Error(`slice ${token} did not settle`); + return next; +} + +const jobStates = (state: DeveloperVideoProcessingState) => + state.sliced?.slices.map((entry) => entry.job.state); + +const fresh = () => processingState({ sliced: slicedHls() }); + +describe("sending slices to the encoder", () => { + it("sends the first three in playback order and holds the rest back", () => { + const { state, begun } = begin(fresh(), ["a", "b", "c"]); + + expect(begun).toEqual([ + { index: 0, token: "a" }, + { index: 1, token: "b" }, + { index: 2, token: "c" }, + ]); + expect(jobStates(state)).toEqual([ + "running", + "running", + "running", + "pending", + ]); + expect(state.sliced?.slices.map((entry) => entry.attempts)).toEqual([ + 1, 1, 1, 0, + ]); + expect(beginSlices(state, { tokens: ["d"], now: NOW })).toBeNull(); + expect(slicedHlsAction(state.sliced ?? slicedHls(), NOW)).toBe("wait"); + }); + + it("sends the next slice when one finishes", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + + const { begun } = begin(state, ["d", "e", "f"]); + + expect(begun).toEqual([{ index: 3, token: "d" }]); + }); + + it("leaves the hls part waiting and never sends the whole job while slices encode", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + + expect(state.hls).toEqual({ state: "pending" }); + expect(slicedHlsEncoding(state)).not.toBeNull(); + }); + + it("sends a slice again when its start was never confirmed", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + const later = NOW + 2 * MINUTE; + + const again = begin(state, ["x", "y", "z"], later); + + expect(again.begun).toEqual([ + { index: 0, token: "x" }, + { index: 1, token: "y" }, + { index: 2, token: "z" }, + ]); + expect(again.state.sliced?.slices.map((entry) => entry.attempts)).toEqual([ + 1, 1, 1, 0, + ]); + }); + + it("gives a slice back when the encoder was busy, at no cost to its attempts", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + + const released = releaseSlice(state, "b"); + + expect(released?.sliced?.slices[1]).toMatchObject({ + job: { state: "pending" }, + attempts: 0, + }); + expect(releaseSlice(state, "nope")).toBeNull(); + }); +}); + +describe("slice results arriving", () => { + it("makes the video ready when the first slice is in", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + expect(resolveDeveloperVideoStatus(state, "processing").status).toBe( + "processing", + ); + + state = complete(state, "a", 20, { audioBytes: 300, posterBytes: 50 }); + + expect(resolveDeveloperVideoStatus(state, "processing")).toEqual({ + status: "ready", + errorReason: null, + }); + expect(state.sliced).toMatchObject({ exposed: true, hasAudio: true }); + expect(state.poster).toEqual({ output: "a" }); + expect(state.bytes).toMatchObject({ hls: 6_000, poster: 50 }); + expect(developerVideoStoredBytes(state)).toBe(1_000 + 6_000 + 50); + }); + + it("ignores a result delivered twice", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + + expect( + settleSlice(state, "a", { type: "complete", result: sliceResult(20) }), + ).toBeNull(); + expect(settleSlice(state, "a", { type: "failed", reason: "x" })).toBeNull(); + }); + + it("ignores a result from an attempt that was replaced", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + const { state: resent } = begin(state, ["x", "y", "z"], NOW + 2 * MINUTE); + + expect( + settleSlice(resent, "a", { type: "complete", result: sliceResult(20) }), + ).toBeNull(); + expect(findRunningSlice(resent, "x")?.index).toBe(0); + }); + + it("is not ready, and offers nothing, when a later slice finishes before the first", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + + state = complete(state, "c", 96); + state = complete(state, "b", 96); + + expect(resolveDeveloperVideoStatus(state, "processing").status).toBe( + "processing", + ); + expect(state.sliced?.exposed).toBeNull(); + expect(state.bytes.hls).toBe(11_400); + + state = complete(state, "a", 20); + + expect(resolveDeveloperVideoStatus(state, "processing").status).toBe( + "ready", + ); + expect(state.sliced?.exposed).toBe(true); + }); + + it("counts the poster and the audio once, with the first slice only", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + + state = complete(state, "b", 96, { posterBytes: 999 }); + expect(state.poster).toBeNull(); + expect(state.bytes.poster).toBe(0); + + state = complete(state, "a", 20, { audioBytes: 300, posterBytes: 50 }); + expect(state.bytes).toMatchObject({ poster: 50, hls: 5_700 * 2 + 300 }); + }); + + it("holds the growing ladder back when the MP4 was already playable", () => { + const { state: started } = begin( + processingState({ + mp4: { state: "complete", output: "mp4-1" }, + sliced: slicedHls(), + }), + ["a", "b", "c"], + ); + + const state = complete(started, "a", 20); + + expect(state.sliced?.exposed).toBe(false); + expect(resolveDeveloperVideoStatus(state, "ready").status).toBe("ready"); + }); + + it("keeps offering the growing ladder when the MP4 finishes after the first slice", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + const token = "mp4-1"; + const running = beginPart(state, "mp4", { token, now: NOW }); + if (!running) throw new Error("mp4 did not start"); + + const done = settlePart(running, "mp4", token, { + type: "complete", + bytes: 2_000, + }); + + expect(done?.sliced?.exposed).toBe(true); + }); + + it("becomes an ordinary complete ladder once every slice is in", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20, { audioBytes: 300 }); + state = complete(state, "b", 96); + state = complete(state, "c", 96); + expect(completeSlicedHls(state)).toBeNull(); + ({ state } = begin(state, ["d"])); + state = complete(state, "d", 88); + + expect(slicedHlsStep(state.sliced ?? slicedHls(), NOW).finalize).toBe(true); + expect(slicedHlsAction(state.sliced ?? slicedHls(), NOW)).toBe("dispatch"); + + const whole = completeSlicedHls(state); + + expect(whole?.hls).toEqual({ state: "complete", output: "sliced-1" }); + expect(whole?.sliced?.state).toBe("complete"); + expect(whole?.bytes.hls).toBe(5_700 * 4 + 300); + expect(whole && slicedHlsEncoding(whole)).toBeNull(); + expect(whole && completeSlicedHls(whole)).toBeNull(); + }); + + it("refuses a slice whose renditions differ from the others", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + const even = sliceResult(96); + const odd = { + ...even, + renditions: even.renditions.map((rendition, index) => + index === 0 ? { ...rendition, width: 1280, height: 720 } : rendition, + ), + }; + + const refused = settleSlice(state, "b", { type: "complete", result: odd }); + + expect(refused?.sliced?.slices[1]?.job).toEqual({ state: "pending" }); + expect(refused?.sliced?.slices[1]?.result).toBeUndefined(); + }); +}); + +describe("a slice that is lost", () => { + it("is asked about after three quiet minutes", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = confirm(state, "a"); + const sliced = state.sliced ?? slicedHls(); + + expect(slicedHlsStep(sliced, NOW + SILENT_JOB_CHECK_MS - 1).check).toEqual( + [], + ); + expect(slicedHlsStep(sliced, NOW + SILENT_JOB_CHECK_MS).check).toEqual([0]); + + const checked = markSliceChecked(state, { + token: "a", + now: NOW + SILENT_JOB_CHECK_MS, + }); + expect( + slicedHlsStep( + checked?.sliced ?? slicedHls(), + NOW + SILENT_JOB_CHECK_MS + 1, + ).check, + ).toEqual([]); + }); + + it("is sent again without costing an attempt", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = confirm(state, "a"); + const missing = markSliceNotFound(state, { token: "a", now: NOW }); + expect( + missing && markSliceNotFound(missing, { token: "a", now: NOW }), + ).toBe(null); + expect(NOW + LOST_JOB_CONFIRM_MS).toBeGreaterThan(NOW); + + const requeued = requeueLostSlice(missing ?? state, "a"); + + expect(requeued?.sliced?.slices[0]).toMatchObject({ + job: { state: "pending" }, + attempts: 0, + lostJobs: 1, + }); + expect(requeued?.sliced?.state).toBe("encoding"); + expect(begin(requeued ?? state, ["x"]).begun).toEqual([ + { index: 0, token: "x" }, + ]); + }); + + it("that keeps vanishing ends the sliced encode and falls back to one job", () => { + const state = processingState({ + sliced: slicedHls({ + slices: [ + slice(0, 20, { + job: runningJob("a"), + attempts: 1, + lostJobs: DEVELOPER_VIDEO_MAX_LOST_JOBS, + }), + slice(20, null), + ], + }), + }); + + const next = requeueLostSlice(state, "a"); + + expect(next?.sliced?.state).toBe("abandoned"); + expect(next?.hls).toEqual({ state: "pending" }); + expect(next && slicedHlsEncoding(next)).toBeNull(); + }); +}); + +describe("a slice that fails", () => { + it("is tried once more", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + + const failed = fail(state, "b"); + + expect(failed.sliced?.state).toBe("encoding"); + expect(failed.sliced?.slices[1]).toMatchObject({ + job: { state: "pending" }, + attempts: 1, + }); + }); + + it("twice hands the whole video to a single job, which gets one attempt", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = fail(state, "b"); + ({ state } = begin(state, ["b2"])); + + const abandoned = fail(state, "b2"); + + expect(abandoned.sliced).toMatchObject({ state: "abandoned" }); + expect(abandoned.sliced?.slices[1]?.job.state).toBe("failed"); + expect(abandoned.hls).toEqual({ state: "pending" }); + expect(slicedHlsEncoding(abandoned)).toBeNull(); + expect(resolveDeveloperVideoStatus(abandoned, "processing").status).toBe( + "processing", + ); + + const whole = beginPart(abandoned, "hls", { token: "w", now: NOW }); + if (!whole) throw new Error("whole job did not start"); + const ended = settlePart(whole, "hls", "w", { + type: "failed", + reason: "still broken", + }); + + expect(ended?.hls).toEqual({ state: "failed", reason: "still broken" }); + }); + + it("ignores the other slices' results once the single job has taken over", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = fail(state, "b"); + ({ state } = begin(state, ["b2"])); + state = fail(state, "b2"); + + expect( + settleSlice(state, "a", { type: "complete", result: sliceResult(20) }), + ).toBeNull(); + expect(beginSlices(state, { tokens: ["x"], now: NOW })).toBeNull(); + }); + + const readyThenAbandoned = () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + state = fail(state, "b"); + ({ state } = begin(state, ["b2"])); + return fail(state, "b2"); + }; + + it("after the video was ready keeps it ready and playing while the single job runs", () => { + const abandoned = readyThenAbandoned(); + + expect(resolveDeveloperVideoStatus(abandoned, "ready").status).toBe( + "ready", + ); + expect(abandoned.bytes.hls).toBe(5_700); + }); + + it("is replaced by the single job's ladder when that finishes", () => { + const whole = beginPart(readyThenAbandoned(), "hls", { + token: "w", + now: NOW, + }); + if (!whole) throw new Error("whole job did not start"); + + const done = settlePart(whole, "hls", "w", { + type: "complete", + bytes: 9_000, + }); + + expect(done?.hls).toEqual({ state: "complete", output: "w" }); + expect(done?.bytes.hls).toBe(9_000); + expect(done && resolveDeveloperVideoStatus(done, "ready").status).toBe( + "ready", + ); + }); + + const wholeJobFailed = (state: DeveloperVideoProcessingState) => { + const whole = beginPart(state, "hls", { token: "w", now: NOW }); + if (!whole) throw new Error("whole job did not start"); + const ended = settlePart(whole, "hls", "w", { + type: "failed", + reason: "This video could not be processed.", + }); + if (!ended) throw new Error("whole job did not settle"); + return ended; + }; + + it("ends as errored when the single job fails too and there is no MP4", () => { + const state = { + ...readyThenAbandoned(), + mp4: { state: "failed" as const, reason: "no mp4 either" }, + }; + + const ended = wholeJobFailed(state); + + expect(resolveDeveloperVideoStatus(ended, "ready")).toEqual({ + status: "errored", + errorReason: "no mp4 either", + }); + expect(ended.bytes.hls).toBe(0); + }); + + it("stays ready on its MP4 when the single job fails too", () => { + const state = { + ...readyThenAbandoned(), + mp4: { state: "complete" as const, output: "mp4-1" }, + }; + + const ended = wholeJobFailed(state); + + expect(resolveDeveloperVideoStatus(ended, "ready").status).toBe("ready"); + expect(ended.bytes.hls).toBe(0); + }); + + it("stays ready while the MP4 is still being made after both failed", () => { + const state = { + ...readyThenAbandoned(), + mp4: runningJob("mp4-1"), + }; + + const ended = wholeJobFailed(state); + + expect(resolveDeveloperVideoStatus(ended, "ready").status).toBe("ready"); + }); +}); + +describe("the ways a sliced encode ends early", () => { + it("is given up when the workflow runs out of time", () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + + const failed = failUnfinishedParts(state, "Processing took too long"); + + expect(failed.sliced?.state).toBe("abandoned"); + expect(failed.hls).toMatchObject({ state: "failed" }); + expect(failed.mp4).toMatchObject({ state: "failed" }); + expect(failed.bytes.hls).toBe(0); + expect(resolveDeveloperVideoStatus(failed, "ready")).toEqual({ + status: "errored", + errorReason: "Processing took too long", + }); + }); + + it("gives way to a single job that finished first", () => { + const { state } = begin(fresh(), ["a", "b", "c"]); + const whole = { ...state, hls: runningJob("w") }; + + const done = settlePart(whole, "hls", "w", { + type: "complete", + bytes: 9_000, + }); + + expect(done?.sliced?.state).toBe("abandoned"); + expect(done?.bytes.hls).toBe(9_000); + }); + + it("can be given up directly, leaving the single job one attempt", () => { + const abandoned = abandonSlicedHls(fresh(), "not available"); + + expect(abandoned.sliced).toMatchObject({ + state: "abandoned", + reason: "not available", + }); + expect(abandoned.attempts.hls).toBe(1); + expect(abandonSlicedHls(abandoned, "again")).toBe(abandoned); + }); + + it("leaves a video with no slices exactly as it was", () => { + const plain = processingState(); + expect(abandonSlicedHls(plain, "x")).toBe(plain); + expect(failUnfinishedParts(plain, "x").sliced).toBeUndefined(); + }); +}); + +describe("a ready video whose sliced encode lost its workflow", () => { + const video = ( + processing: DeveloperVideoProcessingState, + updatedAt: number, + ) => ({ + status: "ready" as const, + playbackId: "playback-id-0123456789ab", + duration: 300, + deletedAt: null, + updatedAt: new Date(updatedAt), + processing, + }); + const playing = () => { + let { state } = begin(fresh(), ["a", "b", "c"]); + state = complete(state, "a", 20); + return state; + }; + + it("is left alone while slices are encoding, however long they take", () => { + const state = { ...playing(), mp4: runningJob("mp4-1") }; + + expect(developerVideoRedrive(video(state, NOW), NOW + 30 * MINUTE)).toBe( + null, + ); + }); + + it("is picked up again when no slice is encoding and nothing has moved for two minutes", () => { + let state = playing(); + state = releaseSlice(state, "b") ?? state; + state = releaseSlice(state, "c") ?? state; + + expect(developerVideoRedrive(video(state, NOW), NOW + MINUTE)).toBeNull(); + expect(developerVideoRedrive(video(state, NOW), NOW + 2 * MINUTE)).toBe( + "retry-hls", + ); + + const retried = retryUnavailableHls(state, NOW + 2 * MINUTE); + expect(retried?.startedAt).toBe(NOW + 2 * MINUTE); + expect(retried?.sliced?.state).toBe("encoding"); + }); + + it("resends a slice that has been encoding for six hours", () => { + const state = playing(); + const later = NOW + 6 * HOUR; + + expect(developerVideoRedrive(video(state, later - 1), later)).toBe( + "retry-hls", + ); + + const retried = retryUnavailableHls(state, later); + expect(jobStates(retried ?? state)).toEqual([ + "complete", + "pending", + "pending", + "pending", + ]); + }); + + it("is picked up again when every slice is in but the ladder was never written", () => { + const state = processingState({ + sliced: slicedHls({ + exposed: true, + slices: [ + finishedSlice(0, 20, "a", sliceResult(20)), + finishedSlice(20, null, "b", sliceResult(100)), + ], + }), + }); + + expect(developerVideoRedrive(video(state, NOW), NOW + 2 * MINUTE)).toBe( + "retry-hls", + ); + expect(retryUnavailableHls(state, NOW + 2 * MINUTE)).not.toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-slices.test.ts b/apps/web/__tests__/unit/developer-video-slices.test.ts new file mode 100644 index 00000000000..ef3abfae5c0 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-slices.test.ts @@ -0,0 +1,476 @@ +import { describe, expect, it } from "vitest"; +import { + parseDeveloperVideoSliceLadder, + slicedHlsKeys, + slicedHlsMediaFiles, + slicedHlsPlaylistFiles, + slicedHlsPlaylistText, +} from "@/lib/developer-video-sliced-ladder"; +import { HLS_SEGMENT_SECONDS } from "@/lib/developer-video-sliced-playlist"; +import { + planDeveloperVideoSlices, + playableSlices, + SLICED_HLS, + slicedFileName, + slicedHlsBytes, + slicedHlsPlayback, + slicedHlsRenditions, +} from "@/lib/developer-video-slices"; +import { processingState } from "../fixtures/developer-video"; +import { + finishedSlice, + runningJob, + slice, + slicedHls, + slicePlaylist, + sliceResult, +} from "../fixtures/developer-video-slices"; + +describe("which videos are encoded in slices", () => { + it("keeps the single job for a video under twenty minutes", () => { + expect(planDeveloperVideoSlices(8)).toBeNull(); + expect(planDeveloperVideoSlices(334)).toBeNull(); + expect(planDeveloperVideoSlices(1199.9)).toBeNull(); + }); + + it("keeps the single job when the length is not known", () => { + expect(planDeveloperVideoSlices(null)).toBeNull(); + expect(planDeveloperVideoSlices(Number.NaN)).toBeNull(); + expect(planDeveloperVideoSlices(Number.POSITIVE_INFINITY)).toBeNull(); + }); + + it("slices a video from twenty minutes", () => { + expect(planDeveloperVideoSlices(1200)).toEqual([ + { startSeconds: 0, durationSeconds: 20 }, + { startSeconds: 20, durationSeconds: 120 }, + { startSeconds: 140, durationSeconds: 356 }, + { startSeconds: 496, durationSeconds: 356 }, + { startSeconds: 852 }, + ]); + }); + + it("puts a two minute slice between the first and the thirds of a long video", () => { + expect(planDeveloperVideoSlices(1266)).toEqual([ + { startSeconds: 0, durationSeconds: 20 }, + { startSeconds: 20, durationSeconds: 120 }, + { startSeconds: 140, durationSeconds: 376 }, + { startSeconds: 516, durationSeconds: 376 }, + { startSeconds: 892 }, + ]); + }); + + it("sends an hour out in even slices once thirds would be long", () => { + const ranges = planDeveloperVideoSlices(3796.5) ?? []; + expect(ranges.slice(0, 3)).toEqual([ + { startSeconds: 0, durationSeconds: 20 }, + { startSeconds: 20, durationSeconds: 120 }, + { startSeconds: 140, durationSeconds: 308 }, + ]); + expect(ranges).toHaveLength(14); + const middle = ranges.slice(2, -1).map((range) => range.durationSeconds); + expect(new Set(middle)).toEqual(new Set([308])); + expect(ranges.at(-1)).toEqual({ startSeconds: 3528 }); + }); + + it("keeps the longest video allowed to a few rounds of jobs per worker", () => { + const ranges = planDeveloperVideoSlices(4 * 60 * 60) ?? []; + expect(ranges).toHaveLength(14); + expect(ranges.slice(0, 4)).toEqual([ + { startSeconds: 0, durationSeconds: 20 }, + { startSeconds: 20, durationSeconds: 120 }, + { startSeconds: 140, durationSeconds: 840 }, + { startSeconds: 980, durationSeconds: 1192 }, + ]); + }); + + it("never cuts a long video into slices shorter than the steady size, apart from the start", () => { + for (const seconds of [1900, 2400, 3600, 7200, 14_400]) { + const ranges = planDeveloperVideoSlices(seconds) ?? []; + const steady = ranges + .slice(2, -1) + .map((range) => range.durationSeconds ?? 0); + for (const size of steady.slice(1)) { + expect(size).toBeGreaterThanOrEqual(SLICED_HLS.steadyMinSliceSeconds); + } + const last = ranges.at(-1); + expect(seconds - (last?.startSeconds ?? 0)).toBeGreaterThan( + SLICED_HLS.steadyMinSliceSeconds / 2, + ); + } + }); + + it("always covers the whole video on segment boundaries with no gap or overlap", () => { + for ( + let duration = SLICED_HLS.minDurationSeconds; + duration <= 4 * 60 * 60; + duration += 37.3 + ) { + const ranges = planDeveloperVideoSlices(duration) ?? []; + expect(ranges.length).toBeGreaterThanOrEqual(2); + expect(ranges.length).toBeLessThanOrEqual( + 2 + SLICED_HLS.tailWorkers * SLICED_HLS.rounds, + ); + expect(ranges[0]).toEqual({ + startSeconds: 0, + durationSeconds: SLICED_HLS.firstSliceSeconds, + }); + for (const [index, range] of ranges.entries()) { + expect(range.startSeconds % HLS_SEGMENT_SECONDS).toBe(0); + const next = ranges[index + 1]; + if (next) { + expect(Number.isInteger(range.durationSeconds)).toBe(true); + expect(range.startSeconds + (range.durationSeconds ?? 0)).toBe( + next.startSeconds, + ); + } else { + expect(range.durationSeconds).toBeUndefined(); + expect(range.startSeconds).toBeLessThan(duration); + } + } + } + }); + + it("never plans a slice a viewer playing from the first one would catch up with", () => { + for ( + let duration = SLICED_HLS.minDurationSeconds; + duration <= 4 * 60 * 60; + duration += 61 + ) { + const ranges = planDeveloperVideoSlices(duration) ?? []; + for (const range of ranges.slice(1)) { + const length = range.durationSeconds ?? duration - range.startSeconds; + expect(length).toBeLessThanOrEqual( + range.startSeconds * SLICED_HLS.rampFactor + HLS_SEGMENT_SECONDS, + ); + } + } + }); +}); + +describe("where slices are stored", () => { + it("puts the slice and the attempt in every file name", () => { + expect(slicedFileName("1080p.m4s", 3, "tok_A-1")).toBe( + "1080p-3-tok_A-1.m4s", + ); + expect(slicedFileName("master.m3u8", 0, "t")).toBe("master-0-t.m3u8"); + }); + + it("keeps media where a ladder's media lives and slice playlists beside it", () => { + expect( + slicedHlsKeys({ appId: "app-1", id: "video-1" }, "encode-1", "out"), + ).toEqual({ + hlsPrefix: "developer/app-1/video-1/encode-1/out/hls/", + slicePrefix: "developer/app-1/video-1/encode-1/out/slices/", + }); + }); +}); + +describe("a slice job's report", () => { + it("is read with its renditions largest first", () => { + expect( + parseDeveloperVideoSliceLadder({ + renditions: [ + { name: "360p", width: 640, height: 360, bandwidth: 5, bytes: 7 }, + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 40, + averageBandwidth: 30, + codecs: "avc1.640028", + bytes: 50, + }, + ], + audio: { bytes: 9 }, + }), + ).toEqual({ + renditions: [ + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 40, + averageBandwidth: 30, + bytes: 50, + codecs: "avc1.640028", + }, + { + name: "360p", + width: 640, + height: 360, + bandwidth: 5, + averageBandwidth: 5, + bytes: 7, + codecs: null, + }, + ], + audioBytes: 9, + }); + }); + + it("is refused when it lists no renditions", () => { + expect(parseDeveloperVideoSliceLadder({ renditions: [] })).toBeNull(); + expect(parseDeveloperVideoSliceLadder(undefined)).toBeNull(); + }); +}); + +describe("what a viewer can be given", () => { + const done = (exposed: boolean | null) => + slicedHls({ + exposed, + slices: [ + finishedSlice(0, 20, "a", sliceResult(20)), + finishedSlice(20, 96, "b", sliceResult(96)), + slice(116, 96, { job: runningJob("c") }), + finishedSlice(212, null, "d", sliceResult(88)), + ], + }); + + it("is the finished slices from the start, stopping at the first gap", () => { + expect(playableSlices(done(true)).map((entry) => entry.job.output)).toEqual( + ["a", "b"], + ); + }); + + it("is nothing until the first slice is in", () => { + const sliced = slicedHls({ + exposed: null, + slices: [ + slice(0, 20, { job: runningJob("a") }), + finishedSlice(20, 96, "b", sliceResult(96)), + ], + }); + expect(slicedHlsPlayback(processingState({ sliced }))).toBeNull(); + }); + + it("is the growing ladder while slices are encoding", () => { + const playback = slicedHlsPlayback(processingState({ sliced: done(true) })); + expect(playback?.slices).toHaveLength(2); + expect(playback?.ended).toBe(false); + expect(playback?.output).toBe("sliced-1"); + }); + + it("is nothing for a video that already played from its MP4 when the first slice landed", () => { + expect( + slicedHlsPlayback(processingState({ sliced: done(false) })), + ).toBeNull(); + }); + + it("is the stored ladder, not the slices, once the ladder is whole", () => { + expect( + slicedHlsPlayback( + processingState({ + hls: { state: "complete", output: "sliced-1" }, + sliced: { ...done(true), state: "complete" }, + }), + ), + ).toBeNull(); + }); + + it("keeps the finished slices while one job encodes the video instead", () => { + const abandoned = { ...done(true), state: "abandoned" as const }; + for (const hls of [{ state: "pending" as const }, runningJob("whole")]) { + const playback = slicedHlsPlayback( + processingState({ hls, sliced: abandoned }), + ); + expect(playback?.slices).toHaveLength(2); + expect(playback?.ended).toBe(false); + } + expect( + slicedHlsPlayback( + processingState({ + hls: { state: "failed", reason: "x" }, + sliced: abandoned, + }), + ), + ).toBeNull(); + }); + + it("carries its end marker once every slice is in", () => { + const sliced = slicedHls({ + exposed: true, + slices: [ + finishedSlice(0, 20, "a", sliceResult(20)), + finishedSlice(20, null, "b", sliceResult(100)), + ], + }); + expect(slicedHlsPlayback(processingState({ sliced }))?.ended).toBe(true); + }); +}); + +describe("accounting for a ladder made of slices", () => { + const sliced = slicedHls({ + exposed: true, + hasAudio: true, + slices: [ + finishedSlice(0, 20, "a", sliceResult(20, { audioBytes: 300 })), + slice(20, 80, { job: runningJob("b") }), + finishedSlice(100, null, "c", sliceResult(60, { scale: 2 })), + ], + }); + + it("counts every stored media file once, the audio with the first slice", () => { + expect(slicedHlsBytes(sliced)).toBe(5_700 + 300 + 11_400); + }); + + it("reports each rendition's bytes summed and its bitrate averaged over time, audio included", () => { + expect(slicedHlsRenditions(sliced)).toEqual([ + { + name: "1080p", + width: 1920, + height: 1080, + bitrate: (3_000_000 * 20 + 6_000_000 * 60) / 80 + 128_000, + bytes: 15_000, + codecs: "avc1.640028,mp4a.40.2", + }, + { + name: "360p", + width: 640, + height: 360, + bitrate: (400_000 * 20 + 800_000 * 60) / 80 + 128_000, + bytes: 2_100, + codecs: "avc1.64001e,mp4a.40.2", + }, + ]); + }); + + it("leaves audio out of a silent video's figures", () => { + const silent = slicedHls({ + slices: [finishedSlice(0, null, "a", sliceResult(20))], + }); + expect(slicedHlsRenditions(silent)[0]).toMatchObject({ + bitrate: 3_000_000, + codecs: "avc1.640028", + }); + }); +}); + +describe("playlists built from stored slices", () => { + const video = { appId: "app-1", id: "video-1" }; + const slices = [ + finishedSlice(0, 8, "a", sliceResult(8, { audioBytes: 300 })), + finishedSlice(8, null, "b", sliceResult(6)), + ]; + const stored: Record = { + "developer/app-1/video-1/encode-1/out/slices/1080p-0-a.m3u8": slicePlaylist( + "1080p", + [4, 4], + ), + "developer/app-1/video-1/encode-1/out/slices/1080p-1-b.m3u8": slicePlaylist( + "1080p", + [4, 2], + ), + "developer/app-1/video-1/encode-1/out/slices/audio-0-a.m3u8": slicePlaylist( + "audio", + [4.01, 4.01, 4.01, 1.9], + ), + }; + const read = async (key: string) => stored[key] ?? null; + const source = (count: number, ended: boolean) => ({ + video, + encodeId: "encode-1", + output: "out", + slices: playableSlices(slicedHls({ slices })).slice(0, count), + hasAudio: true, + ended, + }); + + it("names each slice's media by the attempt that made it", async () => { + expect( + await slicedHlsPlaylistText(source(2, true), "1080p.m3u8", read), + ).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:EVENT", + "#EXT-X-INDEPENDENT-SEGMENTS", + "#EXT-X-START:TIME-OFFSET=0", + '#EXT-X-MAP:URI="1080p-0-a.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p-0-a.m4s", + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@1800", + "1080p-0-a.m4s", + '#EXT-X-MAP:URI="1080p-1-b.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p-1-b.m4s", + "#EXTINF:2.000000,", + "#EXT-X-BYTERANGE:1000@1800", + "1080p-1-b.m4s", + "#EXT-X-ENDLIST", + "", + ].join("\n"), + ); + }); + + it("lists the audio only as far as the video reaches until all slices are in", async () => { + const growing = await slicedHlsPlaylistText( + source(1, false), + "audio.m3u8", + read, + ); + const whole = await slicedHlsPlaylistText( + source(2, true), + "audio.m3u8", + read, + ); + + expect(growing?.match(/#EXTINF/g)).toHaveLength(2); + expect(growing).toContain('#EXT-X-MAP:URI="audio-0-a.m4s"'); + expect(growing).not.toContain("#EXT-X-ENDLIST"); + expect(whole?.match(/#EXTINF/g)).toHaveLength(4); + expect(whole?.endsWith("#EXT-X-ENDLIST\n")).toBe(true); + expect(whole?.startsWith(growing ?? "x")).toBe(true); + }); + + it("builds the master from the slices' results without reading storage", async () => { + const master = await slicedHlsPlaylistText( + source(1, false), + "master.m3u8", + async () => { + throw new Error("storage was read"); + }, + ); + expect(master).toContain("1080p.m3u8"); + expect(master).toContain('URI="audio.m3u8"'); + }); + + it("has no playlist for a rendition the ladder does not have", async () => { + expect( + await slicedHlsPlaylistText(source(2, true), "720p.m3u8", read), + ).toBeNull(); + expect( + await slicedHlsPlaylistText( + { ...source(2, true), hasAudio: false }, + "audio.m3u8", + read, + ), + ).toBeNull(); + }); + + it("fails when a finished slice's playlist is missing from storage", async () => { + await expect( + slicedHlsPlaylistText(source(2, true), "360p.m3u8", read), + ).rejects.toThrow("no stored playlist"); + }); + + it("lists the files a finished ladder keeps", () => { + expect(slicedHlsPlaylistFiles(source(2, true))).toEqual([ + "master.m3u8", + "1080p.m3u8", + "360p.m3u8", + "audio.m3u8", + ]); + expect(slicedHlsMediaFiles(source(2, true))).toEqual([ + "1080p-0-a.m4s", + "360p-0-a.m4s", + "audio-0-a.m4s", + "1080p-1-b.m4s", + "360p-1-b.m4s", + ]); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-workflow-imports.test.ts b/apps/web/__tests__/unit/developer-video-workflow-imports.test.ts new file mode 100644 index 00000000000..e8d41777b31 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-workflow-imports.test.ts @@ -0,0 +1,43 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const LIB = join(import.meta.dirname, "..", "..", "lib"); + +// Every file the processing state reaches through relative imports. +function reachable(entry: string): string[] { + const seen = new Set(); + const visit = (path: string) => { + if (seen.has(path)) return; + seen.add(path); + const source = readFileSync(path, "utf8"); + for (const match of source.matchAll( + /^(?:import|export)\s[^;]*?from\s+"(\.[^"]+)";/gms, + )) { + if (/^(?:import|export)\s+type\s/.test(match[0])) continue; + const target = join(dirname(path), `${match[1]}.ts`); + if (existsSync(target)) visit(target); + } + }; + visit(entry); + return [...seen]; +} + +describe("the processing state the workflow bundles", () => { + // The workflow function imports the state module, and the workflow bundle + // refuses anything that needs a Node module: the whole app stops building. + it("reaches no file that imports a Node module", () => { + const files = reachable(join(LIB, "developer-video-state.ts")); + + expect(files.map((file) => file.slice(LIB.length + 1)).sort()).toEqual([ + "developer-video-sliced-playlist.ts", + "developer-video-slices.ts", + "developer-video-state.ts", + ]); + for (const file of files) { + expect(readFileSync(file, "utf8")).not.toMatch( + /from\s+"(node:[^"]+|crypto|fs|path|zod)"/, + ); + } + }); +}); diff --git a/apps/web/__tests__/unit/infra-delivery-metering.test.ts b/apps/web/__tests__/unit/infra-delivery-metering.test.ts index 6c00388d134..9da10b548da 100644 --- a/apps/web/__tests__/unit/infra-delivery-metering.test.ts +++ b/apps/web/__tests__/unit/infra-delivery-metering.test.ts @@ -256,6 +256,128 @@ describe("HLS segment seconds", () => { }); }); +// A ladder joined from slices: each slice's media file follows its own init +// section, and a range with no offset continues within its own file. +const JOINED_PLAYLIST = [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-PLAYLIST-TYPE:EVENT", + '#EXT-X-MAP:URI="720p-0-tokA.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "720p-0-tokA.m4s", + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000", + "720p-0-tokA.m4s", + '#EXT-X-MAP:URI="720p-1-tokB.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:3000@800", + "720p-1-tokB.m4s", + "#EXTINF:2.000000,", + "#EXT-X-BYTERANGE:1500", + "720p-1-tokB.m4s", + "#EXT-X-ENDLIST", +].join("\n"); + +const SLICE_PLAYLIST = [ + "#EXTM3U", + '#EXT-X-MAP:URI="720p.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:3000@800", + "720p.m4s", + "#EXTINF:2.000000,", + "#EXT-X-BYTERANGE:1500@3800", + "720p.m4s", +].join("\n"); + +describe("a ladder encoded in slices", () => { + const sliceKey = `${base}/hls/720p-1-tokB.m4s`; + const request = (range: [number, number]) => { + const parsed = parseDeliveryLine( + line({ + "cs-uri-stem": sliceKey, + "sc-range-start": String(range[0]), + "sc-range-end": String(range[1]), + "sc-content-len": String(range[1] - range[0] + 1), + }), + ); + if (!parsed) throw new Error("not parsed"); + return parsed; + }; + const resolver = (stored: Record): DeliveryResolver => ({ + async hls(key, mediaName) { + const text = stored[key]; + const name = key.slice(key.lastIndexOf("/") + 1); + return text + ? parseHlsByteRanges(text, mediaName ?? name.replace(/\.m3u8$/, ".m4s")) + : null; + }, + async video() { + return VIDEO_ROW; + }, + }); + + it("counts a slice's media file under its rendition", () => { + expect(request([800, 3799])).toMatchObject({ + appId: APP, + videoId: VIDEO, + kind: "720p", + key: sliceKey.slice(1), + }); + expect( + parseDeliveryLine(line({ "cs-uri-stem": `${base}/hls/audio-0-tokA.m4s` })) + ?.kind, + ).toBe("audio"); + }); + + it("reads only the requested slice's segments from the joined playlist", () => { + expect(parseHlsByteRanges(JOINED_PLAYLIST, "720p-1-tokB.m4s")).toEqual([ + { start: 800, end: 3800, duration: 4 }, + { start: 3800, end: 5300, duration: 2 }, + ]); + expect(parseHlsByteRanges(JOINED_PLAYLIST, "720p-0-tokA.m4s")).toEqual([ + { start: 800, end: 1800, duration: 4 }, + { start: 1800, end: 2800, duration: 4 }, + ]); + expect( + parseHlsByteRanges( + JOINED_PLAYLIST.replace( + "720p-0-tokA.m4s\n#EXTINF:4", + "other.m4s\n#EXTINF:4", + ), + "720p-1-tokB.m4s", + ), + ).toBeNull(); + }); + + it("maps a slice's segment to exact seconds once the ladder is whole", async () => { + const stored = { + [`${base.slice(1)}/hls/720p.m3u8`]: JOINED_PLAYLIST, + }; + expect( + await deliveredSeconds(request([800, 3799]), resolver(stored)), + ).toEqual({ tier: "720p", seconds: 4, method: "playlist" }); + }); + + it("maps it from the slice's own playlist while the ladder is still growing", async () => { + const stored = { + [`${base.slice(1)}/slices/720p-1-tokB.m3u8`]: SLICE_PLAYLIST, + }; + expect( + await deliveredSeconds(request([3800, 5299]), resolver(stored)), + ).toEqual({ tier: "720p", seconds: 2, method: "playlist" }); + }); + + it("counts each slice's bytes once, never against a neighbour", async () => { + const stored = { + [`${base.slice(1)}/hls/720p.m3u8`]: JOINED_PLAYLIST, + }; + const whole = await deliveredSeconds(request([0, 5299]), resolver(stored)); + expect(whole?.seconds).toBeCloseTo(6, 9); + }); +}); + describe("deliveredSeconds", () => { const parsed = (overrides: Record = {}) => { const request = parseDeliveryLine(line(overrides)); diff --git a/apps/web/lib/developer-video-api.ts b/apps/web/lib/developer-video-api.ts index b7792fd0f7b..64c0782438a 100644 --- a/apps/web/lib/developer-video-api.ts +++ b/apps/web/lib/developer-video-api.ts @@ -5,6 +5,7 @@ import { DEVELOPER_VIDEO_MP4_FILE, DEVELOPER_VIDEO_POSTER_FILE, } from "./developer-video-plan"; +import { slicedHlsPlayback } from "./developer-video-slices"; import { developerVideoIsTakenDown, developerVideoPublicMetadata, @@ -47,7 +48,9 @@ export function developerVideoOutputs( return { hls: false, mp4: false, poster: false }; } return { - hls: processing.hls.state === "complete", + hls: + processing.hls.state === "complete" || + slicedHlsPlayback(processing) !== null, mp4: processing.mp4.state === "complete", poster: processing.poster !== null, }; diff --git a/apps/web/lib/developer-video-cancel.ts b/apps/web/lib/developer-video-cancel.ts index 9920271caf2..f48ac3a0a9f 100644 --- a/apps/web/lib/developer-video-cancel.ts +++ b/apps/web/lib/developer-video-cancel.ts @@ -9,10 +9,10 @@ export function runningDeveloperVideoJobIds( processing: DeveloperVideoProcessingState | null, ): string[] { if (!processing) return []; - return DEVELOPER_VIDEO_PARTS.flatMap((part) => { - const current = processing[part]; - return current.state === "running" && current.jobId ? [current.jobId] : []; - }); + return [ + ...DEVELOPER_VIDEO_PARTS.map((part) => processing[part]), + ...(processing.sliced?.slices.map((slice) => slice.job) ?? []), + ].flatMap((job) => (job.state === "running" && job.jobId ? [job.jobId] : [])); } export async function cancelDeveloperVideoJobs( @@ -35,6 +35,7 @@ const hasRunningPart = () => or( sql`JSON_UNQUOTE(JSON_EXTRACT(${developerVideos.processing}, '$.hls.state')) = 'running'`, sql`JSON_UNQUOTE(JSON_EXTRACT(${developerVideos.processing}, '$.mp4.state')) = 'running'`, + sql`JSON_SEARCH(${developerVideos.processing}, 'one', 'running', NULL, '$.sliced.slices[*].job.state') IS NOT NULL`, ); export type DeveloperVideoCancelTarget = diff --git a/apps/web/lib/developer-video-plan.ts b/apps/web/lib/developer-video-plan.ts index 5f9b1b931f6..0edfb1e8049 100644 --- a/apps/web/lib/developer-video-plan.ts +++ b/apps/web/lib/developer-video-plan.ts @@ -218,7 +218,7 @@ export function hlsPlaylistDuration(playlist: string): number { } export function ladderMatchesMaster( - renditions: DeveloperVideoRendition[], + renditions: { name: string }[], master: string, ): boolean { const playlists = new Set(hlsMasterVariantPlaylists(master)); diff --git a/apps/web/lib/developer-video-playback.ts b/apps/web/lib/developer-video-playback.ts index dfb73b3b726..1d1d591529d 100644 --- a/apps/web/lib/developer-video-playback.ts +++ b/apps/web/lib/developer-video-playback.ts @@ -23,6 +23,12 @@ import { developerVideoKeys, HLS_PLAYLIST_CONTENT_TYPE, } from "./developer-video-plan"; +import { slicedHlsPlaylistText } from "./developer-video-sliced-ladder"; +import { + isSlicedMediaFile, + slicedHlsPlayback, + supersededSlicedHls, +} from "./developer-video-slices"; import { developerVideoUrlAnswersAnyOrigin, readDeveloperVideoText, @@ -45,6 +51,11 @@ const SIGNED_URL_WINDOW_SECONDS = 6 * 60 * 60; // for playback after a top-up. const CACHE_SECONDS = 60; +// A ladder still being encoded in slices gains segments every few seconds and +// players ask for its playlists again about once per segment, so those answers +// are shared only for a moment. +const GROWING_CACHE_SECONDS = 2; + const START_MAX_BANDWIDTH = 2_600_000; const START_MAX_SHORT_SIDE = 1080; @@ -60,6 +71,10 @@ const CACHEABLE = { "Cache-Control": `public, max-age=${CACHE_SECONDS}, s-maxage=${CACHE_SECONDS}`, }; +const GROWING = { + "Cache-Control": `public, max-age=${GROWING_CACHE_SECONDS}, s-maxage=${GROWING_CACHE_SECONDS}`, +}; + function notFound(): Response { return new Response("Not found", { status: 404, @@ -467,21 +482,27 @@ async function subtitleTracks(video: PlayableVideo): Promise { // the HLS type, which a player may check before it treats a URL as HLS. const MEDIA_PLAYLIST_CONTENT_TYPE = "text/plain; charset=utf-8"; -function playlistResponse(playlist: string): Response { +function playlistResponse( + playlist: string, + cache: Record = CACHEABLE, +): Response { return new Response(playlist, { headers: { ...PLAYBACK_CORS_HEADERS, - ...CACHEABLE, + ...cache, "Content-Type": HLS_PLAYLIST_CONTENT_TYPE, }, }); } -function mediaPlaylistResponse(playlist: string): Response { +function mediaPlaylistResponse( + playlist: string, + cache: Record = CACHEABLE, +): Response { return new Response(playlist, { headers: { ...PLAYBACK_CORS_HEADERS, - ...CACHEABLE, + ...cache, "Content-Type": MEDIA_PLAYLIST_CONTENT_TYPE, }, }); @@ -587,23 +608,41 @@ async function playableVideoResponse( ? redirectTo(keysFor(processing.poster.output).poster, request) : notFound(); } - if ( - !outputs.hls || - processing.hls.state !== "complete" || - !HLS_FILE_PATTERN.test(file) - ) { + // While a ladder is encoded in slices its playlists are built from the + // slices that are in. Once it is whole they are read from storage like any + // other ladder's. + const slices = slicedHlsPlayback(processing); + const hlsOutput = + processing.hls.state === "complete" + ? processing.hls.output + : slices?.output; + if (!outputs.hls || !hlsOutput || !HLS_FILE_PATTERN.test(file)) { return notFound(); } - const keys = keysFor(processing.hls.output); + const keys = keysFor(hlsOutput); if (MEDIA_FILE_PATTERN.test(file)) { - return redirectTo(`${keys.hlsPrefix}${file}`, request); + const superseded = isSlicedMediaFile(file) + ? supersededSlicedHls(processing) + : null; + const prefix = + superseded && superseded.output !== hlsOutput + ? keysFor(superseded.output).hlsPrefix + : keys.hlsPrefix; + return redirectTo(`${prefix}${file}`, request); } const isMaster = file === DEVELOPER_VIDEO_MASTER_PLAYLIST; const [playlist, subtitles] = await Promise.all([ - readDeveloperVideoText(`${keys.hlsPrefix}${file}`), + slices + ? slicedHlsPlaylistText( + { video, encodeId: processing.encodeId, ...slices }, + file, + readDeveloperVideoText, + ) + : readDeveloperVideoText(`${keys.hlsPrefix}${file}`), isMaster && wantsSubtitles(request) ? subtitleTracks(video) : [], ]); + const cache = slices && !slices.ended ? GROWING : CACHEABLE; if (!playlist) return notFound(); const short = wantsShortSegments(request); if (isMaster) { @@ -615,6 +654,7 @@ async function playableVideoResponse( short ? listShortSegmentPlaylists(master) : master, subtitles, ), + cache, ); } return mediaPlaylistResponse( @@ -628,5 +668,6 @@ async function playableVideoResponse( ), short ? developerVideoUrlAnswersAnyOrigin : undefined, ), + cache, ); } diff --git a/apps/web/lib/developer-video-processing.ts b/apps/web/lib/developer-video-processing.ts index 24abb0ad0e8..de76b83f490 100644 --- a/apps/web/lib/developer-video-processing.ts +++ b/apps/web/lib/developer-video-processing.ts @@ -5,6 +5,7 @@ import type { DeveloperVideoPart, DeveloperVideoProcessingState, DeveloperVideoRendition, + DeveloperVideoSlicedHls, } from "@cap/database/types"; import { serverEnv } from "@cap/env"; import { eq } from "drizzle-orm"; @@ -24,34 +25,68 @@ import { createDeveloperEncodeId, createDeveloperPlaybackId, DEVELOPER_VIDEO_MASTER_PLAYLIST, + type DeveloperVideoLadderTier, developerVideoKeys, developerVideoSourceExtension, + HLS_PLAYLIST_CONTENT_TYPE, hlsPlaylistDuration, ladderMatchesMaster, parseDeveloperVideoLadder, planDeveloperVideoEncode, } from "./developer-video-plan"; import { + parseDeveloperVideoSliceLadder, + readStoredSlicePlaylist, + slicedHlsKeys, + slicedHlsMediaFiles, + slicedHlsPlaylistFiles, + slicedHlsPlaylistText, +} from "./developer-video-sliced-ladder"; +import { HLS_AUDIO_NAME } from "./developer-video-sliced-playlist"; +import { + createSlicedHls, + planDeveloperVideoSlices, + playableSlices, + SLICED_HLS, + slicedFileName, + slicedHlsRenditions, + supersededSlicedHls, +} from "./developer-video-slices"; +import { + abandonSlicedHls, beginPart, + beginSlices, + completeSlicedHls, createDeveloperVideoProcessingState, DEVELOPER_VIDEO_PARTS, type DeveloperVideoPartAction, type DeveloperVideoPartOutcome, type DeveloperVideoRedrive, + type DeveloperVideoSliceOutcome, developerVideoFailureReason, developerVideoPartAction, developerVideoRedrive, developerVideoStoredBytes, failUnfinishedParts, + findRunningSlice, lostJobConfirmed, markPartChecked, markPartDispatched, markPartNotFound, + markSliceChecked, + markSliceDispatched, + markSliceNotFound, releasePart, + releaseSlice, requeueLostPart, + requeueLostSlice, resolveDeveloperVideoStatus, retryUnavailableHls, settlePart, + settleSlice, + slicedHlsAction, + slicedHlsEncoding, + slicedHlsStep, } from "./developer-video-state"; import { deleteDeveloperVideoObjects, @@ -60,6 +95,7 @@ import { headDeveloperVideoObject, internalDeveloperVideoUrl, readDeveloperVideoText, + writeDeveloperVideoText, } from "./developer-video-storage"; import { announceDeveloperVideoEvent } from "./developer-webhook-dispatch"; import { @@ -222,6 +258,86 @@ function processingUpdate( }; } +// Every result of an encode job is written here, whether it is for a whole +// output or for one slice, so this is the one place a video first becomes +// ready or becomes errored: compare the row's status with the update's. +async function commitEncodeState( + videoId: string, + encodeId: string, + change: ( + processing: DeveloperVideoProcessingState, + video: DeveloperVideoRow, + ) => { + processing: DeveloperVideoProcessingState; + renditions?: DeveloperVideoRendition[]; + } | null, + executor?: InfraDbExecutor, +): Promise { + let discarded: string[] = []; + const updated = await mutateEncode( + videoId, + encodeId, + (processing, video) => { + discarded = []; + const next = change(processing, video); + if (!next) return null; + // Slices given up while others still encode: nothing takes their + // results any more, so their workers are freed for the job that + // replaces them. + if ( + processing.sliced?.state === "encoding" && + next.processing.sliced?.state === "abandoned" + ) { + discarded = next.processing.sliced.slices.flatMap(({ job }) => + job.state === "running" && job.jobId ? [job.jobId] : [], + ); + } + // Slices that were given up and could not be replaced leave no + // adaptive stream, so the renditions they listed go with them. + const withdrawn = + next.processing.sliced?.state === "abandoned" && + (next.processing.hls.state === "failed" || + next.processing.hls.state === "unavailable"); + return { + ...processingUpdate(video, next.processing), + ...(next.renditions + ? { renditions: next.renditions } + : withdrawn + ? { renditions: null } + : {}), + }; + }, + executor, + ); + if (updated && discarded.length > 0) { + await cancelDeveloperVideoJobs(videoId, discarded); + } + return updated; +} + +function slicedHlsEnabled(): boolean { + return serverEnv().INFRA_SLICED_HLS !== "off"; +} + +// Ladders that top out above 1080p stay on the single job. Beside a steady +// stream of Cap's own jobs, their slices left the slowest of those jobs a +// little slower than one whole 4K ladder does, in repeated runs. Slices of +// 1080p and lower ladders cost Cap's jobs nothing measurable. +const SLICED_LADDER_TIERS: readonly DeveloperVideoLadderTier[] = [ + "360p", + "720p", + "1080p", +]; + +function slicedHlsTier( + video: DeveloperVideoRow, + source: { width: number; height: number }, +): boolean { + return SLICED_LADDER_TIERS.includes( + planDeveloperVideoEncode(source, video.maxResolutionTier).ladderMaxTier, + ); +} + async function findDeveloperVideo( videoId: string, ): Promise { @@ -284,7 +400,9 @@ export async function releaseDeveloperVideoClaim( ? { status: "uploading", processing: null } : null; } - if (processing.hls.state !== "pending") return null; + if (processing.hls.state !== "pending" || slicedHlsEncoding(processing)) { + return null; + } const token = newAttemptToken(); const now = Date.now(); const running = beginPart(processing, "hls", { token, now }); @@ -579,6 +697,11 @@ export async function prepareDeveloperVideoProcessing( Number.isFinite(metadata.duration) && metadata.duration > 0 ? metadata.duration : null; + const slices = + slicedHlsEnabled() && + slicedHlsTier(video, { width: metadata.width, height: metadata.height }) + ? planDeveloperVideoSlices(probedDuration) + : null; const updated = await underAccountLock(video.appId, async (tx) => { await chargeForProbedDuration( video, @@ -592,6 +715,9 @@ export async function prepareDeveloperVideoProcessing( ({ probing: _probing, ...processing }, current) => processingUpdate(current, { ...processing, + ...(slices + ? { sliced: createSlicedHls(slices, newAttemptToken()) } + : {}), probe: { duration: probedDuration, width: metadata.width, @@ -636,8 +762,11 @@ export async function readDeveloperVideoParts( const encode = await findEncode(videoId, encodeId); if (!encode) return null; const now = Date.now(); + const sliced = slicedHlsEncoding(encode.processing); const parts: DeveloperVideoPartSnapshot = { - hls: developerVideoPartAction(encode.processing.hls, now), + hls: sliced + ? slicedHlsAction(sliced, now) + : developerVideoPartAction(encode.processing.hls, now), mp4: developerVideoPartAction(encode.processing.mp4, now), }; if (!developerVideoReadyEventDue(encode.video, now)) return parts; @@ -753,16 +882,12 @@ async function settleDeveloperVideoPart( renditions?: DeveloperVideoRendition[], executor?: InfraDbExecutor, ): Promise { - const updated = await mutateEncode( + const updated = await commitEncodeState( videoId, encodeId, - (processing, video) => { + (processing) => { const settled = settlePart(processing, part, token, outcome); - if (!settled) return null; - return { - ...processingUpdate(video, settled), - ...(renditions ? { renditions } : {}), - }; + return settled && { processing: settled, renditions }; }, executor, ); @@ -774,8 +899,13 @@ export async function dispatchDeveloperVideoPart( encodeId: string, part: DeveloperVideoPart, ): Promise { + if (part === "hls") { + const sliced = await advanceSlicedHls(videoId, encodeId); + if (sliced !== null) return sliced; + } const token = newAttemptToken(); const video = await mutateEncode(videoId, encodeId, (processing) => { + if (part === "hls" && slicedHlsEncoding(processing)) return null; const started = beginPart(processing, part, { token, now: Date.now() }); return started ? { processing: started } : null; }); @@ -794,6 +924,7 @@ export async function dispatchDeveloperVideoParts( tokens = []; let next = processing; for (const part of DEVELOPER_VIDEO_PARTS) { + if (part === "hls" && slicedHlsEncoding(next)) continue; const token = newAttemptToken(); const started = beginPart(next, part, { token, now }); if (!started) continue; @@ -803,16 +934,36 @@ export async function dispatchDeveloperVideoParts( return tokens.length > 0 ? { processing: next } : null; }); const processing = video?.processing; - if (!video || !processing) return []; + const slices = async (): Promise => + (await advanceSlicedHls(videoId, encodeId)) ?? "skipped"; + if (!video || !processing) { + const encode = await findEncode(videoId, encodeId); + return encode && slicedHlsEncoding(encode.processing) + ? Promise.allSettled([slices()]) + : []; + } const source = knownSource === undefined && tokens.length > 1 && video.s3Key ? await headDeveloperVideoObject(video.s3Key) : knownSource; - return Promise.allSettled( - tokens.map(([part, token]) => - sendDeveloperVideoPart(video, processing, part, token, source), - ), + const send = ([part, token]: [DeveloperVideoPart, string]) => + sendDeveloperVideoPart(video, processing, part, token, source); + if (!slicedHlsEncoding(processing)) { + return Promise.allSettled(tokens.map(send)); + } + // The MP4 goes out before the slices: a worker making an Infra MP4 turns + // slices away, so the slices that follow land on the other workers and the + // MP4 runs on its own, as it does beside a whole-video ladder. + const mp4 = await Promise.allSettled( + tokens.filter(([part]) => part === "mp4").map(send), ); + return [ + ...mp4, + ...(await Promise.allSettled([ + ...tokens.filter(([part]) => part !== "mp4").map(send), + slices(), + ])), + ]; } async function sendDeveloperVideoPart( @@ -902,6 +1053,9 @@ export async function checkDeveloperVideoPart( encodeId: string, part: DeveloperVideoPart, ): Promise { + if (part === "hls" && (await advanceSlicedHls(videoId, encodeId)) !== null) { + return "waiting"; + } const encode = await findEncode(videoId, encodeId); const current = encode?.processing[part]; if ( @@ -955,9 +1109,9 @@ export async function failDeveloperVideoProcessing( encodeId: string, reason: string, ): Promise { - await mutateEncode(videoId, encodeId, (processing, video) => - processingUpdate(video, failUnfinishedParts(processing, reason)), - ); + await commitEncodeState(videoId, encodeId, (processing) => ({ + processing: failUnfinishedParts(processing, reason), + })); } interface DeveloperVideoProgressPayload { @@ -1062,11 +1216,37 @@ async function completedOutcome( }; } +// A slice that has just been settled frees a place for the next one, and the +// last one in is the cue to write the whole ladder. Doing that here, and not +// at the workflow's next look, keeps the workers busy between slices. export async function applyDeveloperVideoProgress( payload: DeveloperVideoProgressPayload, partParam: string, token: string | null, ): Promise<{ success: true; applied: boolean }> { + const result = await applyEncodeProgress(payload, partParam, token); + if (result.slice && result.applied && typeof payload.videoId === "string") { + try { + await advanceSlicedHls(payload.videoId, result.slice.encodeId); + } catch (error) { + console.error("[developer-video] Failed to move the slices on", { + videoId: payload.videoId, + error, + }); + } + } + return { success: true, applied: result.applied }; +} + +async function applyEncodeProgress( + payload: DeveloperVideoProgressPayload, + partParam: string, + token: string | null, +): Promise<{ + success: true; + applied: boolean; + slice?: { encodeId: string }; +}> { const ignored = { success: true as const, applied: false }; if ( (partParam !== "hls" && partParam !== "mp4") || @@ -1079,15 +1259,14 @@ export async function applyDeveloperVideoProgress( const video = await findDeveloperVideo(payload.videoId); const processing = video?.processing; - const current = processing?.[part]; - if ( - !video || - !processing || - current?.state !== "running" || - current.token !== token - ) { - return ignored; - } + const whole = processing?.[part]; + const slice = + part === "hls" && processing ? findRunningSlice(processing, token) : null; + const current = + whole?.state === "running" && whole.token === token + ? whole + : (slice?.job ?? null); + if (!video || !processing || !current) return ignored; const { encodeId } = processing; const finished = payload.phase === "complete" || @@ -1109,7 +1288,9 @@ export async function applyDeveloperVideoProgress( const jobId = payload.jobId; if (current.jobId === null && typeof jobId === "string" && jobId) { const recorded = await mutateEncode(video.id, encodeId, (latest) => { - const confirmed = markPartDispatched(latest, part, { token, jobId }); + const confirmed = slice + ? markSliceDispatched(latest, { token, jobId }) + : markPartDispatched(latest, part, { token, jobId }); return confirmed ? { processing: confirmed } : null; }); // A delete that committed after this callback read the video found @@ -1121,6 +1302,17 @@ export async function applyDeveloperVideoProgress( return ignored; } + if (slice) { + const settled = await applySliceResult( + video, + processing, + slice, + token, + payload, + ); + return { ...settled, slice: { encodeId } }; + } + if ( payload.phase === "cancelled" && payload.message === MEDIA_SERVER_SHUTDOWN_MESSAGE && @@ -1206,10 +1398,570 @@ export async function sweepSupersededDeveloperVideoOutputs( return 0; } const encodePrefix = `${developerVideoPrefix(video)}${encodeId}/`; - const kept = parts.flatMap((part) => - part.state === "complete" ? [`${encodePrefix}${part.output}/`] : [], + // A ladder made of slices keeps its playlists and the media of the attempt + // each slice settled on. The slices' own playlists and anything a second + // attempt left in the folder go. + const { sliced, hls } = processing; + const slicedOutput = + sliced?.state === "complete" && + hls.state === "complete" && + hls.output === sliced.output + ? sliced + : null; + const slicedFiles = new Set(); + // Media of slices viewers were shown before a fallback stays, with each + // slice's own playlists, which delivery metering reads to count its seconds. + const superseded = supersededSlicedHls(processing); + if (superseded) { + const { hlsPrefix, slicePrefix } = slicedHlsKeys( + video, + encodeId, + superseded.output, + ); + for (const file of slicedHlsMediaFiles({ + video, + encodeId, + ...superseded, + ended: true, + })) { + slicedFiles.add(`${hlsPrefix}${file}`); + slicedFiles.add(`${slicePrefix}${file.replace(/\.m4s$/, ".m3u8")}`); + } + } + if (slicedOutput) { + const source = { + video, + encodeId, + output: slicedOutput.output, + slices: playableSlices(slicedOutput), + hasAudio: slicedOutput.hasAudio, + ended: true, + }; + const { hlsPrefix } = slicedHlsKeys(video, encodeId, slicedOutput.output); + for (const file of [ + ...slicedHlsPlaylistFiles(source), + ...slicedHlsMediaFiles(source), + ]) { + slicedFiles.add(`${hlsPrefix}${file}`); + } + } + const kept = [ + ...parts.flatMap((part) => + part.state === "complete" && part.output !== slicedOutput?.output + ? [part.output] + : [], + ), + ...(processing.poster ? [processing.poster.output] : []), + ].map((output) => `${encodePrefix}${output}/`); + return deleteDeveloperVideoObjects( + encodePrefix, + (key) => + slicedFiles.has(key) || kept.some((prefix) => key.startsWith(prefix)), + ); +} + +function isAudioUpload(name: string): boolean { + return name.startsWith(`${HLS_AUDIO_NAME}.`); +} + +async function buildSliceJobRequest( + video: DeveloperVideoRow, + processing: DeveloperVideoProcessingState, + sliced: DeveloperVideoSlicedHls, + index: number, + token: string, +): Promise> { + const slice = sliced.slices[index]; + if (!video.s3Key || !processing.probe || !slice) { + throw new Error("Video is missing its source or dimensions"); + } + const plan = planDeveloperVideoEncode( + { width: processing.probe.width, height: processing.probe.height }, + video.maxResolutionTier, + ); + const keys = slicedHlsKeys(video, processing.encodeId, sliced.output); + const expiresIn = MEDIA_SERVER_URL_EXPIRES_SECONDS; + const source = await headDeveloperVideoObject(video.s3Key); + const inputExtension = developerVideoSourceExtension(source?.contentType); + // The first slice carries the audio for the whole video and the poster. + const first = index === 0; + const uploads = Object.fromEntries( + await Promise.all( + plan.uploads + .filter((upload) => first || !isAudioUpload(upload.name)) + .map(async (upload) => { + const stored = slicedFileName(upload.name, index, token); + const prefix = + upload.contentType === HLS_PLAYLIST_CONTENT_TYPE + ? keys.slicePrefix + : keys.hlsPrefix; + return [ + upload.name, + await developerVideoUploadUrl( + `${prefix}${stored}`, + upload.contentType, + expiresIn, + ), + ] as const; + }), + ), + ); + return { + videoId: video.id, + userId: video.appId, + videoUrl: await internalDeveloperVideoUrl(video.s3Key, expiresIn), + ...(first + ? { + thumbnailPresignedUrl: await developerVideoUploadUrl( + developerVideoKeys(video, processing.encodeId, token).poster, + "image/jpeg", + expiresIn, + ), + } + : {}), + webhookUrl: webhookUrl("hls", token), + ...(inputExtension ? { inputExtension } : {}), + maxTier: plan.ladderMaxTier, + range: { + startSeconds: slice.startSeconds, + ...(slice.durationSeconds === null + ? {} + : { durationSeconds: slice.durationSeconds }), + }, + includeAudio: first, + uploads, + infra: true, + }; +} + +async function settleDeveloperVideoSlice( + videoId: string, + encodeId: string, + token: string, + outcome: DeveloperVideoSliceOutcome, +): Promise { + const updated = await commitEncodeState(videoId, encodeId, (processing) => { + const settled = settleSlice(processing, token, outcome); + if (!settled) return null; + const sliced = settled.sliced; + return { + processing: settled, + ...(sliced?.state === "encoding" && sliced.exposed === true + ? { renditions: slicedHlsRenditions(sliced) } + : {}), + }; + }); + return updated !== null; +} + +// A slice is sent through the load balancer, so a worker that is full can be +// answered by asking again: the next request lands on another worker. +const SLICE_START_TRIES = 2; +// The first slice is what makes the video playable. It is sent on its own +// ahead of the others and asks more workers before it waits, since the MP4's +// worker turns it away and a full answer would hold it for the next pass. +const FIRST_SLICE_START_TRIES = 4; + +async function startSlice( + video: DeveloperVideoRow, + processing: DeveloperVideoProcessingState, + sliced: DeveloperVideoSlicedHls, + pick: { index: number; token: string }, +): Promise { + const { encodeId } = processing; + const release = async () => { + await mutateEncode(video.id, encodeId, (latest) => { + const released = releaseSlice(latest, pick.token); + return released + ? { processing: { ...released, busyAt: Date.now() } } + : null; + }); + return "busy" as const; + }; + let started: Awaited>; + try { + const body = await buildSliceJobRequest( + video, + processing, + sliced, + pick.index, + pick.token, + ); + started = await startMediaServerVideoJob("/video/ladder", body); + for ( + let tries = 1; + started.status === "busy" && + tries < (pick.index === 0 ? FIRST_SLICE_START_TRIES : SLICE_START_TRIES); + tries++ + ) { + started = await startMediaServerVideoJob("/video/ladder", body); + } + } catch (error) { + console.error("[developer-video] Failed to send a slice to the encoder", { + videoId: video.id, + slice: pick.index, + error, + }); + return release(); + } + + if (started.status === "started") { + const { jobId } = started; + const recorded = await mutateEncode(video.id, encodeId, (latest) => { + const dispatched = markSliceDispatched(latest, { + token: pick.token, + jobId, + }); + return dispatched ? { processing: dispatched } : null; + }); + // The slice was given up, or the video deleted, while its start was on + // the way: no result of this job would be taken. + if (!recorded) await cancelDeveloperVideoJobs(video.id, [jobId]); + return "dispatched"; + } + if (started.status === "unconfirmed") return "unconfirmed"; + if (started.status === "busy" || started.status === "unreachable") { + return release(); + } + if (started.status === "unavailable") { + await commitEncodeState(video.id, encodeId, (latest) => ({ + processing: abandonSlicedHls(latest, LADDER_UNAVAILABLE_REASON), + })); + return "unavailable"; + } + console.warn("[developer-video] Slice start rejected", { + videoId: video.id, + slice: pick.index, + detail: developerVideoFailureReason(started.message), + }); + await settleDeveloperVideoSlice(video.id, encodeId, pick.token, { + type: "failed", + reason: fileProblem(started.message, UNENCODABLE_FILE), + }); + return "failed"; +} + +async function dispatchDueSlices( + videoId: string, + encodeId: string, +): Promise { + const tokens = Array.from({ length: SLICED_HLS.maxInFlight }, () => + newAttemptToken(), + ); + let begun: { index: number; token: string }[] = []; + const video = await mutateEncode(videoId, encodeId, (processing) => { + const started = beginSlices(processing, { tokens, now: Date.now() }); + if (!started) return null; + begun = started.begun; + return { processing: started.state }; + }); + const processing = video?.processing; + const sliced = processing?.sliced; + if (!video || !processing || !sliced || begun.length === 0) return "skipped"; + + const start = (pick: { index: number; token: string }) => + startSlice(video, processing, sliced, pick); + const first = await Promise.all( + begun.filter((pick) => pick.index === 0).map(start), + ); + // No ladder route on the encoder: the video falls back to one whole job, + // so the rest of this pass is not sent. + if (first.includes("unavailable")) return "unavailable"; + const results = [ + ...first, + ...(await Promise.all(begun.filter((pick) => pick.index !== 0).map(start))), + ]; + // The workflow takes a busy answer while nothing of the video encodes as + // time spent waiting for room and restarts its deadline. Slices of this + // video that are encoding are processing time, so a slice turned away + // beside them is not reported as busy. + const encoding = + results.some((result) => result === "dispatched") || + sliced.slices.some( + (slice, index) => + slice.job.state === "running" && + !begun.some((pick) => pick.index === index), + ); + if (encoding && results.includes("busy")) return "dispatched"; + return ( + ( + ["busy", "dispatched", "unconfirmed", "unavailable", "failed"] as const + ).find((result) => results.includes(result)) ?? "skipped" + ); +} + +async function checkSlice( + videoId: string, + encodeId: string, + index: number, +): Promise { + const encode = await findEncode(videoId, encodeId); + const sliced = encode ? slicedHlsEncoding(encode.processing) : null; + const job = sliced?.slices[index]?.job; + if ( + job?.state !== "running" || + job.jobId === null || + developerVideoPartAction(job, Date.now()) !== "check" + ) { + return; + } + const { token } = job; + + const status = await readMediaServerVideoJob(job.jobId, videoId); + if (status.status === "unreachable") return; + if (status.status === "finished") { + await applyEncodeProgress(status.progress, "hls", token); + return; + } + const now = Date.now(); + let requeued = false; + await commitEncodeState(videoId, encodeId, (processing) => { + const latest = findRunningSlice(processing, token); + if (!latest) return null; + if (status.status === "missing" && !lostJobConfirmed(latest.job, now)) { + const marked = markSliceNotFound(processing, { token, now }); + return marked ? { processing: marked } : null; + } + const next = + status.status === "missing" + ? requeueLostSlice(processing, token) + : markSliceChecked(processing, { token, now }); + requeued = status.status === "missing" && next !== null; + return next ? { processing: next } : null; + }); + if (requeued) { + console.warn("[developer-video] Slice job was lost, sending it again", { + videoId, + slice: index, + }); + } +} + +// Writes the whole ladder next to the slices' media once every slice is in: +// a master playlist and one playlist per rendition that list each slice's +// file in order. From here the video's adaptive stream is stored and served +// like one made by a single job. +async function finalizeSlicedHls( + videoId: string, + encodeId: string, +): Promise { + const encode = await findEncode(videoId, encodeId); + const sliced = encode ? slicedHlsEncoding(encode.processing) : null; + if (!encode || !sliced) return false; + const slices = playableSlices(sliced); + if (slices.length !== sliced.slices.length) return false; + const source = { + video: encode.video, + encodeId, + output: sliced.output, + slices, + hasAudio: sliced.hasAudio, + ended: true, + }; + const { hlsPrefix } = slicedHlsKeys(encode.video, encodeId, sliced.output); + await Promise.all( + slicedHlsPlaylistFiles(source).map(async (file) => { + const text = await slicedHlsPlaylistText( + source, + file, + readDeveloperVideoText, + ); + if (!text) throw new Error(`Sliced ladder has no ${file}`); + await writeDeveloperVideoText( + `${hlsPrefix}${file}`, + text, + HLS_PLAYLIST_CONTENT_TYPE, + ); + }), + ); + const completed = await commitEncodeState(videoId, encodeId, (processing) => { + const next = completeSlicedHls(processing); + return next?.sliced + ? { processing: next, renditions: slicedHlsRenditions(next.sliced) } + : null; + }); + return completed !== null; +} + +// Moves a sliced encode on: sends the slices that are due, asks about the +// quiet ones, and writes the finished ladder once all are in. Null when the +// video's adaptive stream is not being made in slices. +export async function advanceSlicedHls( + videoId: string, + encodeId: string, +): Promise { + const encode = await findEncode(videoId, encodeId); + const sliced = encode ? slicedHlsEncoding(encode.processing) : null; + if (!sliced) return null; + const step = slicedHlsStep(sliced, Date.now()); + if (step.finalize) { + try { + return (await finalizeSlicedHls(videoId, encodeId)) + ? "dispatched" + : "skipped"; + } catch (error) { + console.error("[developer-video] Failed to write the sliced ladder", { + videoId, + error, + }); + return "busy"; + } + } + const [dispatched] = await Promise.all([ + step.dispatch.length > 0 + ? dispatchDueSlices(videoId, encodeId) + : Promise.resolve("skipped"), + ...step.check.map((index) => + checkSlice(videoId, encodeId, index).catch((error) => { + console.error("[developer-video] Failed to check a quiet slice job", { + videoId, + slice: index, + error, + }); + }), + ), + ]); + return dispatched; +} + +const SLICE_NOT_STORED = { + type: "failed" as const, + reason: "Encoded slice was not stored as expected", +}; + +async function completedSliceOutcome( + video: DeveloperVideoRow, + processing: DeveloperVideoProcessingState, + found: NonNullable>, + token: string, + payload: DeveloperVideoProgressPayload, +): Promise { + const ladder = parseDeveloperVideoSliceLadder(payload.ladder); + if (!ladder) { + return { type: "failed", reason: "Encoder reported no renditions" }; + } + const { sliced, index, slice } = found; + const stored = { + video, + encodeId: processing.encodeId, + output: sliced.output, + }; + const { slicePrefix } = slicedHlsKeys( + video, + processing.encodeId, + sliced.output, ); - return deleteDeveloperVideoObjects(encodePrefix, (key) => - kept.some((prefix) => key.startsWith(prefix)), + const master = await readDeveloperVideoText( + `${slicePrefix}${slicedFileName(DEVELOPER_VIDEO_MASTER_PLAYLIST, index, token)}`, ); + if (!master || !ladderMatchesMaster(ladder.renditions, master)) { + return SLICE_NOT_STORED; + } + const streams = [ + ...ladder.renditions.map((rendition) => rendition.name), + ...(index === 0 && ladder.audioBytes > 0 ? [HLS_AUDIO_NAME] : []), + ]; + const playlists = await Promise.all( + streams.map((stream) => + readStoredSlicePlaylist( + stored, + { index, token }, + stream, + readDeveloperVideoText, + ).catch(() => null), + ), + ); + const [top] = playlists; + if (!top || playlists.some((playlist) => playlist === null)) { + return SLICE_NOT_STORED; + } + const audio = streams.at(-1) === HLS_AUDIO_NAME ? playlists.at(-1) : null; + if ( + slice.durationSeconds !== null && + Math.abs(top.seconds - slice.durationSeconds) > + SLICED_HLS.lengthToleranceSeconds + ) { + return { + type: "failed", + reason: "Encoded slice does not cover its part of the video", + }; + } + const poster = + index === 0 + ? await headDeveloperVideoObject( + developerVideoKeys(video, processing.encodeId, token).poster, + ) + : null; + return { + type: "complete", + result: { + seconds: top.seconds, + renditions: ladder.renditions, + audioBytes: index === 0 ? ladder.audioBytes : 0, + ...(audio && audio.seconds > 0 + ? { + audioBitrate: Math.round((ladder.audioBytes * 8) / audio.seconds), + } + : {}), + }, + posterBytes: poster?.bytes ?? 0, + }; +} + +async function applySliceResult( + video: DeveloperVideoRow, + processing: DeveloperVideoProcessingState, + found: NonNullable>, + token: string, + payload: DeveloperVideoProgressPayload, +): Promise<{ success: true; applied: boolean }> { + const { encodeId } = processing; + if ( + payload.phase === "cancelled" && + payload.message === MEDIA_SERVER_SHUTDOWN_MESSAGE + ) { + const requeued = await commitEncodeState(video.id, encodeId, (latest) => { + const next = requeueLostSlice(latest, token); + return next ? { processing: next } : null; + }); + return { success: true, applied: requeued !== null }; + } + if (payload.phase !== "complete") { + const detail = + (typeof payload.error === "string" && payload.error) || + (typeof payload.message === "string" && payload.message) || + ""; + console.warn("[developer-video] Slice encode failed", { + videoId: video.id, + slice: found.index, + phase: payload.phase, + detail: developerVideoFailureReason(detail), + }); + const applied = await settleDeveloperVideoSlice(video.id, encodeId, token, { + type: "failed", + reason: fileProblem(detail, UNENCODABLE_FILE), + }); + return { success: true, applied }; + } + const outcome = await completedSliceOutcome( + video, + processing, + found, + token, + payload, + ); + if (outcome.type === "failed") { + console.warn("[developer-video] Slice result refused", { + videoId: video.id, + slice: found.index, + reason: outcome.reason, + }); + } + const applied = await settleDeveloperVideoSlice( + video.id, + encodeId, + token, + outcome, + ); + return { success: true, applied }; } diff --git a/apps/web/lib/developer-video-sliced-ladder.ts b/apps/web/lib/developer-video-sliced-ladder.ts new file mode 100644 index 00000000000..6dc9940ded5 --- /dev/null +++ b/apps/web/lib/developer-video-sliced-ladder.ts @@ -0,0 +1,179 @@ +import type { DeveloperVideoSliceRendition } from "@cap/database/types"; +import { z } from "zod"; +import { + DEVELOPER_VIDEO_MASTER_PLAYLIST, + developerVideoKeys, +} from "./developer-video-plan"; +import { + buildSlicedHlsMaster, + HLS_AUDIO_NAME, + joinHlsSlicePlaylists, + readHlsSlicePlaylist, +} from "./developer-video-sliced-playlist"; +import { + type FinishedSlice, + slicedFileName, + slicedHlsMasterVariants, +} from "./developer-video-slices"; + +export function slicedHlsKeys( + video: { appId: string; id: string }, + encodeId: string, + output: string, +): { hlsPrefix: string; slicePrefix: string } { + const { hlsPrefix } = developerVideoKeys(video, encodeId, output); + return { + hlsPrefix, + slicePrefix: `${hlsPrefix.slice(0, -"hls/".length)}slices/`, + }; +} + +const sliceRenditionSchema = z.object({ + name: z.string().regex(/^[A-Za-z0-9_-]+$/), + width: z.number().int().positive(), + height: z.number().int().positive(), + bandwidth: z.number().positive().nullable().optional(), + averageBandwidth: z.number().positive().nullable().optional(), + codecs: z.string().nullable().optional(), + bytes: z.number().int().positive(), +}); + +const sliceLadderSchema = z.object({ + renditions: z.array(sliceRenditionSchema).min(1), + audio: z.object({ bytes: z.number().int().positive() }).nullable().optional(), +}); + +export function parseDeveloperVideoSliceLadder( + ladder: unknown, +): { renditions: DeveloperVideoSliceRendition[]; audioBytes: number } | null { + const parsed = sliceLadderSchema.safeParse(ladder); + if (!parsed.success) return null; + const renditions = parsed.data.renditions + .map((rendition) => { + const bandwidth = rendition.bandwidth ?? rendition.averageBandwidth ?? 0; + return { + name: rendition.name, + width: rendition.width, + height: rendition.height, + bandwidth, + averageBandwidth: rendition.averageBandwidth ?? bandwidth, + bytes: rendition.bytes, + codecs: rendition.codecs ?? null, + }; + }) + .sort((a, b) => b.width * b.height - a.width * a.height); + return { renditions, audioBytes: parsed.data.audio?.bytes ?? 0 }; +} + +type ReadText = (key: string) => Promise; + +interface SlicedHlsSource { + video: { appId: string; id: string }; + encodeId: string; + output: string; + slices: FinishedSlice[]; + hasAudio: boolean; + ended: boolean; +} + +// Reads one slice's stored playlist for a stream and renames its media file +// to the name that file was stored under. +export async function readStoredSlicePlaylist( + source: Pick, + slice: { index: number; token: string }, + stream: string, + read: ReadText, +) { + const { slicePrefix } = slicedHlsKeys( + source.video, + source.encodeId, + source.output, + ); + const text = await read( + `${slicePrefix}${slicedFileName(`${stream}.m3u8`, slice.index, slice.token)}`, + ); + if (!text) throw new Error("A slice of the video has no stored playlist"); + return readHlsSlicePlaylist( + text, + `${stream}.m4s`, + slicedFileName(`${stream}.m4s`, slice.index, slice.token), + ); +} + +export const SLICED_AUDIO_PLAYLIST = `${HLS_AUDIO_NAME}.m3u8`; + +// The text of one playlist of a ladder made of slices, built from what the +// finished slices stored. Null when the ladder has no such playlist. While +// slices are still to come the audio, which is encoded whole with the first +// slice, is listed only as far as the video reaches. +export async function slicedHlsPlaylistText( + source: SlicedHlsSource, + file: string, + read: ReadText, +): Promise { + const { slices, ended } = source; + const [first] = slices; + if (!first) return null; + if (file === DEVELOPER_VIDEO_MASTER_PLAYLIST) { + return buildSlicedHlsMaster(slicedHlsMasterVariants(slices), { + hasAudio: source.hasAudio, + audioBitrate: first.result.audioBitrate, + }); + } + if (file === SLICED_AUDIO_PLAYLIST) { + if (!source.hasAudio) return null; + const audio = await readStoredSlicePlaylist( + source, + { index: 0, token: first.job.output }, + HLS_AUDIO_NAME, + read, + ); + return joinHlsSlicePlaylists([audio], { + ended, + ...(ended + ? {} + : { + untilSeconds: slices.reduce( + (total, slice) => total + slice.result.seconds, + 0, + ), + }), + }); + } + const rendition = first.result.renditions.find( + (candidate) => `${candidate.name}.m3u8` === file, + ); + if (!rendition) return null; + const playlists = await Promise.all( + slices.map((slice, index) => + readStoredSlicePlaylist( + source, + { index, token: slice.job.output }, + rendition.name, + read, + ), + ), + ); + return joinHlsSlicePlaylists(playlists, { ended }); +} + +export function slicedHlsPlaylistFiles(source: SlicedHlsSource): string[] { + const [first] = source.slices; + if (!first) return []; + return [ + DEVELOPER_VIDEO_MASTER_PLAYLIST, + ...first.result.renditions.map((rendition) => `${rendition.name}.m3u8`), + ...(source.hasAudio ? [SLICED_AUDIO_PLAYLIST] : []), + ]; +} + +export function slicedHlsMediaFiles(source: SlicedHlsSource): string[] { + return source.slices.flatMap((slice, index) => [ + ...slice.result.renditions.map((rendition) => + slicedFileName(`${rendition.name}.m4s`, index, slice.job.output), + ), + ...(index === 0 && source.hasAudio + ? [slicedFileName(`${HLS_AUDIO_NAME}.m4s`, index, slice.job.output)] + : []), + ]); +} diff --git a/apps/web/lib/developer-video-sliced-playlist.ts b/apps/web/lib/developer-video-sliced-playlist.ts new file mode 100644 index 00000000000..d959f0d508c --- /dev/null +++ b/apps/web/lib/developer-video-sliced-playlist.ts @@ -0,0 +1,203 @@ +// The slice planner and the playlist joiner below mirror planHlsLadderSlices +// and stitchHlsLadderSlices in apps/media-server/src/lib/hls-ladder.ts. The +// media server image is built from its own folder and cannot import a shared +// package, so the code is kept here as a copy and a test compares the two. + +export const HLS_SEGMENT_SECONDS = 4; +export const HLS_AUDIO_BITRATE = 128_000; +// The media server copies stereo AAC-LC source audio of up to this bitrate +// instead of encoding it at HLS_AUDIO_BITRATE, so a variant's peak allows for it. +export const HLS_AUDIO_PEAK_BITRATE = 192_000; +export const HLS_AUDIO_NAME = "audio"; + +const AUDIO_GROUP = "aud"; +const AUDIO_CODEC = "mp4a.40.2"; + +export interface HlsSliceRange { + startSeconds: number; + durationSeconds?: number; +} + +export function planHlsLadderSlices( + durationSeconds: number, + workers: number, + minSliceSeconds = 20, +): HlsSliceRange[] { + if (!(durationSeconds > 0)) return [{ startSeconds: 0 }]; + const count = Math.max( + 1, + Math.min( + Math.floor(workers), + Math.floor(durationSeconds / minSliceSeconds), + ), + ); + const sliceSeconds = + Math.ceil(durationSeconds / count / HLS_SEGMENT_SECONDS) * + HLS_SEGMENT_SECONDS; + const slices: HlsSliceRange[] = []; + for (let index = 0; index < count; index++) { + const startSeconds = index * sliceSeconds; + if (startSeconds >= durationSeconds) break; + slices.push( + index === count - 1 + ? { startSeconds } + : { startSeconds, durationSeconds: sliceSeconds }, + ); + } + const last = slices.at(-1); + if (last) delete last.durationSeconds; + return slices; +} + +export interface HlsPlaylistSegment { + seconds: number; + lines: string[]; +} + +export interface HlsSlicePlaylist { + map: string; + targetDuration: number; + seconds: number; + segments: HlsPlaylistSegment[]; +} + +// Reads one slice's playlist for a rendition: a single media file addressed by +// byte range. Anything else in it means the slice is not what was asked for. +export function readHlsSlicePlaylist( + playlist: string, + source: string, + name: string, +): HlsSlicePlaylist { + let map: string | null = null; + let targetDuration = 0; + let seconds = 0; + const segments: HlsPlaylistSegment[] = []; + let open: HlsPlaylistSegment | null = null; + for (const raw of playlist.split(/\r?\n/)) { + const line = raw.trim(); + if (line.startsWith("#EXT-X-TARGETDURATION:")) { + targetDuration = Number(line.slice(line.indexOf(":") + 1)); + } else if (line.startsWith("#EXT-X-MAP:")) { + if (!line.includes(`URI="${source}"`)) { + throw new Error("HLS ladder slice has an unexpected init segment"); + } + map = line.replace(`URI="${source}"`, `URI="${name}"`); + } else if (line.startsWith("#EXTINF:")) { + const duration = Number(line.slice("#EXTINF:".length).split(",")[0]); + if (!(duration > 0)) { + throw new Error("HLS ladder slice has a segment with no length"); + } + open = { seconds: duration, lines: [line] }; + } else if (line.startsWith("#EXT-X-BYTERANGE:")) { + open?.lines.push(line); + } else if (line === source) { + if (!open) + throw new Error("HLS ladder slice has a segment with no length"); + open.lines.push(name); + segments.push(open); + seconds += open.seconds; + open = null; + } else if (line.length > 0 && !line.startsWith("#")) { + throw new Error("HLS ladder slice references an unexpected file"); + } + } + if (!map) throw new Error("HLS ladder slice has no init segment"); + if (segments.length === 0) + throw new Error("HLS ladder slice has no segments"); + return { map, targetDuration, seconds, segments }; +} + +// Slices are joined by playlist only: the playlist lists every slice's file in +// order, each behind its own init segment, so no media is copied after +// encoding. It is an event playlist from the first slice on: lines are only +// ever added, and the end marker goes on once every slice is in. A player +// that reaches the last line before then waits for more. Without a start +// offset players join an unfinished playlist near its end, as for a live +// stream. +export function joinHlsSlicePlaylists( + slices: HlsSlicePlaylist[], + options: { ended: boolean; untilSeconds?: number }, +): string { + const [first] = slices; + if (!first) throw new Error("HLS ladder has no slices"); + const body: string[] = []; + let targetDuration = 0; + let position = 0; + for (const slice of slices) { + const lines: string[] = []; + for (const segment of slice.segments) { + if ( + options.untilSeconds !== undefined && + position >= options.untilSeconds + ) { + break; + } + lines.push(...segment.lines); + position += segment.seconds; + } + if (lines.length > 0) body.push(slice.map, ...lines); + targetDuration = Math.max(targetDuration, slice.targetDuration); + } + return `${[ + "#EXTM3U", + "#EXT-X-VERSION:7", + `#EXT-X-TARGETDURATION:${targetDuration}`, + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:EVENT", + "#EXT-X-INDEPENDENT-SEGMENTS", + "#EXT-X-START:TIME-OFFSET=0", + ...body, + ...(options.ended ? ["#EXT-X-ENDLIST"] : []), + ].join("\n")}\n`; +} + +export interface HlsMasterVariant { + name: string; + width: number; + height: number; + bandwidth: number; + averageBandwidth: number; + codecs: string | null; +} + +export function hlsVariantCodecs( + codecs: string | null, + hasAudio: boolean, +): string | null { + const list = [codecs, hasAudio ? AUDIO_CODEC : null].filter( + (codec): codec is string => Boolean(codec), + ); + return list.length > 0 ? list.join(",") : null; +} + +export function buildSlicedHlsMaster( + variants: HlsMasterVariant[], + options: { hasAudio: boolean; audioBitrate?: number }, +): string { + if (variants.length === 0) throw new Error("HLS ladder has no renditions"); + const audioBandwidth = options.hasAudio + ? (options.audioBitrate ?? HLS_AUDIO_BITRATE) + : 0; + const audioPeak = options.hasAudio ? HLS_AUDIO_PEAK_BITRATE : 0; + const master = ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-INDEPENDENT-SEGMENTS"]; + if (options.hasAudio) { + master.push( + `#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="${AUDIO_GROUP}",NAME="${HLS_AUDIO_NAME}",DEFAULT=YES,AUTOSELECT=YES,CHANNELS="2",URI="${HLS_AUDIO_NAME}.m3u8"`, + ); + } + for (const variant of variants) { + const codecs = hlsVariantCodecs(variant.codecs, options.hasAudio); + const attributes = [ + `BANDWIDTH=${Math.round(variant.bandwidth) + audioPeak}`, + `AVERAGE-BANDWIDTH=${Math.round(variant.averageBandwidth) + audioBandwidth}`, + `RESOLUTION=${variant.width}x${variant.height}`, + ...(codecs ? [`CODECS="${codecs}"`] : []), + ...(options.hasAudio ? [`AUDIO="${AUDIO_GROUP}"`] : []), + ]; + master.push( + `#EXT-X-STREAM-INF:${attributes.join(",")}`, + `${variant.name}.m3u8`, + ); + } + return `${master.join("\n")}\n`; +} diff --git a/apps/web/lib/developer-video-slices.ts b/apps/web/lib/developer-video-slices.ts new file mode 100644 index 00000000000..ce1648f703c --- /dev/null +++ b/apps/web/lib/developer-video-slices.ts @@ -0,0 +1,342 @@ +import type { + DeveloperVideoProcessingState, + DeveloperVideoRendition, + DeveloperVideoSlice, + DeveloperVideoSlicedHls, + DeveloperVideoSliceRendition, + DeveloperVideoSliceResult, +} from "@cap/database/types"; +import { + HLS_AUDIO_BITRATE, + HLS_SEGMENT_SECONDS, + type HlsMasterVariant, + type HlsSliceRange, + hlsVariantCodecs, + planHlsLadderSlices, +} from "./developer-video-sliced-playlist"; + +// This file is imported by the processing state, which the workflow bundles, +// so it must not import anything that needs a Node module. + +// A ladder job spends about five seconds before its first frame (download, +// probe, encoder start) and then encodes at roughly 6 to 15 times real time. +// Slices share the workers with Cap's own recordings, and every slice pays +// that fixed cost again. Measured beside a steady stream of Cap jobs, a 5 +// minute 4K video in slices cost those jobs more in total than one whole job +// does, while from 21 minutes up the slices cost them the same or less. So +// only videos of 20 minutes or more are sliced, and shorter ones keep the +// single job and its MP4 for playback. +export const SLICED_HLS = { + minDurationSeconds: 1200, + // Short enough to encode in two or three seconds. + firstSliceSeconds: 20, + // A viewer who presses play the moment the first slice is in must not catch + // up with the encode. At six times real time a slice up to six times as + // long as everything before it is finished before that viewer reaches it, + // so slices grow by this factor until the remaining thirds fit under it. + rampFactor: 6, + tailWorkers: 3, + tailMinSliceSeconds: 60, + // Thirds of a long video are long slices, and one long slice still + // encoding holds its worker while the others sit idle. Past this size the + // rest of the video goes out in even slices, `rounds` per worker, so the + // workers finish together. Every slice downloads the whole source, so the + // slices stay this long at least. + steadyMinSliceSeconds: 300, + rounds: 4, + // One upload may not hold more encoder slots than this at a time. + maxInFlight: 3, + // The seconds of video a slice holds may differ from its range by a frame + // or two at either end. More than this and its neighbours would not meet. + lengthToleranceSeconds: 0.5, +} as const; + +function wholeSegments(seconds: number): number { + return Math.floor(seconds / HLS_SEGMENT_SECONDS) * HLS_SEGMENT_SECONDS; +} + +// Null means the video keeps the single ladder job it has always had. +export function planDeveloperVideoSlices( + durationSeconds: number | null, +): HlsSliceRange[] | null { + if ( + durationSeconds === null || + !Number.isFinite(durationSeconds) || + durationSeconds < SLICED_HLS.minDurationSeconds + ) { + return null; + } + const thirds = planInThirds(durationSeconds); + const longest = Math.max( + ...thirds.map( + (range) => range.durationSeconds ?? durationSeconds - range.startSeconds, + ), + ); + return longest <= SLICED_HLS.steadyMinSliceSeconds * 1.5 + ? thirds + : planInRounds(durationSeconds); +} + +function planInRounds(durationSeconds: number): HlsSliceRange[] { + const first = SLICED_HLS.firstSliceSeconds; + const ranges: HlsSliceRange[] = [{ startSeconds: 0, durationSeconds: first }]; + const ramp = wholeSegments(first * SLICED_HLS.rampFactor); + const steady = Math.max( + SLICED_HLS.steadyMinSliceSeconds, + Math.ceil( + (durationSeconds - first - ramp) / + (SLICED_HLS.tailWorkers * SLICED_HLS.rounds) / + HLS_SEGMENT_SECONDS, + ) * HLS_SEGMENT_SECONDS, + ); + let covered: number = first; + while (durationSeconds - covered > steady * 1.5) { + const size = Math.min( + steady, + wholeSegments(covered * SLICED_HLS.rampFactor), + ); + ranges.push({ startSeconds: covered, durationSeconds: size }); + covered += size; + } + ranges.push({ startSeconds: covered }); + return ranges; +} + +function planInThirds(durationSeconds: number): HlsSliceRange[] { + const ranges: HlsSliceRange[] = [ + { startSeconds: 0, durationSeconds: SLICED_HLS.firstSliceSeconds }, + ]; + let covered: number = SLICED_HLS.firstSliceSeconds; + for (;;) { + const ramp = wholeSegments(covered * SLICED_HLS.rampFactor); + if ((durationSeconds - covered) / SLICED_HLS.tailWorkers <= ramp) break; + ranges.push({ startSeconds: covered, durationSeconds: ramp }); + covered += ramp; + } + const tail = planHlsLadderSlices( + durationSeconds - covered, + SLICED_HLS.tailWorkers, + SLICED_HLS.tailMinSliceSeconds, + ); + for (const range of tail) { + ranges.push({ + startSeconds: covered + range.startSeconds, + ...(range.durationSeconds === undefined + ? {} + : { durationSeconds: range.durationSeconds }), + }); + } + return ranges; +} + +export function createSlicedHls( + ranges: HlsSliceRange[], + output: string, +): DeveloperVideoSlicedHls { + return { + state: "encoding", + output, + exposed: null, + hasAudio: false, + slices: ranges.map((range) => ({ + startSeconds: range.startSeconds, + durationSeconds: range.durationSeconds ?? null, + job: { state: "pending" }, + attempts: 0, + lostJobs: 0, + })), + }; +} + +// Every slice uploads into one folder. The slice and the attempt are part of +// each file name, so a second attempt at a slice never writes over the first, +// and the finished ladder is one folder of playlists and media like any other. +export function slicedFileName( + file: string, + slice: number, + token: string, +): string { + const dot = file.lastIndexOf("."); + return `${file.slice(0, dot)}-${slice}-${token}${file.slice(dot)}`; +} + +const SLICED_MEDIA_FILE = /^[A-Za-z0-9]+-\d+-[A-Za-z0-9_-]+\.m4s$/; + +export function isSlicedMediaFile(file: string): boolean { + return SLICED_MEDIA_FILE.test(file); +} + +// Slices viewers were shown before the video fell back to one whole job. +// Players that loaded their playlists then keep asking for these files. +export function supersededSlicedHls( + state: DeveloperVideoProcessingState, +): { output: string; slices: FinishedSlice[]; hasAudio: boolean } | null { + const { sliced } = state; + if (!sliced || sliced.state !== "abandoned" || sliced.exposed !== true) { + return null; + } + const slices = playableSlices(sliced); + return slices.length > 0 + ? { output: sliced.output, slices, hasAudio: sliced.hasAudio } + : null; +} + +export type FinishedSlice = DeveloperVideoSlice & { + job: { state: "complete"; output: string }; + result: DeveloperVideoSliceResult; +}; + +export function sliceIsFinished( + slice: DeveloperVideoSlice | undefined, +): slice is FinishedSlice { + return slice?.job.state === "complete" && slice.result !== undefined; +} + +export function finishedSlices( + sliced: DeveloperVideoSlicedHls, +): { index: number; slice: FinishedSlice }[] { + return sliced.slices.flatMap((slice, index) => + sliceIsFinished(slice) ? [{ index, slice }] : [], + ); +} + +// The slices a viewer can be given: finished ones from the start of the video +// with no hole between them. +export function playableSlices( + sliced: DeveloperVideoSlicedHls, +): FinishedSlice[] { + const playable: FinishedSlice[] = []; + for (const slice of sliced.slices) { + if (!sliceIsFinished(slice)) break; + playable.push(slice); + } + return playable; +} + +export function slicesMatch( + expected: DeveloperVideoSliceRendition[], + found: DeveloperVideoSliceRendition[], +): boolean { + return ( + expected.length === found.length && + expected.every((rendition, index) => { + const other = found[index]; + return ( + other !== undefined && + other.name === rendition.name && + other.width === rendition.width && + other.height === rendition.height + ); + }) + ); +} + +export interface SlicedHlsPlayback { + output: string; + slices: FinishedSlice[]; + hasAudio: boolean; + // Every slice is in, so the playlists carry their end marker. + ended: boolean; +} + +// Adaptive streaming served from slices while the ladder is not whole yet: +// either more slices are on their way, or a slice could not be made and the +// video is being encoded as one job, in which case the finished slices keep +// playing until that job has an answer. +export function slicedHlsPlayback( + state: DeveloperVideoProcessingState, +): SlicedHlsPlayback | null { + const { sliced, hls } = state; + if (!sliced || sliced.exposed !== true || hls.state === "complete") { + return null; + } + const waitingForWholeJob = + sliced.state === "abandoned" && + (hls.state === "pending" || hls.state === "running"); + if (sliced.state !== "encoding" && !waitingForWholeJob) return null; + const slices = playableSlices(sliced); + if (slices.length === 0) return null; + return { + output: sliced.output, + slices, + hasAudio: sliced.hasAudio, + ended: + sliced.state === "encoding" && slices.length === sliced.slices.length, + }; +} + +interface MergedRendition extends HlsMasterVariant { + bytes: number; +} + +function mergeSliceRenditions(slices: FinishedSlice[]): MergedRendition[] { + const [first] = slices; + if (!first) return []; + return first.result.renditions.map((rendition, index) => { + let bytes = 0; + let seconds = 0; + let weighted = 0; + let peak = 0; + for (const slice of slices) { + const part = slice.result.renditions[index]; + if (!part) continue; + bytes += part.bytes; + seconds += slice.result.seconds; + weighted += part.averageBandwidth * slice.result.seconds; + peak = Math.max(peak, part.bandwidth); + } + return { + name: rendition.name, + width: rendition.width, + height: rendition.height, + bandwidth: peak, + averageBandwidth: seconds > 0 ? Math.round(weighted / seconds) : peak, + codecs: rendition.codecs, + bytes, + }; + }); +} + +export function slicedHlsMasterVariants( + slices: FinishedSlice[], +): HlsMasterVariant[] { + return mergeSliceRenditions(slices).map(({ bytes: _bytes, ...variant }) => ({ + ...variant, + })); +} + +// The same figures a single ladder job reports for its renditions: bitrate is +// the average including audio, bytes are the rendition's media, and the codec +// list names the audio that plays with it. +export function slicedHlsRenditions( + sliced: DeveloperVideoSlicedHls, +): DeveloperVideoRendition[] { + const audio = sliced.hasAudio + ? (sliced.slices[0]?.result?.audioBitrate ?? HLS_AUDIO_BITRATE) + : 0; + return mergeSliceRenditions( + finishedSlices(sliced).map(({ slice }) => slice), + ).map((rendition) => ({ + name: rendition.name, + width: rendition.width, + height: rendition.height, + bitrate: rendition.averageBandwidth + audio, + bytes: rendition.bytes, + codecs: hlsVariantCodecs(rendition.codecs, sliced.hasAudio), + })); +} + +// Media stored by the finished slices, each file counted once: the audio for +// the whole video belongs to the first slice only. +export function slicedHlsBytes(sliced: DeveloperVideoSlicedHls): number { + return finishedSlices(sliced).reduce( + (total, { slice }) => + total + + slice.result.audioBytes + + slice.result.renditions.reduce( + (sum, rendition) => sum + rendition.bytes, + 0, + ), + 0, + ); +} diff --git a/apps/web/lib/developer-video-state.ts b/apps/web/lib/developer-video-state.ts index 9c348e03642..41ba9550e0e 100644 --- a/apps/web/lib/developer-video-state.ts +++ b/apps/web/lib/developer-video-state.ts @@ -2,8 +2,19 @@ import type { DeveloperVideoPart, DeveloperVideoPartState, DeveloperVideoProcessingState, + DeveloperVideoSlice, + DeveloperVideoSlicedHls, + DeveloperVideoSliceResult, DeveloperVideoStatus, } from "@cap/database/types"; +import { + finishedSlices, + SLICED_HLS, + slicedHlsBytes, + slicedHlsPlayback, + sliceIsFinished, + slicesMatch, +} from "./developer-video-slices"; // The MP4 comes first: it is the quicker output, so it is sent to the encoder first. export const DEVELOPER_VIDEO_PARTS = ["mp4", "hls"] as const; @@ -202,6 +213,15 @@ export function settlePart( outcome: DeveloperVideoPartOutcome, ): DeveloperVideoProcessingState | null { if (!runningWithToken(state[part], token)) return null; + return reconcileSlicedHls(settleRunningPart(state, part, token, outcome)); +} + +function settleRunningPart( + state: DeveloperVideoProcessingState, + part: DeveloperVideoPart, + token: string, + outcome: DeveloperVideoPartOutcome, +): DeveloperVideoProcessingState { if (outcome.type === "complete") { return { ...state, @@ -260,7 +280,19 @@ export function failUnfinishedParts( }; } } - return next; + const sliced = next.sliced; + return reconcileSlicedHls( + sliced?.state === "encoding" + ? { + ...next, + sliced: { + ...sliced, + state: "abandoned", + reason: developerVideoFailureReason(reason), + }, + } + : next, + ); } function partWasLost( @@ -296,8 +328,23 @@ export function retryUnavailableHls( next = { ...next, [part]: { state: "pending" } }; } } + const sliced = slicedHlsEncoding(next); + if (sliced) { + const slices = sliced.slices.map((slice) => + slice.job.state === "running" && + now - slice.job.startedAt >= STALE_PROCESSING_MS + ? { ...slice, job: { state: "pending" as const } } + : slice, + ); + if (slices.some((slice, index) => slice !== sliced.slices[index])) { + next = { ...next, sliced: { ...sliced, slices } }; + } + } const states = DEVELOPER_VIDEO_PARTS.map((part) => state[part].state); - const waiting = states.includes("complete") && states.includes("pending"); + const waiting = + states.includes("pending") && + (states.includes("complete") || + (sliced !== null && finishedSlices(sliced).length > 0)); if (next === state && !waiting) return null; // Stamping the retry is what stops the next read from claiming it again // before the workflow has picked the part up. @@ -308,16 +355,21 @@ export function resolveDeveloperVideoStatus( state: DeveloperVideoProcessingState, previous: DeveloperVideoStatus, ): { status: DeveloperVideoStatus; errorReason: string | null } { - if (previous === "ready") return { status: "ready", errorReason: null }; const parts = DEVELOPER_VIDEO_PARTS.map((part) => state[part]); - if (parts.some((part) => part.state === "complete")) { + const playable = + parts.some((part) => part.state === "complete") || + slicedHlsPlayback(state) !== null; + const unfinished = parts.some( + (part) => part.state === "pending" || part.state === "running", + ); + // A ready video stays ready. The one exception is a video that became ready + // on its first slices alone: if the rest could not be made, by slices or as + // one job, and there is no MP4 either, what it has is not the whole video. + if (previous === "ready" && (!state.sliced || playable || unfinished)) { return { status: "ready", errorReason: null }; } - if ( - parts.some((part) => part.state === "pending" || part.state === "running") - ) { - return { status: "processing", errorReason: null }; - } + if (playable) return { status: "ready", errorReason: null }; + if (unfinished) return { status: "processing", errorReason: null }; const failure = [state.mp4, state.hls].find((part) => part.state === "failed") ?? [state.mp4, state.hls].find((part) => part.state === "unavailable"); @@ -378,9 +430,375 @@ export function developerVideoRedrive( video.updatedAt.getTime(), video.processing.startedAt, ); + const sliced = slicedHlsEncoding(video.processing); + if (sliced) { + return partWasLost(mp4, now, lastStarted) || + slicesWereLost(sliced, now, lastStarted) + ? "retry-hls" + : null; + } return [mp4, hls].some((part) => partWasLost(part, now, lastStarted)) ? "retry-hls" : null; } return null; } + +// Slices are encoded while the hls part waits. Once that part is complete, +// the single job owns adaptive streaming. +export function slicedHlsEncoding( + state: DeveloperVideoProcessingState, +): DeveloperVideoSlicedHls | null { + return state.sliced?.state === "encoding" && state.hls.state !== "complete" + ? state.sliced + : null; +} + +// A finished single job replaces slices that are still encoding. Slices that +// were given up stay stored, and counted, only while the single job that +// replaces them is still to come. +function reconcileSlicedHls( + state: DeveloperVideoProcessingState, +): DeveloperVideoProcessingState { + const { sliced, hls } = state; + if (!sliced) return state; + if (sliced.state === "encoding" && hls.state === "complete") { + return { + ...state, + sliced: { + ...sliced, + state: "abandoned", + reason: "Replaced by a single encode", + }, + }; + } + if ( + sliced.state === "abandoned" && + (hls.state === "failed" || hls.state === "unavailable") && + state.bytes.hls !== 0 + ) { + return { ...state, bytes: { ...state.bytes, hls: 0 } }; + } + return state; +} + +function slicesWereLost( + sliced: DeveloperVideoSlicedHls, + now: number, + lastStarted: number, +): boolean { + const running = sliced.slices.flatMap((slice) => + slice.job.state === "running" ? [slice.job] : [], + ); + if (running.some((job) => now - job.startedAt >= STALE_PROCESSING_MS)) { + return true; + } + return running.length === 0 && now - lastStarted >= NEVER_STARTED_GRACE_MS; +} + +export interface SlicedHlsStep { + dispatch: number[]; + check: number[]; + finalize: boolean; +} + +// What the slices need next: the earliest ones not yet sent, up to the number +// one video may have encoding at a time, the quiet jobs to ask about, and the +// final write once every slice is in. +export function slicedHlsStep( + sliced: DeveloperVideoSlicedHls, + now: number, +): SlicedHlsStep { + const actions = sliced.slices.map((slice) => + developerVideoPartAction(slice.job, now), + ); + const inFlight = sliced.slices.filter( + (slice, index) => + slice.job.state === "running" && actions[index] !== "dispatch", + ).length; + const due = actions.flatMap((action, index) => + action === "dispatch" ? [index] : [], + ); + return { + dispatch: due.slice(0, Math.max(0, SLICED_HLS.maxInFlight - inFlight)), + check: actions.flatMap((action, index) => + action === "check" ? [index] : [], + ), + finalize: sliced.slices.every((slice) => sliceIsFinished(slice)), + }; +} + +export function slicedHlsAction( + sliced: DeveloperVideoSlicedHls, + now: number, +): DeveloperVideoPartAction { + const step = slicedHlsStep(sliced, now); + if (step.finalize || step.dispatch.length > 0) return "dispatch"; + return step.check.length > 0 ? "check" : "wait"; +} + +function withSlice( + state: DeveloperVideoProcessingState, + sliced: DeveloperVideoSlicedHls, + index: number, + slice: DeveloperVideoSlice, +): DeveloperVideoProcessingState { + return { + ...state, + sliced: { + ...sliced, + slices: sliced.slices.map((current, at) => + at === index ? slice : current, + ), + }, + }; +} + +interface RunningSlice { + sliced: DeveloperVideoSlicedHls; + index: number; + slice: DeveloperVideoSlice; + job: Extract; +} + +export function findRunningSlice( + state: DeveloperVideoProcessingState, + token: string, +): RunningSlice | null { + const sliced = slicedHlsEncoding(state); + if (!sliced) return null; + for (const [index, slice] of sliced.slices.entries()) { + if (runningWithToken(slice.job, token)) { + return { sliced, index, slice, job: slice.job }; + } + } + return null; +} + +export function beginSlices( + state: DeveloperVideoProcessingState, + input: { tokens: string[]; now: number }, +): { + state: DeveloperVideoProcessingState; + begun: { index: number; token: string }[]; +} | null { + const sliced = slicedHlsEncoding(state); + if (!sliced) return null; + const { dispatch } = slicedHlsStep(sliced, input.now); + const begun = dispatch.flatMap((index, at) => { + const token = input.tokens[at]; + return token ? [{ index, token }] : []; + }); + if (begun.length === 0) return null; + const slices = sliced.slices.map((slice, index) => { + const pick = begun.find((entry) => entry.index === index); + if (!pick) return slice; + return { + ...slice, + job: { + state: "running" as const, + token: pick.token, + jobId: null, + startedAt: input.now, + }, + attempts: + slice.job.state === "pending" ? slice.attempts + 1 : slice.attempts, + }; + }); + return { state: { ...state, sliced: { ...sliced, slices } }, begun }; +} + +export function markSliceDispatched( + state: DeveloperVideoProcessingState, + input: { token: string; jobId: string }, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, input.token); + if (!found) return null; + return withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { ...found.job, jobId: input.jobId }, + }); +} + +export function markSliceChecked( + state: DeveloperVideoProcessingState, + input: { token: string; now: number }, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, input.token); + if (!found) return null; + const { notFoundAt: _notFoundAt, ...alive } = found.job; + return withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { ...alive, checkedAt: input.now }, + }); +} + +export function markSliceNotFound( + state: DeveloperVideoProcessingState, + input: { token: string; now: number }, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, input.token); + if (!found || found.job.notFoundAt !== undefined) return null; + return withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { ...found.job, notFoundAt: input.now }, + }); +} + +export function releaseSlice( + state: DeveloperVideoProcessingState, + token: string, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, token); + if (!found) return null; + return withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { state: "pending" }, + attempts: Math.max(0, found.slice.attempts - 1), + }); +} + +// Gives up on slices and leaves adaptive streaming to one job for the whole +// video, which gets a single attempt. The finished slices stay where they +// are, so a video that is already playing from them keeps playing until that +// job has an answer. +export function abandonSlicedHls( + state: DeveloperVideoProcessingState, + reason: string, +): DeveloperVideoProcessingState { + const { sliced } = state; + if (!sliced || sliced.state !== "encoding") return state; + return reconcileSlicedHls({ + ...state, + sliced: { + ...sliced, + state: "abandoned", + reason: developerVideoFailureReason(reason), + }, + attempts: + state.hls.state === "pending" + ? { ...state.attempts, hls: DEVELOPER_VIDEO_MAX_PART_ATTEMPTS - 1 } + : state.attempts, + }); +} + +const LOST_SLICE = "A slice of the video kept losing its encode job"; + +// A slice whose job ended with its server is sent again without costing an +// attempt. One that keeps vanishing is treated as a slice that cannot be made. +export function requeueLostSlice( + state: DeveloperVideoProcessingState, + token: string, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, token); + if (!found) return null; + if (found.slice.lostJobs >= DEVELOPER_VIDEO_MAX_LOST_JOBS) { + return abandonSlicedHls( + withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { state: "failed", reason: LOST_SLICE }, + }), + LOST_SLICE, + ); + } + return withSlice(state, found.sliced, found.index, { + ...found.slice, + job: { state: "pending" }, + attempts: Math.max(0, found.slice.attempts - 1), + lostJobs: found.slice.lostJobs + 1, + }); +} + +export type DeveloperVideoSliceOutcome = + | { + type: "complete"; + result: DeveloperVideoSliceResult; + posterBytes?: number; + } + | { type: "failed"; reason: string }; + +const MISMATCHED_SLICES = "Slices were encoded with different renditions"; + +export function settleSlice( + state: DeveloperVideoProcessingState, + token: string, + outcome: DeveloperVideoSliceOutcome, +): DeveloperVideoProcessingState | null { + const found = findRunningSlice(state, token); + if (!found) return null; + const { sliced, index, slice } = found; + const other = finishedSlices(sliced)[0]?.slice; + const failure = + outcome.type === "failed" + ? outcome.reason + : other && + !slicesMatch(other.result.renditions, outcome.result.renditions) + ? MISMATCHED_SLICES + : null; + if (outcome.type === "failed" || failure !== null) { + const reason = developerVideoFailureReason(failure ?? MISMATCHED_SLICES); + if (slice.attempts < DEVELOPER_VIDEO_MAX_PART_ATTEMPTS) { + return withSlice(state, sliced, index, { + ...slice, + job: { state: "pending" }, + }); + } + return abandonSlicedHls( + withSlice(state, sliced, index, { + ...slice, + job: { state: "failed", reason }, + }), + reason, + ); + } + + const slices = sliced.slices.map((current, at) => + at === index + ? { + ...slice, + job: { state: "complete" as const, output: token }, + result: outcome.result, + } + : current, + ); + const first = index === 0; + const next: DeveloperVideoSlicedHls = { + ...sliced, + slices, + hasAudio: first ? outcome.result.audioBytes > 0 : sliced.hasAudio, + // Decided once, when the start of the video is first in: a video that + // already plays from its MP4 gets its adaptive stream when it is whole. + exposed: + sliced.exposed === null && sliceIsFinished(slices[0]) + ? state.mp4.state !== "complete" + : sliced.exposed, + }; + const posterBytes = first ? (outcome.posterBytes ?? 0) : 0; + return { + ...state, + sliced: next, + poster: state.poster ?? (posterBytes > 0 ? { output: token } : null), + bytes: { + ...state.bytes, + hls: slicedHlsBytes(next), + poster: state.bytes.poster + posterBytes, + }, + }; +} + +// Every slice is in and the whole ladder has been written to the slices' +// folder, so the hls part is complete and points at it like any other ladder. +export function completeSlicedHls( + state: DeveloperVideoProcessingState, +): DeveloperVideoProcessingState | null { + const sliced = slicedHlsEncoding(state); + if (!sliced || !sliced.slices.every((slice) => sliceIsFinished(slice))) { + return null; + } + return { + ...state, + hls: { state: "complete", output: sliced.output }, + sliced: { ...sliced, state: "complete" }, + bytes: { ...state.bytes, hls: slicedHlsBytes(sliced) }, + }; +} diff --git a/apps/web/lib/developer-video-storage.ts b/apps/web/lib/developer-video-storage.ts index 193f2cf9bf7..55c208676e7 100644 --- a/apps/web/lib/developer-video-storage.ts +++ b/apps/web/lib/developer-video-storage.ts @@ -112,6 +112,18 @@ export function readDeveloperVideoText(key: string): Promise { ); } +export async function writeDeveloperVideoText( + key: string, + text: string, + contentType: string, +): Promise { + await StorageRuntime.runPromise( + bucket.pipe( + Effect.flatMap((access) => access.putObject(key, text, { contentType })), + ), + ); +} + export async function headDeveloperVideoObject( key: string, ): Promise<{ bytes: number; contentType: string | null } | null> { diff --git a/apps/web/lib/infra-delivery-log.ts b/apps/web/lib/infra-delivery-log.ts index 146b234913b..9ced1c4ec47 100644 --- a/apps/web/lib/infra-delivery-log.ts +++ b/apps/web/lib/infra-delivery-log.ts @@ -41,8 +41,11 @@ export type DeliveryPart = [ const ID = "[A-Za-z0-9_-]{1,64}"; const DELIVERY_PATH = new RegExp( - `^/(developer/(${ID})/(${ID})/${ID}/${ID}/(?:hls/(360p|720p|1080p|1440p|2160p|audio)\\.m4s|(video)\\.mp4))$`, + `^/(developer/(${ID})/(${ID})/${ID}/${ID}/(?:hls/(360p|720p|1080p|1440p|2160p|audio)(?:-\\d+-${ID})?\\.m4s|(video)\\.mp4))$`, ); +// A ladder encoded in slices stores each slice's media as +// --.m4s next to the whole ladder's playlists. +const SLICE_MEDIA_FILE = /^([A-Za-z0-9]+)-\d+-[A-Za-z0-9_-]+\.m4s$/; const DAY = /^\d{4}-\d{2}-\d{2}$/; const DELIVERED_STATUSES = new Set(["200", "206"]); const TIERS = new Set(DELIVERY_RENDITION_TIERS); @@ -123,9 +126,11 @@ export function parseHlsByteRanges( mediaName: string, ): HlsSegment[] | null { const segments: HlsSegment[] = []; + // A ladder joined from slices lists one media file per slice. A range with + // no offset continues from the previous segment of the same file. + const nextOffsets = new Map(); let duration: number | null = null; - let range: { start: number; length: number } | null = null; - let nextOffset = 0; + let range: { start: number | null; length: number } | null = null; for (const raw of playlist.split(/\r?\n/)) { const line = raw.trim(); if (line.length === 0) continue; @@ -136,26 +141,23 @@ export function parseHlsByteRanges( } const byterange = /^#EXT-X-BYTERANGE:(\d+)(?:@(\d+))?$/.exec(line); if (byterange) { - const start = - byterange[2] === undefined ? nextOffset : Number(byterange[2]); - range = { start, length: Number(byterange[1]) }; + range = { + start: byterange[2] === undefined ? null : Number(byterange[2]), + length: Number(byterange[1]), + }; continue; } if (line.startsWith("#")) continue; - if ( - line !== mediaName || - range === null || - duration === null || - !Number.isFinite(duration) - ) { + if (range === null || duration === null || !Number.isFinite(duration)) { + return null; + } + const start = range.start ?? nextOffsets.get(line) ?? 0; + nextOffsets.set(line, start + range.length); + if (line === mediaName) { + segments.push({ start, end: start + range.length, duration }); + } else if (!SLICE_MEDIA_FILE.test(line)) { return null; } - segments.push({ - start: range.start, - end: range.start + range.length, - duration, - }); - nextOffset = range.start + range.length; range = null; duration = null; } @@ -317,7 +319,9 @@ export function ladderTopTier( } export interface DeliveryResolver { - hls(playlistKey: string): Promise; + // The segments `mediaName` holds in the playlist at `playlistKey`. Without + // a name, the media file named like the playlist. + hls(playlistKey: string, mediaName?: string): Promise; video(request: { videoId: string; key: string; @@ -343,7 +347,10 @@ export function cachedDeliveryResolver( return entry; }; return { - hls: (key) => remember(playlists, key, () => resolver.hls(key)), + hls: (key, mediaName) => + remember(playlists, `${key}\n${mediaName ?? ""}`, () => + resolver.hls(key, mediaName), + ), video: (request) => remember(videos, request.videoId, () => resolver.video(request)), }; @@ -370,6 +377,27 @@ async function ownedVideo( return video && video.appId === request.appId ? video : null; } +// A slice's media is listed in the whole ladder's playlist once every slice is +// in. Before that, only the slice's own playlist in the encode's slices folder +// lists it, under the name the encoder gave it. +async function hlsSegments( + request: DeliveryRequest, + resolver: DeliveryResolver, +): Promise { + const folder = request.key.slice(0, request.key.lastIndexOf("/") + 1); + const file = request.key.slice(folder.length); + const slice = SLICE_MEDIA_FILE.exec(file); + if (!slice) return resolver.hls(request.key.replace(/\.m4s$/, ".m3u8")); + const [, rendition] = slice; + return ( + (await resolver.hls(`${folder}${rendition}.m3u8`, file)) ?? + (await resolver.hls( + `${folder.replace(/hls\/$/, "slices/")}${file.replace(/\.m4s$/, ".m3u8")}`, + `${rendition}.m4s`, + )) + ); +} + /** * Measurement only. HLS seconds are exact: a byte range maps to the playlist's * segments, and a range that covers part of a segment counts it pro rata by @@ -394,7 +422,7 @@ export async function deliveredSeconds( ? { tier, seconds: request.bodyBytes / rate, method: "mp4-estimate" } : { tier, seconds: 0, method: "no-rate" }; } - const segments = await resolver.hls(request.key.replace(/\.m4s$/, ".m3u8")); + const segments = await hlsSegments(request, resolver); if (segments) { return { tier: request.kind, diff --git a/apps/web/lib/infra-delivery-store.ts b/apps/web/lib/infra-delivery-store.ts index ade727517cc..2184ee4667c 100644 --- a/apps/web/lib/infra-delivery-store.ts +++ b/apps/web/lib/infra-delivery-store.ts @@ -90,11 +90,11 @@ export function s3DeliveryLogSource( } export const storageDeliveryResolver: DeliveryResolver = { - async hls(playlistKey) { + async hls(playlistKey, mediaName) { const text = await readDeveloperVideoText(playlistKey); const name = playlistKey.slice(playlistKey.lastIndexOf("/") + 1); return text - ? parseHlsByteRanges(text, name.replace(/\.m3u8$/, ".m4s")) + ? parseHlsByteRanges(text, mediaName ?? name.replace(/\.m3u8$/, ".m4s")) : null; }, async video({ videoId }) { diff --git a/packages/database/types/developer-video.ts b/packages/database/types/developer-video.ts index b22b84f50dc..b037f6f3906 100644 --- a/packages/database/types/developer-video.ts +++ b/packages/database/types/developer-video.ts @@ -48,6 +48,53 @@ export interface DeveloperVideoProbe { fps: number | null; } +export interface DeveloperVideoSliceRendition { + name: string; + width: number; + height: number; + bandwidth: number; + averageBandwidth: number; + bytes: number; + codecs: string | null; +} + +export interface DeveloperVideoSliceResult { + seconds: number; + renditions: DeveloperVideoSliceRendition[]; + audioBytes: number; + // The first slice carries the whole video's audio, which may be copied from + // the source at its own rate instead of encoded at 128 kbps. + audioBitrate?: number; +} + +// One stretch of the video encoded as its own ladder job. The job moves +// through the same states as a whole output and never becomes "unavailable". +export interface DeveloperVideoSlice { + startSeconds: number; + // Null on the last slice, which runs to the end of the video. + durationSeconds: number | null; + job: DeveloperVideoPartState; + attempts: number; + lostJobs: number; + result?: DeveloperVideoSliceResult; +} + +// Adaptive streaming encoded in slices on several workers. While it is +// "encoding" the hls part stays pending and playlists are built from the +// finished slices. "complete" means the whole ladder was written to `output` +// and the hls part points at it. "abandoned" means a slice could not be made +// and the hls part is encoded as one job instead. +export interface DeveloperVideoSlicedHls { + state: "encoding" | "complete" | "abandoned"; + output: string; + // Set when the first slice lands: whether the growing ladder is offered to + // viewers, or held back until it is whole because the MP4 already plays. + exposed: boolean | null; + hasAudio: boolean; + slices: DeveloperVideoSlice[]; + reason?: string; +} + export interface DeveloperVideoProcessingState { encodeId: string; startedAt: number; @@ -55,6 +102,7 @@ export interface DeveloperVideoProcessingState { probing?: { token: string; startedAt: number }; hls: DeveloperVideoPartState; mp4: DeveloperVideoPartState; + sliced?: DeveloperVideoSlicedHls; attempts: Record; lostJobs?: Partial>; // When every worker last answered busy to a start of this encode. diff --git a/packages/env/server.ts b/packages/env/server.ts index 7e764bb5036..2e3ef6ce099 100644 --- a/packages/env/server.ts +++ b/packages/env/server.ts @@ -262,6 +262,12 @@ function createServerEnv() { .describe( "Base URL for developer video playback links. Defaults to WEB_URL/api/developer/playback; set it to a hostname that forwards to that path", ), + INFRA_SLICED_HLS: z + .string() + .optional() + .describe( + "Set to off to encode every developer video's adaptive stream as one job instead of in slices", + ), LOOM_IMPORTER_URL: z .string() .optional()