diff --git a/apps/mobile-ios/Cap/App/CapApp.swift b/apps/mobile-ios/Cap/App/CapApp.swift index d8e65691353..6662e6c875b 100644 --- a/apps/mobile-ios/Cap/App/CapApp.swift +++ b/apps/mobile-ios/Cap/App/CapApp.swift @@ -157,6 +157,7 @@ struct MainTabView: View { @State private var analyticsPath: [Route] = [] @State private var settingsPath: [Route] = [] @State private var recorder: RecorderRequest? + @State private var studio = StudioSession.shared @State private var capModels = CapDetailModels() @State private var heldLayout: CapsLayout? @@ -182,6 +183,10 @@ struct MainTabView: View { .fullScreenCover(item: $recorder) { request in RecordView(session: session, folderId: request.folderId) } + // From a notification while the recorder isn't up; the recorder shows the editor itself. + .fullScreenCover(item: Binding(get: { recorder == nil ? studio.editing : nil }, set: { studio.editing = $0 })) { project in + StudioEditorCover(project: project, session: session) + } .onReceive(NotificationCenter.default.publisher(for: .capOpenRecorder)) { note in guard recorder == nil else { return } recorder = RecorderRequest(folderId: note.object as? String) @@ -201,6 +206,7 @@ struct MainTabView: View { } .onAppear { RecordingCoordinator.shared.session = session + studio.attach(session) let coordinator = RecordingCoordinator.shared coordinator.collectFinishedScreenRecording(showHome: false) // A fresh launch after a recording stopped while Cap was away. @@ -253,6 +259,10 @@ struct MainTabView: View { settingsPath = [.profile] case ("analytics", let id?): if let capId = CapID(id) { homePath = [.capAnalytics(capId, title: "Walkthrough: the new Cap 0.6.2 editor")] } + case ("studio", let directory?): + // A folder with screen.mp4 and/or camera.mp4, opened as a Studio project for tests and measurements. + studio.attach(session) + StudioDebug.openFixture(URL(fileURLWithPath: directory)) default: break } diff --git a/apps/mobile-ios/Cap/App/PushNotifications.swift b/apps/mobile-ios/Cap/App/PushNotifications.swift index 31601a7e3c2..1a5be734215 100644 --- a/apps/mobile-ios/Cap/App/PushNotifications.swift +++ b/apps/mobile-ios/Cap/App/PushNotifications.swift @@ -82,6 +82,10 @@ final class PushNotifications { } func open(_ userInfo: [AnyHashable: Any]) -> Bool { + if let projectId = userInfo["studioProject"] as? String { + Task { await StudioSession.shared.open(projectId: projectId) } + return true + } guard let id = (userInfo["capId"] as? String).flatMap(CapID.init) else { return false } pendingCapId = id return true diff --git a/apps/mobile-ios/Cap/App/SessionModel.swift b/apps/mobile-ios/Cap/App/SessionModel.swift index faf4a4ed10c..05424de85c4 100644 --- a/apps/mobile-ios/Cap/App/SessionModel.swift +++ b/apps/mobile-ios/Cap/App/SessionModel.swift @@ -200,6 +200,7 @@ final class SessionModel { func tearDown() { tornDown = true uploads.cancelAll() + StudioSession.shared.signedOut() for library in libraries.values { library.cancel() } libraries.removeAll() } diff --git a/apps/mobile-ios/Cap/Features/Cap/CapDetailView.swift b/apps/mobile-ios/Cap/Features/Cap/CapDetailView.swift index ae228321562..36dfb3fb6d5 100644 --- a/apps/mobile-ios/Cap/Features/Cap/CapDetailView.swift +++ b/apps/mobile-ios/Cap/Features/Cap/CapDetailView.swift @@ -68,7 +68,8 @@ struct CapDetailView: View { EditorView(model: editor) { outcome in self.editor = nil switch outcome { - case .cancelled: + case .cancelled, .rendered: + // A rendered edit uploads as a new Cap; this one stays as it is. break case .saved: session.toasts.show("Saving your edit. Your Cap updates in a moment.", systemImage: "scissors") diff --git a/apps/mobile-ios/Cap/Features/Edit/EditorModel.swift b/apps/mobile-ios/Cap/Features/Edit/EditorModel.swift index ff946adcc2b..0ba5564687d 100644 --- a/apps/mobile-ios/Cap/Features/Edit/EditorModel.swift +++ b/apps/mobile-ios/Cap/Features/Edit/EditorModel.swift @@ -3,8 +3,8 @@ import CapKit import QuartzCore import SwiftUI -/// Edits never render on the phone: the preview is an `AVMutableComposition` of the kept clips, rebuilt -/// in a few milliseconds per change, and Cap renders the saved version. +/// The preview is an `AVMutableComposition` of the kept clips, rebuilt in a few milliseconds per change, drawn through +/// Studio's compositor when the look changes anything. A Cap's cuts are rendered by Cap; anything styled renders here. @MainActor @Observable final class EditorModel { @@ -14,15 +14,20 @@ final class EditorModel { case failed(String) } - let capId: CapID + enum Document { + case cap(CapID, preloaded: AVURLAsset?) + case studio(StudioProject) + } + + let document: Document let title: String let state: CapEditState private let session: SessionModel private(set) var phase: Phase = .loading private(set) var timeline: EditTimeline - private(set) var undoStack: [EditTimeline] = [] - private(set) var redoStack: [EditTimeline] = [] + private(set) var undoStack: [Snapshot] = [] + private(set) var redoStack: [Snapshot] = [] var selectedClip: Int? /// Two players for the edit: a rebuilt preview is prepared on the hidden one and swapped in once its frame is @@ -48,36 +53,119 @@ final class EditorModel { private static let detailCount = thumbnailCount * Int(TrimTrack.zoomFactor) private var detailTask: Task? private(set) var videoAspect: CGFloat = 9.0 / 16 - - private var asset: AVURLAsset? - private var videoTrack: AVAssetTrack? - private var audioTrack: AVAssetTrack? - private var assetDuration: CMTime = .zero - private var videoTransform: CGAffineTransform = .identity + private(set) var sourceAspect: CGFloat = 9.0 / 16 + + // MARK: Look + + private(set) var look: StudioLook + private(set) var music: StudioMusic? + private(set) var voiceVolume: Double + /// Shown over the player after a look change while paused: the last decoded frame drawn with the new look. + private(set) var still: CVPixelBuffer? + private(set) var stillGeneration = 0 + private(set) var renderSize: CGSize = .zero + private(set) var musicWaveform: [Float] = [] + let scene: StudioScene + + private var screenSource: StudioTrackSource? + private var cameraSource: StudioTrackSource? + private var musicSource: StudioMusicSource? + private var primaryAsset: AVAsset? + private var primaryOffset: Double = 0 private var timeObservers: [Any] = [] private var rebuildGeneration = 0 private var endObserver: NSObjectProtocol? private var itemReady: NSKeyValueObservation? private var rebuildPending = false private var thumbnailTask: Task? + private var compositionStale = false + private var prewarm: Task? let metrics = EditorMetrics() - private let preloaded: AVURLAsset? + struct Snapshot: Equatable { + var timeline: EditTimeline + var look: StudioLook + var music: StudioMusic? + var voiceVolume: Double + } init(capId: CapID, title: String, state: CapEditState, session: SessionModel, preloaded: AVURLAsset? = nil) { - self.preloaded = preloaded - self.capId = capId + document = .cap(capId, preloaded: preloaded) self.title = title self.state = state self.session = session timeline = EditTimeline(sourceDuration: state.duration) + look = .untouched + music = nil + voiceVolume = 1 + scene = StudioScene(look: .untouched, screen: nil, camera: nil) + for player in players { player.automaticallyWaitsToMinimizeStalling = false } + sourcePlayer.automaticallyWaitsToMinimizeStalling = false + } + + init(project: StudioProject, session: SessionModel) { + document = .studio(project) + draftStore = StudioSession.shared.store + title = project.title + state = CapEditState(status: .available, duration: project.duration, hasEdits: false) + self.session = session + timeline = project.timeline + look = project.look + music = project.music + voiceVolume = project.voiceVolume + scene = StudioScene(look: project.look, screen: nil, camera: nil) for player in players { player.automaticallyWaitsToMinimizeStalling = false } sourcePlayer.automaticallyWaitsToMinimizeStalling = false } + var capId: CapID? { + if case let .cap(id, _) = document { id } else { nil } + } + + var project: StudioProject? { + guard case var .studio(project) = document else { return nil } + project.look = look + project.music = music + project.voiceVolume = voiceVolume + project.clips = timeline.isUnchanged ? nil : timeline.clips + return project + } + + var isStudio: Bool { project != nil } + + /// The account a Studio project was opened in. Signing out (or a session expiring) while it's open must not send + /// its saves anywhere else. + @ObservationIgnored private var draftStore: StudioStore? + private var store: StudioStore { draftStore ?? StudioSession.shared.store } + var hasCamera: Bool { cameraSource != nil } + var hasScreen: Bool { screenSource != nil } + + var exportSize: CGSize { + StudioLayout.canvasSize(look: look, primary: screenSource?.info.displaySize, camera: cameraSource?.info.displaySize, longEdge: StudioLayout.exportLongEdge) + } + + private var snapshot: Snapshot { Snapshot(timeline: timeline, look: look, music: music, voiceVolume: voiceVolume) } + + /// A Cap with only cuts saves on Cap's servers; anything styled has to be rendered on the phone. + var needsRender: Bool { + isStudio || !look.isIdentity(hasCamera: false) || music != nil || voiceVolume != 1 + } + + var workDirectory: URL { + switch document { + case let .studio(project): + return store.directory(for: project.id) + case let .cap(id, _): + let caches = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + let directory = caches.appending(path: "EditorMedia/\(id.rawValue)", directoryHint: .isDirectory) + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + } + var canUndo: Bool { !undoStack.isEmpty } var canRedo: Bool { !redoStack.isEmpty } - var hasChanges: Bool { !timeline.isUnchanged } + var hasChanges: Bool { !timeline.isUnchanged || needsRender && !isStudio } var canSplit: Bool { timeline.splitting(atOutput: outputTime) != nil } var canDelete: Bool { guard let selectedClip else { return false } @@ -93,41 +181,68 @@ final class EditorModel { func load() async { metrics.begin() do { - let asset: AVURLAsset - if let preloaded { - asset = preloaded - metrics.mark("reusedPlayback") - } else { - asset = AVURLAsset(url: try await sourceURL(), options: [AVURLAssetPreferPreciseDurationAndTimingKey: true]) + switch document { + case let .cap(_, preloaded): + let asset: AVURLAsset + if let preloaded { + asset = preloaded + metrics.mark("reusedPlayback") + } else { + asset = AVURLAsset(url: try await sourceURL(), options: [AVURLAssetPreferPreciseDurationAndTimingKey: true]) + } + screenSource = try await StudioTrackSource.load(asset: asset, offset: 0) + case let .studio(project): + let store = self.store + async let screen = Self.loadTrack(project.screen, url: { store.url($0, in: project.id) }) + async let camera = Self.loadTrack(project.camera, url: { store.url($0, in: project.id) }) + screenSource = try await screen + cameraSource = try? await camera + guard screenSource != nil || cameraSource != nil else { throw EditorError.noVideo } + } + let primary = screenSource ?? cameraSource + primaryAsset = primary?.asset + primaryOffset = primary?.offset ?? 0 + if let size = primary?.info.displaySize, size.height > 0 { + sourceAspect = min(max(size.width / size.height, 0.4), 2.4) } - async let videoTracks = asset.loadTracks(withMediaType: .video) - async let audioTracks = asset.loadTracks(withMediaType: .audio) - async let duration = asset.load(.duration) - guard let video = try await videoTracks.first else { throw EditorError.noVideo } - let audio = try await audioTracks.first - assetDuration = try await duration - videoTransform = (try? await video.load(.preferredTransform)) ?? .identity - if let size = try? await video.load(.naturalSize).applying(videoTransform), size.height != 0 { - videoAspect = min(max(abs(size.width / size.height), 0.4), 2.4) + scene.update { + $0.screen = self.screenSource?.info + $0.camera = self.cameraSource?.info } - self.asset = asset - videoTrack = video - audioTrack = audio metrics.mark("tracksLoaded") + if !look.isIdentity(hasCamera: cameraSource != nil) { + let scene = scene + let size = StudioLayout.canvasSize(look: look, primary: screenSource?.info.displaySize, camera: cameraSource?.info.displaySize, longEdge: StudioLayout.previewLongEdge) + let presets = presetRenderer + prewarm = Task.detached(priority: .userInitiated) { + scene.renderer.prewarm(scene: scene, size: size) + // The first picture made for the interface compiles its own path; the Presets strip shouldn't pay for it. + presets.prewarmImages(scene: scene) + } + } + await loadBackgroundImage() + await loadMusic() + // The kernels compile while the decoders start up; neither waits for the other. - sourcePlayer.replaceCurrentItem(with: AVPlayerItem(asset: asset)) + try rebuildSource() try rebuild(seekTo: 0) observePlayback() phase = .ready - Usage.screen("editor") - loadThumbnails(asset: asset) + Usage.screen(isStudio ? "studio_editor" : "editor") + if let primaryAsset { loadThumbnails(asset: primaryAsset) } } catch { phase = .failed((error as? CapAPIError)?.errorDescription ?? "Cap couldn't open this video for editing. Try again.") } } + nonisolated private static func loadTrack(_ track: StudioProject.Track?, url: (String) -> URL) async throws -> StudioTrackSource? { + guard let track else { return nil } + return try await StudioTrackSource.load(url: url(track.file), offset: track.offset, gaps: track.gaps ?? [], audio: track.audio.map { (url($0.file), $0.offset, $0.duration) }) + } + /// Playback is an MP4 for every editable Cap; the download link covers anything else. private func sourceURL() async throws -> URL { + guard let capId else { throw EditorError.noVideo } let environment = session.services.environment let playback = try await session.api.playback(capId) if playback.kind == .mp4, let url = SafeURL.media(playback.url, environment: environment) { return url } @@ -136,7 +251,7 @@ final class EditorModel { return url } - private func loadThumbnails(asset: AVURLAsset) { + private func loadThumbnails(asset: AVAsset) { let duration = timeline.sourceDuration let scale = UITraitCollection.current.displayScale thumbnailTask?.cancel() @@ -148,8 +263,9 @@ final class EditorModel { let tolerance = CMTime(seconds: max(0.5, duration / Double(Self.thumbnailCount)), preferredTimescale: 600) generator.requestedTimeToleranceBefore = tolerance generator.requestedTimeToleranceAfter = tolerance + let shift = self?.primaryOffset ?? 0 let times = (0.. 0 else { return } + guard let asset = primaryAsset, timeline.sourceDuration > 0 else { return } + let shift = primaryOffset let duration = timeline.sourceDuration let step = duration / Double(Self.detailCount) let centre = Int(time / step) @@ -207,7 +324,7 @@ final class EditorModel { let tolerance = CMTime(seconds: step / 2, preferredTimescale: 600) generator.requestedTimeToleranceBefore = tolerance generator.requestedTimeToleranceAfter = tolerance - let times = wanted.map { CMTime(seconds: (Double($0) + 0.5) * step, preferredTimescale: 600) } + let times = wanted.map { CMTime(seconds: max(0, (Double($0) + 0.5) * step - shift), preferredTimescale: 600) } for await result in generator.images(for: times) { guard let self, !Task.isCancelled else { return } guard let image = try? result.image, @@ -228,28 +345,16 @@ final class EditorModel { /// Only track references change, so building costs milliseconds; the new item loads on the hidden /// player and replaces the visible one when its first frame is ready. private func rebuild(seekTo output: Double) throws { - guard let videoTrack else { return } let started = CACurrentMediaTime() - let composition = AVMutableComposition() - guard let video = composition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid) else { - throw EditorError.noVideo - } - let audio = audioTrack == nil ? nil : composition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) - var cursor = CMTime.zero - for clip in timeline.clips { - let start = CMTime(seconds: clip.start, preferredTimescale: 600) - let end = CMTimeMinimum(CMTime(seconds: clip.end, preferredTimescale: 600), assetDuration) - guard end > start else { continue } - let range = CMTimeRange(start: start, end: end) - try video.insertTimeRange(range, of: videoTrack, at: cursor) - if let audio, let audioTrack { - try? audio.insertTimeRange(range, of: audioTrack, at: cursor) - } - cursor = cursor + range.duration - } - video.preferredTransform = videoTransform - let item = AVPlayerItem(asset: composition) + let built = try compose(clips: timeline.clips, remembersFrames: true) + let item = AVPlayerItem(asset: built.composition) + item.videoComposition = built.videoComposition + item.audioMix = built.audioMix item.preferredForwardBufferDuration = 2 + renderSize = built.renderSize + if built.renderSize.height > 0 { videoAspect = built.renderSize.width / built.renderSize.height } + compositionStale = false + clearStill() metrics.record("rebuildMs", (CACurrentMediaTime() - started) * 1000) let target = min(max(0, output), timeline.outputDuration) @@ -309,6 +414,30 @@ final class EditorModel { } } + /// An untouched Cap plays straight from the decoder; anything styled, or with a camera, goes through Studio's + /// compositor at a size made for the screen, not the export. + private func compose(clips: [EditRange], remembersFrames: Bool, longEdge: Double = StudioLayout.previewLongEdge) throws -> StudioComposition { + try StudioComposition.build( + clips: clips, + screen: screenSource, + camera: cameraSource, + music: musicSource, + voiceVolume: voiceVolume, + scene: scene, + longEdge: longEdge, + remembersFrames: remembersFrames, + passthrough: look.isIdentity(hasCamera: cameraSource != nil) + ) + } + + /// The whole recording with the current look, shown while a trim handle is dragged. + private func rebuildSource() throws { + let built = try compose(clips: [EditRange(start: 0, end: timeline.sourceDuration)], remembersFrames: false) + let item = AVPlayerItem(asset: built.composition) + item.videoComposition = built.videoComposition + sourcePlayer.replaceCurrentItem(with: item) + } + /// The last good preview stays on its player, and Try again rebuilds from the timeline as it stands. private func previewFailed() { guard !closed else { return } @@ -363,18 +492,29 @@ final class EditorModel { player.pause() isPlaying = false } else { + refreshStaleComposition() + clearStill() if outputTime >= timeline.outputDuration - 0.05 { seek(toOutput: 0) } player.play() isPlaying = true } } + /// Frames the player composed ahead of time still have the old look; a fresh composition object drops them. + private func refreshStaleComposition() { + guard compositionStale, let item = player.currentItem, let current = item.videoComposition else { return } + compositionStale = false + item.videoComposition = current.mutableCopy() as? AVVideoComposition + } + private(set) var isScrubbing = false private var seekInFlight = false private var pendingSeek: Seek? private var settleTask: Task? func beginScrub() { + clearStill() + refreshStaleComposition() isScrubbing = true if isPlaying { player.pause() @@ -393,6 +533,8 @@ final class EditorModel { } func seek(toOutput time: Double) { + clearStill() + refreshStaleComposition() outputTime = min(max(0, time), timeline.outputDuration) player.seek(to: CMTime(seconds: outputTime, preferredTimescale: 600), toleranceBefore: .zero, toleranceAfter: .zero) } @@ -513,11 +655,16 @@ final class EditorModel { private func apply(_ next: EditTimeline, seekTo output: Double) { let started = CACurrentMediaTime() let previous = timeline - undoStack.append(timeline) - redoStack.removeAll() + pushUndo() timeline = next show(next, previous: previous, seekTo: output) metrics.record("editMs", (CACurrentMediaTime() - started) * 1000) + changed() + } + + private func pushUndo() { + undoStack.append(snapshot) + redoStack.removeAll() } func split() { @@ -535,22 +682,411 @@ final class EditorModel { func undo() { guard let previous = undoStack.popLast() else { return } - redoStack.append(timeline) - let source = timeline.sourceTime(forOutput: outputTime) - let current = timeline - timeline = previous - selectedClip = nil - show(previous, previous: current, seekTo: previous.outputTime(forSource: source)) + redoStack.append(snapshot) + restore(previous) } func redo() { guard let next = redoStack.popLast() else { return } - undoStack.append(timeline) + undoStack.append(snapshot) + restore(next) + } + + private func restore(_ target: Snapshot) { + let current = snapshot let source = timeline.sourceTime(forOutput: outputTime) - let current = timeline - timeline = next + timeline = target.timeline selectedClip = nil - show(next, previous: current, seekTo: next.outputTime(forSource: source)) + let audioChanged = current.music != target.music || current.voiceVolume != target.voiceVolume + music = target.music + voiceVolume = target.voiceVolume + // The look goes back at once, so a look chosen while the music loads is newer and isn't replaced by it. + show(look: target.look, timelineFrom: current.timeline, forceRebuild: false, seekTo: target.timeline.outputTime(forSource: source)) + if audioChanged { + Task { + guard await loadMusic() else { return } + show(look: look, timelineFrom: timeline, forceRebuild: true, seekTo: outputTime) + } + } + changed() + } + + // MARK: Look + + private var lookGesture = false + + func beginLookGesture() { + guard !lookGesture else { return } + lookGesture = true + pushUndo() + } + + func endLookGesture() { + lookGesture = false + } + + func setLook(_ change: (inout StudioLook) -> Void) { + var next = look + change(&next) + guard next != look else { return } + if !lookGesture { pushUndo() } + show(look: next, timelineFrom: timeline, forceRebuild: false, seekTo: outputTime) + changed() + } + + // MARK: Presets + + /// Each preset drawn on the paused frame, keyed by look so a renamed preset keeps its picture. + private(set) var presetThumbnails: [StudioLook: UIImage] = [:] + @ObservationIgnored private var presetThumbnailTime: Double? + @ObservationIgnored private var presetTask: Task? + @ObservationIgnored private var presetRerun = false + /// Its own cache, so drawing a dozen looks never makes the live preview redraw its background. + @ObservationIgnored private lazy var presetRenderer = scene.renderer.sibling() + nonisolated static let presetThumbnailEdge: Double = 360 + + func apply(_ preset: StudioPreset) { + store.presets.provideImage(for: preset.look, in: workDirectory) + setLook { $0 = preset.look } + Usage.track("studio_preset_applied", ["preset": .string(preset.isBuiltIn ? preset.id : "saved")]) + } + + /// Draws every preset on the frame that's paused on, from frames already decoded where it can. Pictures for + /// another moment stay up until their replacements are ready, so the strip never blinks. + func loadPresetThumbnails(_ presets: [StudioPreset]) { + guard phase == .ready, !isPlaying else { return } + guard presetTask == nil else { + presetRerun = true + return + } + let time = outputTime + let moved = presetThumbnailTime.map { abs($0 - time) > 0.25 } ?? true + var looks: [StudioLook] = [] + for preset in presets where (moved || presetThumbnails[preset.look] == nil) && !looks.contains(preset.look) { + looks.append(preset.look) + } + guard !looks.isEmpty else { return } + presetThumbnailTime = time + let renderer = presetRenderer + let backgrounds = Dictionary(looks.map { ($0, presetBackgroundURL($0.background)) }, uniquingKeysWith: { a, _ in a }) + var remembered = PresetFrames() + if let sources = scene.sources(near: CMTime(seconds: time, preferredTimescale: 600), tolerance: 0.25) { + if let info = screenSource?.info, let buffer = sources.screen { remembered.screen = (info, buffer) } + if let info = cameraSource?.info, let buffer = sources.camera { remembered.camera = (info, buffer) } + } + let screen = screenSource + let camera = cameraSource + let sourceTime = timeline.sourceTime(forOutput: time) + let metrics = metrics + presetTask = Task { [weak self] in + let started = CACurrentMediaTime() + let frames = await Self.presetFrames(remembered: remembered, screen: screen, camera: camera, at: sourceTime) + for look in looks { + let url = backgrounds[look] ?? nil + let drawn = CACurrentMediaTime() + let image = await Task.detached(priority: .userInitiated) { + Self.drawPreset(look, frames: frames, background: url.flatMap { StudioAssets.loadImage($0, maxPixels: 480) }, renderer: renderer) + }.value + metrics.record("presetThumbMs", (CACurrentMediaTime() - drawn) * 1000) + guard let self, !Task.isCancelled else { return } + if let image { presetThumbnails[look] = image } + } + metrics.record("presetStripMs", (CACurrentMediaTime() - started) * 1000) + guard let self else { return } + presetTask = nil + if presetRerun { + presetRerun = false + loadPresetThumbnails(StudioSession.shared.presetLibrary.all) + } + } + } + + private func presetBackgroundURL(_ background: StudioBackground) -> URL? { + switch background { + case let .wallpaper(name): + return StudioAssets.wallpaperThumbnailURL(name) + case let .image(name): + let local = workDirectory.appending(path: name) + return FileManager.default.fileExists(atPath: local.path) ? local : store.presets.imageURL(name) + default: + return nil + } + } + + nonisolated struct PresetFrames: @unchecked Sendable { + var screen: (info: StudioSourceInfo, buffer: CVPixelBuffer)? + var camera: (info: StudioSourceInfo, buffer: CVPixelBuffer)? + } + + /// The frames the preview composed at this moment, or, when it hasn't composed one yet (a seek still settling, + /// or an unstyled Cap, which plays without the compositor), the same moment decoded once at a small size. + nonisolated private static func presetFrames(remembered: PresetFrames, screen: StudioTrackSource?, camera: StudioTrackSource?, at time: Double) async -> PresetFrames { + if remembered.screen != nil || remembered.camera != nil { return remembered } + var frames = PresetFrames() + async let screenFrame = decodeFrame(screen, at: time) + async let cameraFrame = decodeFrame(camera, at: time) + frames.screen = await screenFrame + frames.camera = await cameraFrame + return frames + } + + nonisolated private static func decodeFrame(_ source: StudioTrackSource?, at time: Double) async -> (info: StudioSourceInfo, buffer: CVPixelBuffer)? { + guard let source else { return nil } + let generator = AVAssetImageGenerator(asset: source.asset) + // Already upright, so it's drawn with no rotation of its own. + generator.appliesPreferredTrackTransform = true + generator.maximumSize = CGSize(width: 720, height: 720) + let tolerance = CMTime(seconds: 0.5, preferredTimescale: 600) + generator.requestedTimeToleranceBefore = tolerance + generator.requestedTimeToleranceAfter = tolerance + let local = min(max(0, time - source.offset), max(0, source.duration - 0.1)) + guard let image = try? await generator.image(at: CMTime(seconds: local, preferredTimescale: 600)).image, + let buffer = pixelBuffer(image) + else { return nil } + return (StudioSourceInfo(naturalSize: CGSize(width: image.width, height: image.height)), buffer) + } + + nonisolated private static func pixelBuffer(_ image: CGImage) -> CVPixelBuffer? { + var buffer: CVPixelBuffer? + let attributes = [kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any](), kCVPixelBufferCGBitmapContextCompatibilityKey as String: true] as CFDictionary + guard CVPixelBufferCreate(kCFAllocatorDefault, image.width, image.height, kCVPixelFormatType_32BGRA, attributes, &buffer) == kCVReturnSuccess, let buffer else { return nil } + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + guard let context = CGContext(data: CVPixelBufferGetBaseAddress(buffer), width: image.width, height: image.height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpace(name: CGColorSpace.sRGB)!, bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else { return nil } + context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + return buffer + } + + nonisolated private static func drawPreset(_ look: StudioLook, frames: PresetFrames, background: CGImage?, renderer: StudioRenderer) -> UIImage? { + let scene = StudioScene(look: look, screen: frames.screen?.info, camera: frames.camera?.info, backgroundImage: background, renderer: renderer) + let size = StudioLayout.canvasSize(look: look, primary: frames.screen?.info.displaySize, camera: frames.camera?.info.displaySize, longEdge: presetThumbnailEdge) + return renderer.image(scene: scene, screen: frames.screen?.buffer, camera: frames.camera?.buffer, size: size).map { UIImage(cgImage: $0) } + } + + /// Applies a look. Only what changed is redone: a new frame size or the switch between plain playback and + /// compositing rebuilds the preview, anything else just redraws the frame on screen. + private func show(look next: StudioLook, timelineFrom previous: EditTimeline, forceRebuild: Bool, seekTo output: Double) { + let started = CACurrentMediaTime() + let old = look + look = next + let hasCamera = cameraSource != nil + let backgroundChanged = old.background != next.background + scene.update { $0.look = next } + let size = StudioLayout.canvasSize(look: next, primary: screenSource?.info.displaySize, camera: cameraSource?.info.displaySize, longEdge: StudioLayout.previewLongEdge) + let needsRebuild = forceRebuild + || !timeline.playsSame(as: previous) + || old.isIdentity(hasCamera: hasCamera) != next.isIdentity(hasCamera: hasCamera) + || (videoCompositionActive && size != renderSize) + if needsRebuild { + do { + try rebuildSource() + try rebuild(seekTo: output) + } catch { + previewFailed() + } + } else { + outputTime = min(max(0, output), timeline.outputDuration) + compositionStale = true + refreshStill() + } + metrics.record("lookMs", (CACurrentMediaTime() - started) * 1000) + if backgroundChanged { Task { await loadBackgroundImage() } } + } + + private var videoCompositionActive: Bool { player.currentItem?.videoComposition != nil } + + /// Redraws the paused frame with the current look from the frame already decoded, in a couple of milliseconds. + private func refreshStill() { + guard !isPlaying, !isScrubbing, !isTrimming, videoCompositionActive else { return } + let started = CACurrentMediaTime() + if let frame = scene.renderStill(near: CMTime(seconds: outputTime, preferredTimescale: 600)) { + still = frame + stillGeneration += 1 + metrics.record("stillMs", (CACurrentMediaTime() - started) * 1000) + } else { + // Nothing decoded near here yet: an exact seek composes the frame with the new look. + player.seek(to: CMTime(seconds: outputTime, preferredTimescale: 600), toleranceBefore: .zero, toleranceAfter: .zero) + } + } + + private func clearStill() { + guard still != nil else { return } + still = nil + stillGeneration += 1 + } + + @ObservationIgnored private var backgroundLoaded: StudioBackground? + + /// Wallpapers and photos are decoded off the main thread at the size they're drawn, then the frame is redrawn. + private func loadBackgroundImage() async { + beginMedia() + defer { endMedia() } + let background = look.background + guard background != backgroundLoaded else { return } + let url: URL? = switch background { + case let .wallpaper(name): StudioAssets.wallpaperURL(name) + case let .image(file): workDirectory.appending(path: file) + default: nil + } + let image: CGImage? = if let url { await Task.detached(priority: .userInitiated) { StudioAssets.loadImage(url) }.value } else { nil } + guard look.background == background else { return } + backgroundLoaded = background + scene.update { $0.backgroundImage = image } + if url != nil { + compositionStale = true + refreshStill() + } + } + + // MARK: Music + + func setMusic(_ next: StudioMusic?) async { + guard next != music else { return } + pushUndo() + music = next + changed() + guard await loadMusic() else { return } + show(look: look, timelineFrom: timeline, forceRebuild: true, seekTo: outputTime) + } + + func beginAudioGesture() { + guard !lookGesture else { return } + lookGesture = true + pushUndo() + } + + /// Volumes live in the audio mix, which only a rebuild changes; it keeps the frame on screen meanwhile. + func setVolumes(music musicVolume: Double?, voice: Double?) { + if let musicVolume, var current = music { + current.volume = musicVolume + music = current + } + if let voice { voiceVolume = voice } + if !lookGesture { pushUndo() } + Task { + guard await loadMusic() else { return } + show(look: look, timelineFrom: timeline, forceRebuild: true, seekTo: outputTime) + } + changed() + } + + @ObservationIgnored private var loadedMusic: (source: StudioMusic.Source, url: URL)? + @ObservationIgnored private var waveformLoad: (url: URL, task: Task<[Float], Never>)? + + private func musicURL(_ source: StudioMusic.Source) -> URL? { + switch source { + case let .builtIn(name): StudioAssets.music.first { $0.id == name }?.url + case let .file(file): workDirectory.appending(path: file) + } + } + + /// False when a newer choice was made while this one loaded; that choice's own load applies it. + @discardableResult + private func loadMusic() async -> Bool { + let wanted = music + guard let wanted, let url = musicURL(wanted.source) else { + musicSource = nil + musicWaveform = [] + // Choosing the same song again later reads its waveform again. + loadedMusic = nil + return true + } + beginMedia() + defer { endMedia() } + let source = try? await StudioMusicSource.load(url: url, volume: wanted.volume, offset: wanted.offset) + guard music == wanted else { return false } + musicSource = source + if loadedMusic?.url != url { + // One read of the file per song, however many volume changes ask for it meanwhile. + let task: Task<[Float], Never> + if let pending = waveformLoad, pending.url == url { + task = pending.task + } else { + task = Task.detached(priority: .utility) { await StudioWaveform.peaks(url: url, count: 160) } + waveformLoad = (url, task) + } + let peaks = await task.value + if waveformLoad?.url == url { waveformLoad = nil } + // A new volume for the same song still wants this song's waveform. + if music?.source == wanted.source { + musicWaveform = peaks + loadedMusic = (wanted.source, url) + } + } + return music == wanted + } + + // MARK: Pending media + + private var mediaInFlight = 0 + private var mediaWaiters: [CheckedContinuation] = [] + + private func beginMedia() { + mediaInFlight += 1 + } + + private func endMedia() { + mediaInFlight -= 1 + guard mediaInFlight == 0 else { return } + let waiters = mediaWaiters + mediaWaiters = [] + for waiter in waiters { waiter.resume() } + } + + /// A wallpaper or song picked a moment ago may still be loading; a render made now would leave it out. + func waitForMedia() async { + guard mediaInFlight > 0 else { return } + await withCheckedContinuation { mediaWaiters.append($0) } + } + + /// Lets a cancelled render stop waiting, however long the media takes. + func stopWaitingForMedia() { + let waiters = mediaWaiters + mediaWaiters = [] + for waiter in waiters { waiter.resume() } + } + + func pauseForExport() { + if isPlaying { + player.pause() + isPlaying = false + } + } + + // MARK: Saving drafts + + @ObservationIgnored private var autosaveTask: Task? + + /// Studio projects save themselves as they're edited; there's nothing to lose by closing. + private func changed() { + guard isStudio else { return } + autosaveTask?.cancel() + let store = self.store + autosaveTask = Task { [weak self] in + try? await Task.sleep(for: .milliseconds(400)) + guard !Task.isCancelled, let project = self?.project else { return } + StudioSession.shared.save(project, in: store) + } + } + + func saveDraftNow() { + autosaveTask?.cancel() + if let project { StudioSession.shared.save(project, in: store) } + } + + /// The edit at full quality with a look that no longer changes, for rendering while the editor carries on. + func exportComposition() throws -> StudioComposition { + let frozen = scene.frozen() + return try StudioComposition.build( + clips: timeline.clips, + screen: screenSource, + camera: cameraSource, + music: musicSource, + voiceVolume: voiceVolume, + scene: frozen, + longEdge: StudioLayout.exportLongEdge, + remembersFrames: false + ) } // MARK: Trimming @@ -563,6 +1099,8 @@ final class EditorModel { var displayedTimeline: EditTimeline { trimPreview ?? timeline } func beginTrim() { + clearStill() + refreshStaleComposition() if isPlaying { player.pause() isPlaying = false @@ -599,8 +1137,9 @@ final class EditorModel { // MARK: Saving + /// Cuts to a Cap, saved and rendered by Cap. Styled edits go through `EditorExport` instead. func save() async -> Bool { - guard hasChanges, !isSaving else { return false } + guard hasChanges, !isSaving, !needsRender, let capId else { return false } isSaving = true defer { isSaving = false } do { @@ -616,6 +1155,7 @@ final class EditorModel { } func restoreOriginal() async -> Bool { + guard let capId else { return false } isSaving = true defer { isSaving = false } do { @@ -642,6 +1182,9 @@ final class EditorModel { for player in players { player.pause() } sourcePlayer.pause() removePlaybackObservers() + saveDraftNow() + scene.forgetFrames() + for value in StudioCompositor.takeRenderTimes() { metrics.record("composeMs", value) } metrics.finish(sourceSeconds: timeline.sourceDuration, clips: timeline.clips.count) } @@ -710,12 +1253,110 @@ enum EditorDebug { static var runsScript: Bool { ProcessInfo.processInfo.environment["CAP_DEBUG_EDITOR_SCRIPT"] == "1" } } +extension EditorModel { + /// Look changes as a person makes them: taps through backgrounds, a padding drag, the camera moved around, + /// a new frame shape, music, playback, then a render. False when a check along the way failed. + func runLookScript(pause: (Double) async -> Void) async -> Bool { + var passed = true + func check(_ name: String, _ ok: Bool) { + ProcessStats.note("Studio script check \(name): \(ok ? "ok" : "FAILED")") + if !ok { passed = false } + } + func memory(_ stage: String) { + ProcessStats.note("Studio script stage \(stage)") + let footprint = ProcessStats.footprint() + metrics.record("memoryMB.\(stage)", footprint.current) + metrics.record("peakMB.\(stage)", footprint.peak) + } + memory("lookStart") + seek(toOutput: timeline.outputDuration / 2) + await pause(0.6) + for background in StudioBackground.gradients.prefix(6) { + setLook { $0.background = background } + await pause(0.15) + } + beginLookGesture() + for step in 0...40 { + setLook { $0.padding = 0.15 * Double(step) / 40 } + await pause(1.0 / 60) + } + endLookGesture() + await pause(0.3) + beginLookGesture() + for step in 0...30 { + setLook { $0.rounding = 0.12 * Double(step) / 30 } + await pause(1.0 / 60) + } + endLookGesture() + setLook { $0.background = .wallpaper("blue-1") } + await pause(0.6) + if hasCamera, hasScreen { + for corner in StudioCorner.allCases { + setLook { $0.camera.corner = corner } + await pause(0.2) + } + for layout in [StudioCameraLayout.split, .cameraOnly, .bubble] { + setLook { $0.camera.layout = layout } + await pause(0.5) + } + } + setLook { $0.aspect = .square } + await pause(0.6) + setLook { $0.aspect = .auto } + await pause(0.6) + memory("afterLooks") + let lofi = StudioMusic(source: .builtIn("lofi-beats-mirostar"), title: "Lofi Beats") + await setMusic(lofi) + ProcessStats.note("Studio script stage music") + await pause(0.4) + // None, then the same song again, still draws its waveform. + await setMusic(nil) + await setMusic(lofi) + check("waveformAfterReselect", !musicWaveform.isEmpty) + // Undoing a music change reloads the music; a background chosen meanwhile is newer and stays. + await setMusic(nil) + undo() + setLook { $0.background = .wallpaper("purple-2") } + await pause(1.5) + check("lookKeptAfterUndo", look.background == .wallpaper("purple-2") && music?.source == lofi.source) + seek(toOutput: 0) + await pause(0.3) + togglePlay() + await pause(4) + if isPlaying { togglePlay() } + memory("afterPlay") + await pause(0.3) + do { + ProcessStats.note("Studio script stage export") + let built = try exportComposition() + let url = FileManager.default.temporaryDirectory.appending(path: "studio-script-export.mp4") + let result = try await StudioExporter().export(built, to: url) + metrics.record("exportSeconds", result.seconds) + metrics.record("exportRealtime", result.duration / max(result.seconds, 0.001)) + metrics.record("exportKbps", Double(result.bytes) * 8 / max(result.duration, 0.001) / 1000) + memory("afterExport") + ProcessStats.note("Studio script export \(Int(result.size.width))x\(Int(result.size.height)) \(String(format: "%.1f", result.duration)) s in \(String(format: "%.2f", result.seconds)) s, \(result.bytes / 1024) KB") + try? FileManager.default.removeItem(at: url) + } catch { + ProcessStats.note("Studio script export failed: \(error)") + } + return passed + } +} + extension EditorModel { func runScript(onDone: @escaping () -> Void) async { func pause(_ seconds: Double) async { try? await Task.sleep(for: .seconds(seconds)) } while phase == .loading { await pause(0.05) } guard phase == .ready else { return } await pause(1.5) + metrics.record("memoryMB.open", ProcessStats.footprint().current) + metrics.record("peakMB.open", ProcessStats.footprint().peak) + // A failed check leaves the editor open, which fails the UI test waiting for it to close. + if ProcessInfo.processInfo.environment["CAP_DEBUG_EDITOR_SCRIPT_LOOK_ONLY"] == "1" { + if await runLookScript(pause: pause) { onDone() } + return + } let length = timeline.outputDuration seek(toOutput: length / 3) await pause(0.4) @@ -753,8 +1394,9 @@ extension EditorModel { togglePlay() await pause(2) togglePlay() + let passed = await runLookScript(pause: pause) ProcessStats.note("Editor script finished: \(timeline.clips.count) clips, \(timeline.outputDuration) s") - onDone() + if passed { onDone() } } } #endif diff --git a/apps/mobile-ios/Cap/Features/Edit/EditorPanels.swift b/apps/mobile-ios/Cap/Features/Edit/EditorPanels.swift new file mode 100644 index 00000000000..cff8dc79a3a --- /dev/null +++ b/apps/mobile-ios/Cap/Features/Edit/EditorPanels.swift @@ -0,0 +1,1035 @@ +@preconcurrency import AVFoundation +import CapKit +import Photos +import PhotosUI +import SwiftUI +import UniformTypeIdentifiers + +enum EditorTool: String, CaseIterable, Identifiable { + case trim + case presets + case background + case frame + case camera + case music + + var id: String { rawValue } + + var title: String { + switch self { + case .trim: "Trim" + case .presets: "Presets" + case .background: "Background" + case .frame: "Frame" + case .camera: "Camera" + case .music: "Music" + } + } + + var systemImage: String { + switch self { + case .trim: "scissors" + case .presets: "square.stack" + case .background: "photo.on.rectangle.angled" + case .frame: "aspectratio" + case .camera: "person.crop.circle" + case .music: "music.note" + } + } +} + +enum EditorPanels { + static func height(for tool: EditorTool) -> CGFloat { + switch tool { + case .trim: 56 + case .background: 68 + case .presets: 132 + case .camera: 118 + case .frame, .music: 156 + } + } +} + +struct EditorToolBar: View { + let tools: [EditorTool] + @Binding var selection: EditorTool + + var body: some View { + HStack(spacing: 0) { + ForEach(tools) { tool in + Button { + selection = tool + } label: { + VStack(spacing: 5) { + Image(systemName: tool.systemImage) + .font(.system(size: 19, weight: .medium)) + .frame(height: 24) + Text(tool.title) + .font(.caption2.weight(.medium)) + .lineLimit(1) + .minimumScaleFactor(0.8) + } + .foregroundStyle(selection == tool ? Color.white : Color.white.opacity(0.5)) + .frame(maxWidth: .infinity, minHeight: 50) + .contentShape(.rect) + } + .buttonStyle(.plain) + .accessibilityAddTraits(selection == tool ? .isSelected : []) + .accessibilityIdentifier("editor-tool-\(tool.rawValue)") + } + } + .padding(.horizontal, 12) + .sensoryFeedback(.selection, trigger: selection) + .dynamicTypeSize(...DynamicTypeSize.xLarge) + } +} + +// MARK: - Preview + +struct EditorPreview: View { + let model: EditorModel + /// While the camera tool is open, the bubble's place is outlined even where the camera isn't recording. + var outlinesCamera = false + @State private var dragPoint: CGPoint? + + var body: some View { + GeometryReader { proxy in + let video = Self.videoRect(canvas: model.renderSize, in: proxy.size) + ZStack(alignment: .topLeading) { + ZStack { + ForEach(model.players.indices, id: \.self) { index in + EditorPlayerSurface(player: model.players[index]) { model.metrics.mark("firstFrame") } + .zIndex(index == model.activePlayer ? 1 : 0) + } + EditorPlayerSurface(player: model.sourcePlayer) + .zIndex(model.showsSource ? 2 : -1) + StillSurface(buffer: model.still, generation: model.stillGeneration) + .opacity(model.still == nil ? 0 : 1) + .zIndex(3) + } + .frame(width: proxy.size.width, height: proxy.size.height) + .accessibilityHidden(true) + + if let bubble = bubbleRect(in: video) { + bubbleHandle(bubble, video: video) + } + } + } + .accessibilityElement(children: .contain) + .accessibilityIdentifier("editor-preview") + } + + static func videoRect(canvas: CGSize, in size: CGSize) -> CGRect { + guard canvas.width > 0, canvas.height > 0 else { return CGRect(origin: .zero, size: size) } + return StudioLayout.fit(canvas, in: CGRect(origin: .zero, size: size)) + } + + private func bubbleRect(in video: CGRect) -> CGRect? { + guard model.hasCamera, model.hasScreen, model.look.camera.layout == .bubble, model.renderSize.width > 0 else { return nil } + let layout = StudioLayout(look: model.look, canvas: model.renderSize, primary: model.scene.current.state.screen?.displaySize, camera: model.scene.current.state.camera?.displaySize) + guard let frame = layout.camera?.frame else { return nil } + let scale = video.width / model.renderSize.width + return CGRect(x: video.minX + frame.minX * scale, y: video.minY + frame.minY * scale, width: frame.width * scale, height: frame.height * scale) + } + + private func bubbleHandle(_ bubble: CGRect, video: CGRect) -> some View { + let isCircle = model.look.camera.shape == .circle + let shape = RoundedRectangle(cornerRadius: isCircle ? bubble.width / 2 : bubble.width * 0.22, style: .continuous) + return ZStack { + Color.clear + .frame(width: bubble.width, height: bubble.height) + .contentShape(shape) + .position(x: bubble.midX, y: bubble.midY) + .gesture( + DragGesture(minimumDistance: 4) + .onChanged { value in dragPoint = value.location } + .onEnded { value in + dragPoint = nil + let scale = model.renderSize.width / max(video.width, 1) + let point = CGPoint(x: (value.predictedEndLocation.x - video.minX) * scale, y: (value.predictedEndLocation.y - video.minY) * scale) + let corner = StudioLayout.nearestCorner(to: point, in: model.renderSize) + model.setLook { $0.camera.corner = corner } + } + ) + .accessibilityElement() + .accessibilityLabel("Camera bubble") + .accessibilityHint("Drag to another corner") + .accessibilityIdentifier("editor-camera-bubble") + if outlinesCamera, dragPoint == nil { + shape + .strokeBorder(.white.opacity(0.55), style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) + .frame(width: bubble.width, height: bubble.height) + .position(x: bubble.midX, y: bubble.midY) + .allowsHitTesting(false) + .transition(.opacity) + } + if let dragPoint { + shape + .strokeBorder(.white, lineWidth: 2.5) + .background(shape.fill(.white.opacity(0.18))) + .frame(width: bubble.width, height: bubble.height) + .position(dragPoint) + .allowsHitTesting(false) + .transition(.opacity) + } + } + .animation(.spring(duration: 0.25), value: dragPoint == nil) + .sensoryFeedback(.impact(weight: .light), trigger: model.look.camera.corner) + } +} + +/// Shows one already-rendered frame. A sample buffer layer takes the GPU buffer as it is, with no copy. +struct StillSurface: UIViewRepresentable { + let buffer: CVPixelBuffer? + let generation: Int + + func makeUIView(context: Context) -> StillView { StillView() } + + func updateUIView(_ view: StillView, context: Context) { + view.show(buffer, generation: generation) + } + + final class StillView: UIView { + override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self } + private var displayLayer: AVSampleBufferDisplayLayer { layer as! AVSampleBufferDisplayLayer } + private var shown = -1 + + override init(frame: CGRect) { + super.init(frame: frame) + isUserInteractionEnabled = false + displayLayer.videoGravity = .resizeAspect + } + + required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } + + func show(_ buffer: CVPixelBuffer?, generation: Int) { + guard generation != shown else { return } + shown = generation + let renderer = displayLayer.sampleBufferRenderer + guard let buffer else { + renderer.flush(removingDisplayedImage: true, completionHandler: nil) + return + } + var format: CMVideoFormatDescription? + CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: buffer, formatDescriptionOut: &format) + guard let format else { return } + var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: .zero, decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: buffer, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample) + guard let sample else { return } + if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: true), CFArrayGetCount(attachments) > 0 { + let dictionary = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue(dictionary, Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(), Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) + } + if renderer.status == .failed { renderer.flush() } + renderer.enqueue(sample) + } + } +} + +// MARK: - Panels + +private struct PanelSlider: View { + let title: String + var showsTitle = true + let systemImage: String + let value: Double + let range: ClosedRange + let onBegin: () -> Void + let onChange: (Double) -> Void + let onEnd: () -> Void + + var body: some View { + HStack(spacing: 12) { + Image(systemName: systemImage) + .font(.system(size: 15, weight: .medium)) + .foregroundStyle(.secondary) + .frame(width: 22) + if showsTitle { + Text(title) + .font(.subheadline) + .frame(width: 74, alignment: .leading) + } + Slider(value: Binding(get: { value }, set: { onChange($0) }), in: range) { editing in + if editing { onBegin() } else { onEnd() } + } + .tint(.white) + } + .accessibilityElement(children: .combine) + .accessibilityLabel(title) + .accessibilityValue("\(Int(((value - range.lowerBound) / max(0.0001, range.upperBound - range.lowerBound) * 100).rounded())) percent") + .accessibilityAdjustableAction { direction in + let step = (range.upperBound - range.lowerBound) / 10 + onBegin() + onChange(min(max(range.lowerBound, value + (direction == .increment ? step : -step)), range.upperBound)) + onEnd() + } + } +} + +private struct Chip: View { + let title: String + var systemImage: String? + let selected: Bool + let action: () -> Void + + var body: some View { + Button(action: action) { + HStack(spacing: 6) { + if let systemImage { Image(systemName: systemImage).font(.system(size: 13, weight: .semibold)) } + Text(title).font(.subheadline.weight(.semibold)).lineLimit(1) + } + .padding(.horizontal, 14) + .frame(height: 34) + .foregroundStyle(selected ? Color.black : Color.white) + .background { + if selected { Capsule().fill(.white) } + } + .contentShape(Capsule()) + } + .buttonStyle(.plain) + .glassEffect(.regular.interactive(), in: Capsule()) + .accessibilityAddTraits(selected ? .isSelected : []) + } +} + +struct PresetPanel: View { + let model: EditorModel + @State private var naming: Naming? + @State private var name = "" + + private enum Naming: Equatable { + case new + case rename(StudioPreset) + } + + private var studio: StudioSession { StudioSession.shared } + private var library: StudioPresetLibrary { studio.presetLibrary } + private static let tile = CGSize(width: 74, height: 98) + + private var pausedMoment: Double? { + model.isPlaying ? nil : (model.outputTime * 10).rounded() / 10 + } + + var body: some View { + let selected = library.preset(matching: model.look) + ScrollViewReader { scroller in + ScrollView(.horizontal) { + LazyHStack(alignment: .top, spacing: 12) { + saveCard(isSaved: selected != nil) + ForEach(library.all) { preset in + card(preset, selected: preset.id == selected?.id) + } + } + .padding(.horizontal, 16) + .padding(.top, 4) + } + .scrollIndicators(.hidden) + .onAppear { + if let selected { scroller.scrollTo(selected.id, anchor: .center) } + } + } + .sensoryFeedback(.selection, trigger: selected?.id) + .onChange(of: library) { _, library in model.loadPresetThumbnails(library.all) } + // Redrawn once a paused seek settles: a scrub changes this many times, and each change cancels the last wait. + .task(id: pausedMoment) { + guard pausedMoment != nil else { return } + try? await Task.sleep(for: .milliseconds(150)) + guard !Task.isCancelled else { return } + model.loadPresetThumbnails(library.all) + } + .alert(naming == .new ? "Save Preset" : "Rename Preset", isPresented: Binding(get: { naming != nil }, set: { if !$0 { naming = nil } })) { + TextField("Name", text: $name) + .submitLabel(.done) + Button("Save") { finishNaming() } + Button("Cancel", role: .cancel) { naming = nil } + } message: { + if naming == .new { Text("Keeps this background, frame and camera layout for any recording.") } + } + } + + private func saveCard(isSaved: Bool) -> some View { + Button { + name = library.uniqueName("My Look") + naming = .new + } label: { + VStack(spacing: 7) { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) + .foregroundStyle(.white.opacity(0.35)) + .frame(width: Self.tile.width, height: Self.tile.height) + .overlay { + Image(systemName: "plus") + .font(.system(size: 22, weight: .medium)) + .foregroundStyle(.white.opacity(0.85)) + } + Text("Save") + .font(.caption.weight(.medium)) + .foregroundStyle(.white.opacity(0.75)) + } + .contentShape(.rect) + } + .buttonStyle(.plain) + .opacity(isSaved ? 0.4 : 1) + .disabled(isSaved) + .accessibilityLabel("Save as Preset") + .accessibilityHint(isSaved ? "This look is already a preset." : "") + } + + private func card(_ preset: StudioPreset, selected: Bool) -> some View { + let isDefault = library.defaultPreset.id == preset.id + return Button { + model.apply(preset) + } label: { + VStack(spacing: 7) { + ZStack { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .fill(.white.opacity(0.06)) + PresetPicture(preset: preset, image: model.presetThumbnails[preset.look], tile: Self.tile) + } + .frame(width: Self.tile.width, height: Self.tile.height) + .overlay { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(selected ? Color.white : Color.white.opacity(0.1), lineWidth: selected ? 2.5 : 1) + } + .scaleEffect(selected ? 1.03 : 1) + .animation(.spring(duration: 0.22), value: selected) + HStack(spacing: 3) { + if isDefault { + Image(systemName: "pin.fill").font(.system(size: 8, weight: .semibold)) + } + Text(preset.name).lineLimit(1) + } + .font(.caption.weight(.medium)) + .foregroundStyle(selected ? Color.white : Color.white.opacity(0.65)) + .frame(maxWidth: Self.tile.width + 8) + } + .contentShape(.rect) + } + .buttonStyle(.plain) + .id(preset.id) + .contextMenu { + if !isDefault { + Button("Use for New Recordings", systemImage: "pin") { studio.makeDefault(preset.id) } + } + if !preset.isBuiltIn { + if preset.look != model.look { + Button("Update to Current Look", systemImage: "arrow.triangle.2.circlepath") { + studio.updatePreset(preset.id, look: model.look, from: model.workDirectory) + } + } + Button("Rename", systemImage: "pencil") { + name = preset.name + naming = .rename(preset) + } + Button("Delete", systemImage: "trash", role: .destructive) { studio.deletePreset(preset.id) } + } + } + .accessibilityLabel(preset.name) + .accessibilityValue(isDefault ? "Used for new recordings" : "") + .accessibilityIdentifier("editor-preset") + .accessibilityAddTraits(selected ? .isSelected : []) + } + + private func finishNaming() { + switch naming { + case .new: + studio.savePreset(named: name, look: model.look, from: model.workDirectory) + case let .rename(preset): + studio.renamePreset(preset.id, to: name) + case nil: + break + } + naming = nil + } +} + +/// The same blooms as the renderer draws, in SwiftUI, for swatches and placeholders. +struct GlowSwatch: View { + let glow: StudioGlow + + var body: some View { + GeometryReader { proxy in + Canvas { context, size in + let frame = Path(CGRect(origin: .zero, size: size)) + let long = max(size.width, size.height) + context.fill(frame, with: .color(swiftUI(glow.base))) + for bloom in glow.blooms { + let color = swiftUI(bloom.color) + context.fill(frame, with: .radialGradient(Gradient(colors: [color.opacity(0.95), color.opacity(0)]), center: CGPoint(x: bloom.x * size.width, y: bloom.y * size.height), startRadius: 0, endRadius: bloom.radius * long)) + } + } + .blur(radius: StudioGlow.softness * min(proxy.size.width, proxy.size.height) * 2, opaque: true) + } + } + + private func swiftUI(_ color: StudioColor) -> Color { + let c = color.components + return Color(red: c.red, green: c.green, blue: c.blue) + } +} + +/// The preset drawn on this video's frame once it's ready, its background until then. +private struct PresetPicture: View { + let preset: StudioPreset + let image: UIImage? + let tile: CGSize + + var body: some View { + let inner = CGSize(width: tile.width - 10, height: tile.height - 10) + let shape = fitted(inner) + ZStack { + placeholder + .frame(width: shape.width, height: shape.height) + if let image { + Image(uiImage: image) + .resizable() + .interpolation(.high) + .frame(width: shape.width, height: shape.height) + .transition(.opacity) + } + } + .clipShape(RoundedRectangle(cornerRadius: 7, style: .continuous)) + .shadow(color: .black.opacity(0.35), radius: 4, y: 2) + .animation(.easeOut(duration: 0.2), value: image != nil) + } + + private func fitted(_ box: CGSize) -> CGSize { + let ratio = image.map { $0.size.width / max(1, $0.size.height) } ?? preset.look.aspect.ratio.map { CGFloat($0) } ?? 9.0 / 16 + return box.width / box.height > ratio + ? CGSize(width: (box.height * ratio).rounded(), height: box.height) + : CGSize(width: box.width, height: (box.width / ratio).rounded()) + } + + @ViewBuilder + private var placeholder: some View { + switch preset.look.background { + case let .wallpaper(name): + if let thumbnail = StudioAssets.wallpaperThumbnail(name) { + Image(uiImage: thumbnail).resizable().scaledToFill() + } else { + Color.white.opacity(0.08) + } + case .color, .gradient, .glow: + BackgroundPanel.fill(preset.look.background) + default: + Color.white.opacity(0.08) + } + } +} + +struct BackgroundPanel: View { + let model: EditorModel + @State private var picked: PhotosPickerItem? + + var body: some View { + ScrollViewReader { scroller in + ScrollView(.horizontal) { + LazyHStack(spacing: 10) { + swatch(.none, label: "None") { + Image(systemName: "nosign") + .font(.system(size: 20, weight: .medium)) + .foregroundStyle(.white.opacity(0.7)) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .background(Color.white.opacity(0.08)) + } + let photo = photo + let isPhoto = isPhoto + let side = Self.side + PhotosPicker(selection: $picked, matching: .images, photoLibrary: .shared()) { + PhotoSwatch(image: photo, selected: isPhoto, side: side) + } + .accessibilityLabel("Choose a photo") + divider + ForEach(Array(StudioBackground.glows.enumerated()), id: \.offset) { _, background in + swatch(background, label: "Glow") { Self.fill(background) } + } + divider + ForEach(StudioAssets.wallpapers, id: \.self) { name in + swatch(.wallpaper(name), label: "Wallpaper") { + if let image = StudioAssets.wallpaperThumbnail(name) { + Image(uiImage: image).resizable().scaledToFill() + } + } + } + divider + ForEach(Array(StudioBackground.gradients.enumerated()), id: \.offset) { _, background in + swatch(background, label: "Gradient") { Self.fill(background) } + } + divider + ForEach(Array(StudioBackground.colors.enumerated()), id: \.offset) { _, background in + swatch(background, label: "Colour") { Self.fill(background) } + } + } + .padding(.horizontal, 16) + .frame(height: Self.side + 8) + } + .scrollIndicators(.hidden) + .onAppear { + if let id = selectedID { scroller.scrollTo(id, anchor: .center) } + } + } + .padding(.top, 2) + .onChange(of: picked) { _, item in + guard let item else { return } + Task { await usePhoto(item) } + } + } + + private static let side: CGFloat = 52 + + private var photo: UIImage? { + guard case let .image(file) = model.look.background else { return nil } + return UIImage(contentsOfFile: model.workDirectory.appending(path: file).path) + } + + private var isPhoto: Bool { + if case .image = model.look.background { true } else { false } + } + + private var selectedID: String? { String(describing: model.look.background) } + + private var divider: some View { + Capsule().fill(.white.opacity(0.15)).frame(width: 1, height: 30) + } + + private func ring(selected: Bool) -> some View { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(selected ? Color.white : Color.white.opacity(0.12), lineWidth: selected ? 2.5 : 1) + } + + private func swatch(_ background: StudioBackground, label: String, @ViewBuilder content: () -> some View) -> some View { + let selected = model.look.background == background + return Button { + model.setLook { look in + look.background = background + // Picking a background for an untouched video frames it, or the background would never show. + if !background.isNone, look.padding == 0, look.aspect == .auto { + look.padding = StudioLook.studioDefault.padding + look.rounding = StudioLook.studioDefault.rounding + look.shadow = StudioLook.studioDefault.shadow + } + } + } label: { + content() + .frame(width: Self.side, height: Self.side) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay(ring(selected: selected)) + .scaleEffect(selected ? 1.04 : 1) + .animation(.spring(duration: 0.2), value: selected) + } + .buttonStyle(.plain) + .id(String(describing: background)) + .accessibilityLabel(label) + .accessibilityAddTraits(selected ? .isSelected : []) + } + + static func fill(_ background: StudioBackground) -> some View { + Group { + switch background { + case let .color(color): + let c = color.components + Color(red: c.red, green: c.green, blue: c.blue) + case let .gradient(from, to, _): + let a = from.components + let b = to.components + LinearGradient(colors: [Color(red: a.red, green: a.green, blue: a.blue), Color(red: b.red, green: b.green, blue: b.blue)], startPoint: .topLeading, endPoint: .bottomTrailing) + case let .glow(id): + if let glow = StudioGlow.named(id) { GlowSwatch(glow: glow) } else { Color.clear } + default: + Color.clear + } + } + } + + /// Kept next to the edit at a size that covers any frame, so the original photo is never needed again. + private func usePhoto(_ item: PhotosPickerItem) async { + defer { picked = nil } + guard let data = try? await item.loadTransferable(type: Data.self) else { return } + let directory = model.workDirectory + let name = "background-\(UUID().uuidString.prefix(8)).jpg" + let saved: Bool = await Task.detached(priority: .userInitiated) { + guard let source = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateThumbnailAtIndex(source, 0, [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceThumbnailMaxPixelSize: 2400, + kCGImageSourceCreateThumbnailWithTransform: true, + ] as CFDictionary), + let jpeg = UIImage(cgImage: image).jpegData(compressionQuality: 0.85) + else { return false } + return (try? jpeg.write(to: directory.appending(path: name), options: .atomic)) != nil + }.value + guard saved else { return } + model.setLook { look in + look.background = .image(name) + if look.padding == 0, look.aspect == .auto { + look.padding = StudioLook.studioDefault.padding + look.rounding = StudioLook.studioDefault.rounding + look.shadow = StudioLook.studioDefault.shadow + } + } + } +} + +private struct PhotoSwatch: View { + let image: UIImage? + let selected: Bool + let side: CGFloat + + var body: some View { + ZStack { + if let image { + Image(uiImage: image).resizable().scaledToFill() + } else { + Color.white.opacity(0.08) + Image(systemName: "photo.on.rectangle") + .font(.system(size: 19, weight: .medium)) + .foregroundStyle(.white.opacity(0.75)) + } + } + .frame(width: side, height: side) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(selected ? Color.white : Color.white.opacity(0.12), lineWidth: selected ? 2.5 : 1) + ) + } +} + +struct FramePanel: View { + let model: EditorModel + + var body: some View { + VStack(spacing: 10) { + ScrollView(.horizontal) { + HStack(spacing: 8) { + ForEach(StudioAspect.allCases) { aspect in + Chip(title: aspect.title, systemImage: Self.symbol(aspect), selected: model.look.aspect == aspect) { + model.setLook { $0.aspect = aspect } + } + } + } + .padding(.horizontal, 16) + } + .scrollIndicators(.hidden) + + VStack(spacing: 2) { + slider("Padding", "square.dashed", \.padding, StudioLook.paddingRange) + slider("Corners", "rectangle.roundedtop", \.rounding, StudioLook.roundingRange) + slider("Shadow", "shadow", \.shadow, 0...1) + } + .padding(.horizontal, 20) + } + .padding(.top, 6) + } + + private func slider(_ title: String, _ symbol: String, _ path: WritableKeyPath, _ range: ClosedRange) -> some View { + PanelSlider(title: title, systemImage: symbol, value: model.look[keyPath: path], range: range) { + model.beginLookGesture() + } onChange: { value in + model.setLook { look in + look[keyPath: path] = value + // Padding, corners and shadow only show against a background. + if look.background.isNone, value > 0 { look.background = StudioBackground.gradients[0] } + } + } onEnd: { + model.endLookGesture() + } + } + + private static func symbol(_ aspect: StudioAspect) -> String { + switch aspect { + case .auto: "arrow.up.left.and.arrow.down.right" + case .portrait: "rectangle.portrait" + case .fourFive: "rectangle.portrait" + case .square: "square" + case .landscape: "rectangle" + } + } +} + +struct CameraPanel: View { + let model: EditorModel + + var body: some View { + VStack(spacing: 12) { + HStack(spacing: 8) { + ForEach(StudioCameraLayout.allCases) { layout in + Chip(title: layout.title, selected: model.look.camera.layout == layout) { + model.setLook { $0.camera.layout = layout } + } + } + } + if model.look.camera.layout == .bubble { + HStack(spacing: 14) { + ForEach(StudioCameraShape.allCases) { shape in + Button { + model.setLook { $0.camera.shape = shape } + } label: { + Image(systemName: Self.symbol(shape)) + .font(.system(size: 18, weight: .medium)) + .frame(width: 40, height: 34) + .foregroundStyle(model.look.camera.shape == shape ? .white : .white.opacity(0.45)) + .contentShape(.rect) + } + .buttonStyle(.plain) + .accessibilityLabel(shape.rawValue.capitalized) + .accessibilityAddTraits(model.look.camera.shape == shape ? .isSelected : []) + } + Capsule().fill(.white.opacity(0.15)).frame(width: 1, height: 24) + PanelSlider(title: "Size", showsTitle: false, systemImage: "arrow.up.left.and.arrow.down.right", value: model.look.camera.size, range: StudioCameraLook.sizeRange) { + model.beginLookGesture() + } onChange: { value in + model.setLook { $0.camera.size = value } + } onEnd: { + model.endLookGesture() + } + } + .padding(.horizontal, 20) + } + } + .padding(.top, 8) + } + + private static func symbol(_ shape: StudioCameraShape) -> String { + switch shape { + case .circle: "circle" + case .rounded: "app" + case .square: "square" + } + } +} + +struct MusicPanel: View { + let model: EditorModel + @State private var importing = false + + var body: some View { + VStack(spacing: 10) { + ScrollView(.horizontal) { + HStack(spacing: 8) { + Chip(title: "None", systemImage: "speaker.slash", selected: model.music == nil) { + Task { await model.setMusic(nil) } + } + Chip(title: "Import", systemImage: "folder", selected: isImported) { + importing = true + } + ForEach(StudioAssets.music) { track in + Chip(title: track.title, systemImage: "music.note", selected: model.music?.source == .builtIn(track.id)) { + Task { await model.setMusic(StudioMusic(source: .builtIn(track.id), title: track.title, volume: model.music?.volume ?? 0.35)) } + } + } + } + .padding(.horizontal, 16) + } + .scrollIndicators(.hidden) + + if model.music != nil { + Waveform(peaks: model.musicWaveform) + .frame(height: 16) + .padding(.horizontal, 22) + .accessibilityHidden(true) + } + + VStack(spacing: 2) { + if let music = model.music { + PanelSlider(title: "Music", systemImage: "music.note", value: music.volume, range: 0...1) { + model.beginAudioGesture() + } onChange: { value in + model.setVolumes(music: value, voice: nil) + } onEnd: { + model.endLookGesture() + } + } + PanelSlider(title: "Voice", systemImage: "waveform", value: model.voiceVolume, range: 0...1.5) { + model.beginAudioGesture() + } onChange: { value in + model.setVolumes(music: nil, voice: value) + } onEnd: { + model.endLookGesture() + } + } + .padding(.horizontal, 20) + } + .padding(.top, 6) + .fileImporter(isPresented: $importing, allowedContentTypes: [.audio]) { result in + guard case let .success(url) = result else { return } + Task { await importMusic(url) } + } + } + + private var isImported: Bool { + if case .file = model.music?.source { true } else { false } + } + + private func importMusic(_ url: URL) async { + let accessing = url.startAccessingSecurityScopedResource() + defer { if accessing { url.stopAccessingSecurityScopedResource() } } + let name = "music-\(UUID().uuidString.prefix(8)).\(url.pathExtension.isEmpty ? "m4a" : url.pathExtension)" + let destination = model.workDirectory.appending(path: name) + do { + try FileManager.default.copyItem(at: url, to: destination) + } catch { + return + } + await model.setMusic(StudioMusic(source: .file(name), title: url.deletingPathExtension().lastPathComponent, volume: model.music?.volume ?? 0.35)) + } +} + +private struct Waveform: View { + let peaks: [Float] + + var body: some View { + Canvas { context, size in + guard !peaks.isEmpty else { return } + let step = size.width / CGFloat(peaks.count) + var path = Path() + for (index, peak) in peaks.enumerated() { + let height = max(1.5, CGFloat(peak) * size.height) + path.addRoundedRect(in: CGRect(x: CGFloat(index) * step, y: (size.height - height) / 2, width: max(1, step * 0.55), height: height), cornerSize: CGSize(width: 0.75, height: 0.75)) + } + context.fill(path, with: .color(.white.opacity(0.45))) + } + } +} + +// MARK: - Export + +struct ExportProgressCard: View { + let progress: Double + let title: String + let onCancel: () -> Void + + var body: some View { + ZStack { + Color.black.opacity(0.45).ignoresSafeArea() + VStack(spacing: 14) { + Text(title) + .font(.headline) + ProgressView(value: progress) + .progressViewStyle(.linear) + .tint(.white) + .frame(width: 200) + Text("\(Int((progress * 100).rounded()))%") + .font(.subheadline.monospacedDigit()) + .foregroundStyle(.secondary) + .contentTransition(.numericText()) + Button("Cancel", role: .cancel, action: onCancel) + .buttonStyle(.glass) + } + .padding(24) + .glassEffect(.regular, in: RoundedRectangle(cornerRadius: 28, style: .continuous)) + } + .accessibilityElement(children: .contain) + .accessibilityIdentifier("editor-export-progress") + } +} + +/// Renders on the phone while Cap is in front (iOS refuses GPU work in the background), then hands the file to the +/// background upload or Photos. +@MainActor +@Observable +final class EditorExport { + private(set) var isRunning = false + private(set) var progress: Double = 0 + private(set) var title = "Rendering" + var errorMessage: String? + @ObservationIgnored private var exporter: StudioExporter? + @ObservationIgnored private weak var rendering: EditorModel? + @ObservationIgnored private var cancelRequested = false + + func cancel() { + cancelRequested = true + exporter?.cancel() + rendering?.stopWaitingForMedia() + } + + private func render(_ model: EditorModel, title: String) async -> StudioExporter.Result? { + guard !isRunning else { return nil } + isRunning = true + progress = 0 + self.title = title + cancelRequested = false + rendering = model + defer { + isRunning = false + exporter = nil + rendering = nil + } + model.pauseForExport() + let exporter = StudioExporter() + self.exporter = exporter + // iOS refuses GPU work from the background, so leaving Cap would fail the render part way. + let leftScreen = LeftScreen() + let observer = NotificationCenter.default.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: .main) { _ in + leftScreen.value = true + exporter.cancel() + } + defer { NotificationCenter.default.removeObserver(observer) } + do { + await model.waitForMedia() + guard !cancelRequested else { return nil } + let built = try model.exportComposition() + let url = try UploadCenter.stagingURL(extension: "mp4") + UIApplication.shared.isIdleTimerDisabled = true + defer { UIApplication.shared.isIdleTimerDisabled = false } + let result = try await exporter.export(built, to: url) { [weak self] value in + Task { @MainActor in + guard let self, value > self.progress + 0.01 || value >= 1 else { return } + self.progress = value + } + } + model.metrics.record("exportSeconds", result.seconds) + model.metrics.record("exportRealtime", result.duration / max(result.seconds, 0.001)) + ProcessStats.note("Studio export \(Int(result.size.width))x\(Int(result.size.height)) \(String(format: "%.1f", result.duration)) s in \(String(format: "%.2f", result.seconds)) s, \(result.bytes / 1024) KB") + return result + } catch StudioExporter.Failure.cancelled { + if leftScreen.value { errorMessage = "Rendering stopped when Cap left the screen. Keep Cap open until it finishes." } + return nil + } catch { + errorMessage = (error as? LocalizedError)?.errorDescription ?? "Cap couldn't render this video. Try again." + return nil + } + } + + /// Uploads the rendered video as a new Cap through the usual background upload. + func share(_ model: EditorModel, session: SessionModel) async -> Bool { + // Cap holds free Caps to five minutes however they arrive. Said before rendering rather than after it. + if !session.isPro, model.timeline.outputDuration > RecordingLimits.freeDuration + RecordingLimits.freeTolerance { + errorMessage = "Caps on the free plan can be up to 5 minutes. Trim this one to 5 minutes, save it to Photos, or upgrade to Pro." + Usage.track("editor_share_over_free_limit", ["studio": .bool(model.isStudio), "seconds": .int(Int(model.timeline.outputDuration.rounded()))]) + return false + } + guard let result = await render(model, title: "Rendering") else { return false } + let name = model.title + let url = result.url + session.uploads.enqueue(displayName: name, folderId: nil) { + try await UploadCenter.probe(url, fileName: "\(name).mp4", contentType: "video/mp4") + } + Usage.track("editor_rendered", ["studio": .bool(model.isStudio), "seconds": .int(Int(result.duration.rounded()))]) + session.toasts.show(model.isStudio ? "Uploading your video. Its link is ready when it finishes." : "Uploading as a new Cap. Your original stays as it is.", systemImage: "arrow.up.circle.fill") + return true + } + + func saveToPhotos(_ model: EditorModel) async -> Bool { + let status = await PHPhotoLibrary.requestAuthorization(for: .addOnly) + guard status == .authorized || status == .limited else { + errorMessage = "Allow Cap to add to Photos in Settings to save videos there." + return false + } + guard let result = await render(model, title: "Saving to Photos") else { return false } + defer { try? FileManager.default.removeItem(at: result.url) } + do { + try await PhotoLibrary.addVideo(at: result.url) + Usage.track("editor_saved_to_photos", ["studio": .bool(model.isStudio)]) + return true + } catch { + errorMessage = "Cap couldn't save the video to Photos. Try again." + return false + } + } +} + +nonisolated private final class LeftScreen: @unchecked Sendable { + var value = false +} diff --git a/apps/mobile-ios/Cap/Features/Edit/EditorView.swift b/apps/mobile-ios/Cap/Features/Edit/EditorView.swift index 8fec37d4f3a..6a48efa11c3 100644 --- a/apps/mobile-ios/Cap/Features/Edit/EditorView.swift +++ b/apps/mobile-ios/Cap/Features/Edit/EditorView.swift @@ -9,10 +9,13 @@ struct EditorView: View { @State private var confirmingDiscard = false @State private var confirmingRestore = false + @State private var tool: EditorTool = .trim + @State private var export = EditorExport() + @Environment(SessionModel.self) private var session @Environment(\.horizontalSizeClass) private var horizontalSizeClass @Environment(\.verticalSizeClass) private var verticalSizeClass - enum EditorOutcome { case cancelled, saved, restored } + enum EditorOutcome { case cancelled, saved, restored, rendered } var body: some View { NavigationStack { @@ -33,24 +36,15 @@ struct EditorView: View { case .ready: content } - } - .navigationTitle("Edit") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .cancellationAction) { - Button("Cancel") { - if model.hasChanges { confirmingDiscard = true } else { finish(.cancelled) } - } - } - ToolbarItem(placement: .confirmationAction) { - Button { - Task { if await model.save() { finish(.saved) } } - } label: { - if model.isSaving { ProgressView() } else { Text("Save") } - } - .disabled(!model.hasChanges || model.isSaving) + if export.isRunning { + ExportProgressCard(progress: export.progress, title: export.title) { export.cancel() } + .transition(.opacity.combined(with: .scale(scale: 0.96))) } } + .animation(.easeOut(duration: 0.2), value: export.isRunning) + .navigationTitle(model.isStudio ? "Studio" : "Edit") + .navigationBarTitleDisplayMode(.inline) + .toolbar { toolbar } } .preferredColorScheme(.dark) .task { @@ -64,7 +58,10 @@ struct EditorView: View { #endif await model.load() } - .onDisappear { model.close() } + .onDisappear { + export.cancel() + model.close() + } .confirmationDialog("Discard your changes?", isPresented: $confirmingDiscard, titleVisibility: .visible) { Button("Discard changes", role: .destructive) { finish(.cancelled) } Button("Keep editing", role: .cancel) {} @@ -77,12 +74,49 @@ struct EditorView: View { } message: { Text("Every edit made to this Cap is undone. Comments stay where they are.") } - .alert("Couldn't save the edit", isPresented: Binding(get: { model.errorMessage != nil }, set: { if !$0 { model.errorMessage = nil } })) { + .alert("Couldn't save the edit", isPresented: Binding(get: { model.errorMessage != nil || export.errorMessage != nil }, set: { if !$0 { model.errorMessage = nil; export.errorMessage = nil } })) { Button("OK", role: .cancel) {} } message: { - Text(model.errorMessage ?? "") + Text(model.errorMessage ?? export.errorMessage ?? "") + } + .interactiveDismissDisabled(model.hasChanges || export.isRunning) + } + + @ToolbarContentBuilder + private var toolbar: some ToolbarContent { + ToolbarItem(placement: .cancellationAction) { + if model.isStudio { + Button("Done") { finish(.cancelled) } + .disabled(export.isRunning) + } else { + Button("Cancel") { + if model.hasChanges { confirmingDiscard = true } else { finish(.cancelled) } + } + .disabled(export.isRunning) + } + } + ToolbarItem(placement: .confirmationAction) { + if model.isStudio { + Menu { + Button("Share as a Cap", systemImage: "link") { share() } + Button("Save to Photos", systemImage: "square.and.arrow.down") { saveToPhotos() } + } label: { + Text("Share") + } primaryAction: { + share() + } + .disabled(model.phase != .ready || export.isRunning) + .accessibilityIdentifier("editor-share") + } else { + Button { + if model.needsRender { share() } else { Task { if await model.save() { finish(.saved) } } } + } label: { + // A styled edit renders on the phone and uploads as a new Cap; the original stays as it is. + if model.isSaving { ProgressView() } else { Text(model.needsRender ? "Save as New" : "Save") } + } + .disabled(!model.hasChanges || model.isSaving || export.isRunning) + } } - .interactiveDismissDisabled(model.hasChanges) } private func finish(_ outcome: EditorOutcome) { @@ -90,28 +124,53 @@ struct EditorView: View { onFinished(outcome) } + private func share() { + Task { + if await export.share(model, session: session) { finish(.rendered) } + } + } + + private func saveToPhotos() { + Task { + if await export.saveToPhotos(model) { + session.toasts.show("Saved to Photos", systemImage: "checkmark.circle.fill") + } + } + } + + private var tools: [EditorTool] { + EditorTool.allCases.filter { tool in + switch tool { + case .camera: model.hasCamera && model.hasScreen + default: true + } + } + } + private var content: some View { - VStack(spacing: 18) { - ZStack { - ForEach(model.players.indices, id: \.self) { index in - EditorPlayerSurface(player: model.players[index]) { model.metrics.mark("firstFrame") } - .zIndex(index == model.activePlayer ? 1 : 0) + VStack(spacing: 14) { + EditorPreview(model: model, outlinesCamera: tool == .camera) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .padding(.horizontal, model.renderSize.width > model.renderSize.height ? 0 : 12) + + HStack { + if model.needsRender { + let size = model.exportSize + Text(verbatim: "\(Int(size.width))×\(Int(size.height))") + .font(.caption.monospacedDigit()) + .foregroundStyle(.tertiary) + .accessibilityLabel("Renders at \(Int(size.width)) by \(Int(size.height))") } - EditorPlayerSurface(player: model.sourcePlayer) - .zIndex(model.showsSource ? 2 : -1) + Spacer() + Text("\(CapFormat.timestamp(model.outputTime)) / \(CapFormat.timestamp(model.displayedTimeline.outputDuration))") + .font(.subheadline.monospacedDigit()) + .foregroundStyle(.secondary) + .accessibilityLabel("Edited length \(CapFormat.timestamp(model.displayedTimeline.outputDuration))") } - .frame(maxWidth: .infinity, maxHeight: .infinity) - .accessibilityHidden(true) - - Text("\(CapFormat.timestamp(model.outputTime)) / \(CapFormat.timestamp(model.displayedTimeline.outputDuration))") - .font(.subheadline.monospacedDigit()) - .foregroundStyle(.secondary) - .accessibilityLabel("Edited length \(CapFormat.timestamp(model.displayedTimeline.outputDuration))") - .opacity(model.isTrimming ? 0 : 1) - .frame(maxWidth: .infinity, alignment: .trailing) - .padding(.horizontal, 16) + .opacity(model.isTrimming ? 0 : 1) + .padding(.horizontal, 16) - if wide { + if wide, tool == .trim { HStack(spacing: 10) { historyTools EditorTimeline(model: model) @@ -119,21 +178,48 @@ struct EditorView: View { clipTools } .padding(.horizontal, 16) - .padding(.bottom, 8) } else { EditorTimeline(model: model) .padding(.horizontal, 15) + } + + panel + .frame(height: tool == .trim && wide ? 0 : EditorPanels.height(for: tool), alignment: .top) + .frame(maxWidth: 640) + .clipped() + + if tools.count > 1 { + EditorToolBar(tools: tools, selection: $tool) + .padding(.bottom, 4) + } + } + .sensoryFeedback(.impact(weight: .medium), trigger: model.timeline.clips.count) + } + @ViewBuilder + private var panel: some View { + switch tool { + case .trim: + if !wide { HStack(spacing: 10) { historyTools Spacer() clipTools } .padding(.horizontal, 16) - .padding(.bottom, 8) + .padding(.top, 4) } + case .presets: + PresetPanel(model: model) + case .background: + BackgroundPanel(model: model) + case .frame: + FramePanel(model: model) + case .camera: + CameraPanel(model: model) + case .music: + MusicPanel(model: model) } - .sensoryFeedback(.impact(weight: .medium), trigger: model.timeline.clips.count) } private var wide: Bool { @@ -507,7 +593,7 @@ private struct FilmStrip: View { let size: CGSize var body: some View { - let tile = max(12, size.height * model.videoAspect) + let tile = max(12, size.height * model.sourceAspect) let scale = track.pointsPerSecond let span = scale > 0 ? Double(tile) / scale : 0 let low = max(0, track.time(for: -Double(inset))) diff --git a/apps/mobile-ios/Cap/Features/Record/CameraBubble.swift b/apps/mobile-ios/Cap/Features/Record/CameraBubble.swift index bb082570363..eecb60d620c 100644 --- a/apps/mobile-ios/Cap/Features/Record/CameraBubble.swift +++ b/apps/mobile-ios/Cap/Features/Record/CameraBubble.swift @@ -53,7 +53,10 @@ final class CameraBubble: NSObject { } } - func prepare() async { + /// Studio Mode records the bubble's camera as its own track, so it asks the camera for 720p rather than the + /// small picture the bubble alone needs. + func prepare(studio: Bool = false) async { + capture.usePreset(studio ? .hd1280x720 : CameraCapture.Configuration.bubble.preset) guard state != .ready else { return } switch AVCaptureDevice.authorizationStatus(for: .video) { case .authorized: @@ -123,6 +126,24 @@ final class CameraBubble: NSObject { if !isFloating { stop() } } + func beginStudioRecording(_ writer: StudioTrackWriter) { + capture.beginRecording(writer) + } + + /// Closes the camera's file. Nil if the camera never recorded a frame, for example when iOS kept it off while + /// another app was in front. + func endStudioRecording(_ writer: StudioTrackWriter) async -> StudioTrackInfo? { + let result = await withCheckedContinuation { continuation in + capture.endRecording { continuation.resume(returning: $0) } + } + // A file that stopped early (a full disk) still holds what it wrote before. + guard var info = writer.info else { return nil } + if let duration = try? result.get(), duration > 0 { info.duration = duration } + return info.duration > 0 ? info : nil + } + + var captureClock: CMClock? { capture.captureClock } + nonisolated private static func show(_ buffer: CMSampleBuffer, on layer: AVSampleBufferDisplayLayer) { // Live frames: show each as it arrives rather than on a playback clock. if let attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, createIfNecessary: true), CFArrayGetCount(attachments) > 0 { diff --git a/apps/mobile-ios/Cap/Features/Record/CameraCapture.swift b/apps/mobile-ios/Cap/Features/Record/CameraCapture.swift index 51e8b5f032d..d2dfe431ea0 100644 --- a/apps/mobile-ios/Cap/Features/Record/CameraCapture.swift +++ b/apps/mobile-ios/Cap/Features/Record/CameraCapture.swift @@ -46,6 +46,16 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV static let bubble = Configuration(preset: .vga640x480, mirrored: true, multitasking: true) } + func usePreset(_ preset: AVCaptureSession.Preset) { + sessionQueue.async { [self] in + presetOverride = preset + guard configured, session.sessionPreset != preset, session.canSetSessionPreset(preset) else { return } + session.beginConfiguration() + session.sessionPreset = preset + session.commitConfiguration() + } + } + private static let log = Logger(subsystem: "so.cap.mobile", category: "recorder") let session = AVCaptureSession() @@ -58,6 +68,7 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV // sessionQueue private var videoInput: AVCaptureDeviceInput? private var configured = false + private var presetOverride: AVCaptureSession.Preset? private var rotationAngle: CGFloat = 90 private var observers: [NSObjectProtocol] = [] #if DEBUG @@ -65,7 +76,7 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV #endif // outputQueue - private var writer: SegmentWriter? + private var writer: (any RecordingSink)? var onEvent: (@Sendable (Event) -> Void)? var onFrame: (@Sendable (CMSampleBuffer) -> Void)? @@ -124,8 +135,9 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV private func configure(position: AVCaptureDevice.Position, microphone: Bool) throws { session.beginConfiguration() defer { session.commitConfiguration() } - if session.canSetSessionPreset(configuration.preset) { - session.sessionPreset = configuration.preset + let preset = presetOverride ?? configuration.preset + if session.canSetSessionPreset(preset) { + session.sessionPreset = preset } else { session.sessionPreset = .high } @@ -235,7 +247,7 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV // MARK: Recording - func beginRecording(_ writer: SegmentWriter) { + func beginRecording(_ writer: any RecordingSink) { outputQueue.async { self.writer = writer } } @@ -246,7 +258,7 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV return } self.writer = nil - writer.finish(completion) + writer.finish(at: nil, completion) } } @@ -257,6 +269,15 @@ nonisolated final class CameraCapture: NSObject, @unchecked Sendable, AVCaptureV } } + /// The clock frames are timed on. Studio converts it to the host clock, which the screen recording uses. + var captureClock: CMClock? { + #if DEBUG + // Played-in files are timed on the host clock already. + if DebugFileSource.configuredFile != nil { return nil } + #endif + return session.synchronizationClock + } + func recordedDuration(_ completion: @escaping @Sendable (Double) -> Void) { outputQueue.async { [self] in completion(writer?.duration ?? 0) } } diff --git a/apps/mobile-ios/Cap/Features/Record/RecordModel.swift b/apps/mobile-ios/Cap/Features/Record/RecordModel.swift index db112dd64d1..53a513deccd 100644 --- a/apps/mobile-ios/Cap/Features/Record/RecordModel.swift +++ b/apps/mobile-ios/Cap/Features/Record/RecordModel.swift @@ -34,6 +34,8 @@ final class RecordModel { @ObservationIgnored private var limitTask: Task? @ObservationIgnored private var stopping = false @ObservationIgnored private var wantsCamera = true + var captureMode: CaptureMode = .instant + @ObservationIgnored private var studio: (writer: StudioTrackWriter, projectId: String, store: StudioStore)? @ObservationIgnored private var rotationAngle: CGFloat = 90 init(session: SessionModel, folderId: String?) { @@ -54,7 +56,7 @@ final class RecordModel { return false } - var limit: TimeInterval? { session.isPro ? nil : RecordingLimits.freeDuration } + var limit: TimeInterval? { session.isPro || captureMode == .studio ? nil : RecordingLimits.freeDuration } var upgradeURL: URL { session.services.environment.upgradeURL } // MARK: Camera @@ -143,6 +145,10 @@ final class RecordModel { func record() { guard phase == .ready else { return } + if captureMode == .studio { + recordStudio() + return + } let recordings = session.services.recordings let format = CameraCapture.format(rotationAngle: rotationAngle) let job = RecordingJob( @@ -192,6 +198,10 @@ final class RecordModel { } func stop() async { + if studio != nil { + await stopStudio() + return + } guard isRecording, !stopping, let jobId else { return } stopping = true limitTask?.cancel() @@ -234,6 +244,15 @@ final class RecordModel { } func discard() { + if let studio { + self.studio = nil + Usage.track("recording_discarded", ["mode": "studio", "reason": "cancelled"]) + UIApplication.shared.isIdleTimerDisabled = false + capture.cancelRecording() + studio.store.delete(studio.projectId) + becomeReady() + return + } guard isRecording || phase == .finishing, let jobId else { return } Usage.track("recording_discarded", ["mode": "camera", "reason": "cancelled"]) limitTask?.cancel() @@ -258,6 +277,79 @@ final class RecordModel { } } + // MARK: Studio + + private func recordStudio() { + let projectId = UUID().uuidString.lowercased() + let store = StudioSession.shared.store + guard store.hasRoomToRecord() else { + session.toasts.show("Free up at least 1 GB on this iPhone to record in Studio Mode.", systemImage: "externaldrive.badge.exclamationmark", style: .info) + return + } + let writer: StudioTrackWriter + do { + let directory = store.directory(for: projectId) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + writer = try StudioTrackWriter(url: directory.appending(path: "camera.mp4"), settings: .init(kind: .camera, audio: hasMicrophone), clock: capture.captureClock) { started in + // Noted as soon as picture or sound starts, so a recording cut short by a crash can still be recovered. + var pending = started + pending.duration = 0 + pending.audio?.duration = 0 + try? pending.save(as: "camera", in: directory) + } + } catch { + session.toasts.show(error: error) + return + } + studio = (writer, projectId, store) + recordingRotationAngle = rotationAngle + stopping = false + RecordingCoordinator.shared.activeModel = self + capture.beginRecording(writer) + phase = .recording(startedAt: .now) + Usage.track("recording_started", ["mode": "studio_camera", "microphone": .bool(hasMicrophone)]) + UIApplication.shared.isIdleTimerDisabled = true + UIImpactFeedbackGenerator(style: .medium).impactOccurred() + // A file that can't take more (a full disk) ends the recording, keeping what was written. + limitTask = Task { [weak self] in + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1)) + guard let self, let studio = self.studio else { return } + if studio.writer.failed { + self.session.toasts.show("Recording stopped because this iPhone is out of space. What was recorded is kept.", systemImage: "externaldrive.badge.exclamationmark", style: .info) + await self.stopStudio() + return + } + } + } + } + + private func stopStudio() async { + guard let studio, isRecording, !stopping else { return } + stopping = true + limitTask?.cancel() + phase = .finishing + UIApplication.shared.isIdleTimerDisabled = false + UIImpactFeedbackGenerator(style: .light).impactOccurred() + let result = await withCheckedContinuation { continuation in + capture.endRecording { continuation.resume(returning: $0) } + } + self.studio = nil + becomeReady() + guard var info = studio.writer.info else { + studio.store.delete(studio.projectId) + session.toasts.show("Recording was too short", systemImage: "record.circle", style: .info) + return + } + if let duration = try? result.get(), duration > 0 { info.duration = duration } + guard info.duration >= Self.minimumDuration else { + studio.store.delete(studio.projectId) + session.toasts.show("Recording was too short", systemImage: "record.circle", style: .info) + return + } + await StudioSession.shared.cameraRecordingFinished(projectId: studio.projectId, in: studio.store, camera: info) + } + private func writerFailed(_ error: any Error) { Self.log.error("Writer failed: \(String(describing: error), privacy: .public)") session.toasts.show(error: error) diff --git a/apps/mobile-ios/Cap/Features/Record/RecordView.swift b/apps/mobile-ios/Cap/Features/Record/RecordView.swift index 0e30d5435ec..a44cdfffc20 100644 --- a/apps/mobile-ios/Cap/Features/Record/RecordView.swift +++ b/apps/mobile-ios/Cap/Features/Record/RecordView.swift @@ -10,6 +10,9 @@ struct RecordView: View { private let bubble = CameraBubble.shared private let recent = RecentRecordings.shared @AppStorage("so.cap.mobile.recordMode") private var mode: RecordMode = .camera + @AppStorage("so.cap.mobile.captureMode") private var captureMode: CaptureMode = .instant + @State private var studio = StudioSession.shared + @Environment(SessionModel.self) private var session @Environment(\.dismiss) private var dismiss @Environment(\.openURL) private var openURL @Environment(\.horizontalSizeClass) private var horizontalSizeClass @@ -61,9 +64,19 @@ struct RecordView: View { .persistentSystemOverlays(model.isRecording ? .hidden : .automatic) .task { upgrade.start() } .onAppear { - if !RecordMode.offered.contains(mode) { mode = .screen } + if !captureMode.isAvailable { captureMode = .instant } + if !RecordMode.offered(for: captureMode).contains(mode) { mode = .screen } + model.captureMode = captureMode + studio.attach(session) + studio.reload() + } + .onChange(of: captureMode) { _, capture in + model.captureMode = capture + if !RecordMode.offered(for: capture).contains(mode) { mode = .camera } + armStudio() } .task(id: mode) { + armStudio() // Each camera only runs while it's being shown. switch mode { case .camera: @@ -74,14 +87,26 @@ struct RecordView: View { bubble.releaseUnlessFloating() case .screenAndCamera: model.pauseCamera() - await bubble.prepare() + await bubble.prepare(studio: captureMode == .studio) + } + } + .task(id: captureMode == .studio && mode.recordsScreen) { + // The request the extension takes has a lifetime; keep it fresh while the screen option is showing. + while captureMode == .studio, mode.recordsScreen, !Task.isCancelled { + armStudio() + try? await Task.sleep(for: .seconds(60)) } } .onChange(of: screen.isCapturing) { _, capturing in guard capturing, mode.recordsScreen else { return } - ScreenRecordingSession.shared.began(withCamera: mode == .screenAndCamera) + let studioRecording = captureMode == .studio + ScreenRecordingSession.shared.began(withCamera: mode == .screenAndCamera, studio: studioRecording) + if studioRecording { studio.screenRecordingBegan(withCamera: mode == .screenAndCamera, bubble: bubble) } if mode == .screenAndCamera { bubble.float() } } + .fullScreenCover(item: $studio.editing) { project in + StudioEditorCover(project: project, session: session) + } .onAppear { recent.isRecorderVisible = true Usage.screen("record") @@ -90,6 +115,7 @@ struct RecordView: View { recent.isRecorderVisible = false model.teardown() bubble.releaseUnlessFloating() + if !screen.isCapturing { studio.disarmScreenRecording() } } .onChange(of: scenePhase) { _, phase in // Coming back from Settings after granting access. @@ -128,7 +154,7 @@ struct RecordView: View { RecordingTimer(startedAt: startedAt, limit: model.limit) .transition(.scale(scale: 0.8).combined(with: .opacity)) } else if !screen.isCapturing, model.phase != .finishing { - CaptureModeSwitch() + CaptureModeSwitch(selection: $captureMode) .transition(.opacity) } @@ -139,7 +165,11 @@ struct RecordView: View { .padding(.horizontal, 20) .padding(.top, 8) - if !model.isRecording, !screen.isCapturing, !recent.items.isEmpty { + if captureMode == .studio, !model.isRecording, !screen.isCapturing, !studio.projects.isEmpty { + StudioDrafts(studio: studio) + .padding(.top, 12) + .transition(.move(edge: .top).combined(with: .opacity)) + } else if captureMode == .instant, !model.isRecording, !screen.isCapturing, !recent.items.isEmpty { RecentRecordingsList(recent: recent) .padding(.horizontal, 16) .padding(.top, 12) @@ -150,7 +180,7 @@ struct RecordView: View { if mode.recordsScreen { // In the same column as the list above, so the two never overlap. - ScreenModePanel(recorder: screen, mode: mode, bubble: bubble, compact: !recent.items.isEmpty) + ScreenModePanel(recorder: screen, mode: mode, bubble: bubble, compact: captureMode == .studio ? !studio.projects.isEmpty : !recent.items.isEmpty, studio: captureMode == .studio) .transition(.opacity) Spacer(minLength: 12) } @@ -162,8 +192,15 @@ struct RecordView: View { .transition(.opacity) } + if studio.isFinishingScreen { + Label("Saving your recording", systemImage: "film") + .font(.footnote.weight(.semibold)) + .padding(.bottom, 14) + .transition(.opacity) + } + if !model.isRecording, model.phase != .finishing { - ModePicker(mode: $mode) + ModePicker(mode: $mode, options: RecordMode.offered(for: captureMode)) .padding(.bottom, 20) .transition(.opacity) } @@ -229,6 +266,16 @@ struct RecordView: View { .animation(.spring(duration: 0.35), value: model.isRecording) .animation(.spring(duration: 0.35), value: mode) .animation(.spring(duration: 0.35), value: recent.items) + .animation(.spring(duration: 0.35), value: captureMode) + .animation(.spring(duration: 0.35), value: studio.projects) + } + + private func armStudio() { + if captureMode == .studio, mode.recordsScreen { + studio.armScreenRecording() + } else { + studio.disarmScreenRecording() + } } private func blocked(title: String, message: String, systemImage: String, action: (String, () -> Void)?) -> some View { @@ -251,13 +298,14 @@ struct RecordView: View { } private struct CaptureModeSwitch: View { + @Binding var selection: CaptureMode @State private var comingSoon: CaptureMode? @State private var taps = 0 var body: some View { HStack(spacing: 2) { ForEach(CaptureMode.allCases) { option in - if option.isAvailable { + if option == selection { Label(option.title, systemImage: option.systemImage) .font(.subheadline.weight(.semibold)) .lineLimit(1) @@ -266,11 +314,17 @@ private struct CaptureModeSwitch: View { .background(Capsule().fill(.white.opacity(0.22))) .accessibilityElement(children: .combine) .accessibilityAddTraits(.isSelected) + .transition(.opacity) } else { Button { - comingSoon = option + if option.isAvailable { + selection = option + Usage.track("capture_mode_changed", ["mode": .string(option.rawValue)]) + } else { + comingSoon = option + AccessibilityNotification.Announcement(option.comingSoonMessage).post() + } taps += 1 - AccessibilityNotification.Announcement(option.comingSoonMessage).post() } label: { Image(systemName: option.systemImage) .font(.system(size: 15, weight: .semibold)) @@ -278,9 +332,10 @@ private struct CaptureModeSwitch: View { .contentShape(Capsule()) } .buttonStyle(.plain) - .foregroundStyle(.white.opacity(0.55)) + .foregroundStyle(.white.opacity(option.isAvailable ? 0.85 : 0.55)) .accessibilityLabel(option.title) - .accessibilityValue("Coming soon") + .accessibilityValue(option.isAvailable ? "" : "Coming soon") + .accessibilityIdentifier("capture-mode-\(option.rawValue)") } } } @@ -301,6 +356,7 @@ private struct CaptureModeSwitch: View { .glassEffect(.regular, in: Capsule()) .dynamicTypeSize(...DynamicTypeSize.large) .animation(.easeOut(duration: 0.2), value: comingSoon) + .animation(.spring(duration: 0.3), value: selection) .sensoryFeedback(.selection, trigger: taps) .task(id: taps) { guard comingSoon != nil else { return } @@ -313,10 +369,11 @@ private struct CaptureModeSwitch: View { private struct ModePicker: View { @Binding var mode: RecordMode + let options: [RecordMode] var body: some View { HStack(spacing: 4) { - ForEach(RecordMode.offered) { option in + ForEach(options) { option in Button { mode = option } label: { diff --git a/apps/mobile-ios/Cap/Features/Record/ScreenRecording.swift b/apps/mobile-ios/Cap/Features/Record/ScreenRecording.swift index 04b2d6606c0..5c791d0bbbf 100644 --- a/apps/mobile-ios/Cap/Features/Record/ScreenRecording.swift +++ b/apps/mobile-ios/Cap/Features/Record/ScreenRecording.swift @@ -1,4 +1,5 @@ import ActivityKit +@preconcurrency import AVFoundation import CapKit import notify import Observation @@ -31,7 +32,7 @@ enum CaptureMode: String, CaseIterable, Identifiable { } } - var isAvailable: Bool { self == .instant } + var isAvailable: Bool { self != .screenshot } var comingSoonMessage: String { "\(title) is coming soon" } } @@ -56,6 +57,15 @@ enum RecordMode: String, CaseIterable, Identifiable { /// Screen + Camera keeps the camera running behind other apps, which needs Apple's multitasking camera /// entitlement. It stays hidden until the app has it. static let offered: [RecordMode] = [.camera, .screen] + + /// Studio records the camera into its own track while the screen records, where the phone allows the camera to + /// keep running behind other apps. + static func offered(for capture: CaptureMode) -> [RecordMode] { + guard capture == .studio, cameraWhileMultitasking else { return offered } + return [.camera, .screen, .screenAndCamera] + } + + static let cameraWhileMultitasking = AVCaptureSession().isMultitaskingCameraAccessSupported } /// iOS only lets its own button start a broadcast, so that button sits invisibly on top of Cap's. @@ -118,6 +128,8 @@ final class ScreenRecordingSession { private static let log = Logger(subsystem: "so.cap.mobile", category: "screen-recording") private var activity: Activity? private var recording: (withCamera: Bool, startedAt: Date, stats: ProcessStats.Mark)? + /// A Studio recording stays on the phone, so none of the upload hand-over applies when it stops. + private var isStudio = false private var handOffToken: Int32 = NOTIFY_TOKEN_INVALID private var started = false @@ -143,10 +155,11 @@ final class ScreenRecordingSession { } } - func began(withCamera: Bool) { + func began(withCamera: Bool, studio: Bool = false) { guard recording == nil else { return } recording = (withCamera, .now, ProcessStats.mark()) - Usage.track("recording_started", ["mode": "screen", "camera": .bool(withCamera)]) + isStudio = studio + if !studio { Usage.track("recording_started", ["mode": "screen", "camera": .bool(withCamera)]) } startActivityIfNeeded() // iOS only starts Live Activities while Cap is in front, and the system sheet is still // closing as recording begins. @@ -190,7 +203,7 @@ final class ScreenRecordingSession { endActivity() return } - RecentRecordings.shared.savingScreen() + if !isStudio { RecentRecordings.shared.savingScreen() } ScreenRecordingStop.request() if let found = activity { nonisolated(unsafe) let activity = found @@ -202,6 +215,12 @@ final class ScreenRecordingSession { guard !ScreenRecorder.isScreenCaptured else { return } let wasRecording = recording != nil || activity != nil endActivity() + if isStudio { + isStudio = false + recording = nil + StudioSession.shared.screenRecordingEnded(bubble: CameraBubble.shared) + return + } if let recording { self.recording = nil // No length here: Cap may only notice the stop when it is next opened. The recorded length comes with @@ -269,6 +288,7 @@ struct ScreenModePanel: View { let mode: RecordMode let bubble: CameraBubble var compact = false + var studio = false var body: some View { VStack(spacing: 24) { @@ -279,7 +299,7 @@ struct ScreenModePanel: View { Text(recorder.isCapturing ? "Recording your screen" : title) .font(.title2.weight(.semibold)) if recorder.isCapturing { - Text("Use any app. To stop, tap the Dynamic Island or the Stop button on the Lock Screen, or come back here. Cap opens your recording.") + Text(studio ? "Use any app. To stop, tap the Dynamic Island or come back here. Your recording opens in the editor." : "Use any app. To stop, tap the Dynamic Island or the Stop button on the Lock Screen, or come back here. Cap opens your recording.") .font(.subheadline) .foregroundStyle(.white.opacity(0.7)) .multilineTextAlignment(.center) @@ -294,7 +314,7 @@ struct ScreenModePanel: View { VStack(alignment: .leading, spacing: 10) { step(1, "Tap the record button, then **Start Recording**. Turn the microphone on there to add your voice.") step(2, mode == .screenAndCamera ? "Use any app. Your camera floats on top and is recorded too. Drag it anywhere, pinch to resize, or swipe it to the edge to hide it." : "Use any app. Cap keeps recording.") - step(3, "Tap the Dynamic Island to stop. Your link is ready straight away.") + step(3, studio ? "Tap the Dynamic Island to stop. Your recording opens in the editor." : "Tap the Dynamic Island to stop. Your link is ready straight away.") } } } diff --git a/apps/mobile-ios/Cap/Features/Studio/StudioSession.swift b/apps/mobile-ios/Cap/Features/Studio/StudioSession.swift new file mode 100644 index 00000000000..1017b7ab343 --- /dev/null +++ b/apps/mobile-ios/Cap/Features/Studio/StudioSession.swift @@ -0,0 +1,454 @@ +@preconcurrency import AVFoundation +import CapKit +import Observation +import os +import UIKit + +/// Studio Mode keeps recordings on the phone as separate screen and camera tracks, so they can be laid out, styled +/// and cut before anything is shared. +@MainActor +@Observable +final class StudioSession { + static let shared = StudioSession() + private static let log = Logger(subsystem: "so.cap.mobile", category: "studio") + + @ObservationIgnored private(set) var store = StudioStore.shared(owner: "signed-out") + @ObservationIgnored private var owner: String? + private let requests = StudioStore.requests() + private(set) var projects: [StudioProject] = [] + private(set) var presetLibrary = StudioPresetLibrary() + var editing: StudioProject? + private(set) var isFinishingScreen = false + + /// The project the next screen recording goes into, set before the system sheet can start it. + @ObservationIgnored private var armedProjectId: String? + @ObservationIgnored private var armedAt: Date = .distantPast + @ObservationIgnored private var screenCamera: (writer: StudioTrackWriter, projectId: String)? + @ObservationIgnored private var recordingProjectId: String? + /// The account a screen recording started in. Sign-out mid-recording mustn't send its files elsewhere. + @ObservationIgnored private var recordingStore: StudioStore? + @ObservationIgnored private(set) weak var session: SessionModel? + @ObservationIgnored private var pendingOpen: String? + + private init() {} + + func attach(_ session: SessionModel) { + self.session = session + let userId = session.credentials.userId + guard owner != userId else { return } + owner = userId + store = StudioStore.shared(owner: userId) + presetLibrary = store.presets.load() + thumbnails = [:] + lastRecovery = .distantPast + reload() + if let pendingOpen { + self.pendingOpen = nil + Task { await open(projectId: pendingOpen) } + } + } + + func signedOut() { + disarmScreenRecording() + editing = nil + projects = [] + presetLibrary = StudioPresetLibrary() + thumbnails = [:] + owner = nil + session = nil + pendingOpen = nil + store = StudioStore.shared(owner: "signed-out") + } + + func reload() { + guard owner != nil else { return } + projects = store.all() + Task { await recoverInterrupted() } + } + + // MARK: Screen + + /// iOS only lets its own button start a broadcast, so the extension can't be told at tap time. Cap leaves a + /// request for it whenever Studio's screen mode is showing; the extension takes it if it starts within its life. + func armScreenRecording() { + guard recordingProjectId == nil else { return } + if armedProjectId == nil || Date.now.timeIntervalSince(armedAt) > StudioScreenRequest.lifetime / 2 { + armedProjectId = UUID().uuidString.lowercased() + armedAt = .now + } + guard let armedProjectId, let owner else { return } + StudioScreenRequest.set(.init(projectId: armedProjectId, ownerId: owner, requestedAt: armedAt), in: requests) + } + + /// Instant Mode's screen recordings must never land in Studio. + func disarmScreenRecording() { + guard recordingProjectId == nil else { return } + armedProjectId = nil + StudioScreenRequest.set(nil, in: requests) + } + + var isRecordingScreen: Bool { recordingProjectId != nil } + + /// The broadcast started. Whether it took the request is only known from the request being gone. + func screenRecordingBegan(withCamera: Bool, bubble: CameraBubble) { + guard let projectId = armedProjectId, StudioScreenRequest.peek(in: requests) == nil else { return } + recordingProjectId = projectId + recordingStore = store + armedProjectId = nil + Usage.track("recording_started", ["mode": "studio_screen", "camera": .bool(withCamera)]) + guard withCamera else { return } + do { + let directory = store.directory(for: projectId) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let writer = try StudioTrackWriter(url: directory.appending(path: "camera.mp4"), settings: .init(kind: .camera, audio: false), clock: bubble.captureClock) { started in + var pending = started + pending.duration = 0 + try? pending.save(as: "camera", in: directory) + } + screenCamera = (writer, projectId) + bubble.beginStudioRecording(writer) + } catch { + Self.log.error("Studio camera didn't start: \(String(describing: error), privacy: .public)") + } + } + + /// The broadcast ended, from anywhere. Waits for the extension to close its file, then opens the editor. + func screenRecordingEnded(bubble: CameraBubble) { + guard let projectId = recordingProjectId else { return } + let store = recordingStore ?? self.store + recordingProjectId = nil + recordingStore = nil + isFinishingScreen = true + let camera = screenCamera + screenCamera = nil + Task { + defer { isFinishingScreen = false } + var cameraInfo: StudioTrackInfo? + if let camera { + cameraInfo = await bubble.endStudioRecording(camera.writer) + try? cameraInfo?.save(as: "camera", in: store.directory(for: projectId)) + } + let directory = store.directory(for: projectId) + // The extension notes the track with no length on its first frame and again with its length once the file + // is closed. Only the second is safe to open. An extension killed while closing never writes it, so after a + // while the file's own length is used instead. + for _ in 0..<150 where (StudioTrackInfo.load("screen", in: directory)?.duration ?? 0) <= 0 { + try? await Task.sleep(for: .milliseconds(100)) + } + await assemble(projectId: projectId, in: store, screen: StudioTrackInfo.load("screen", in: directory), camera: cameraInfo, title: Self.title(for: .now, screen: true)) + } + } + + // MARK: Camera + + func cameraRecordingFinished(projectId: String, in store: StudioStore, camera: StudioTrackInfo) async { + try? camera.save(as: "camera", in: store.directory(for: projectId)) + await assemble(projectId: projectId, in: store, screen: nil, camera: camera, title: Self.title(for: .now, screen: false)) + } + + /// A track whose process ended before closing it has a start but no length; its file says how much was kept. + private func resolve(_ info: StudioTrackInfo?, in directory: URL) async -> StudioTrackInfo? { + guard var info else { return nil } + let url = directory.appending(path: info.file) + guard FileManager.default.fileExists(atPath: url.path) else { return nil } + if info.duration <= 0 { + info.duration = (try? await AVURLAsset(url: url).load(.duration).seconds).flatMap { $0.isFinite ? $0 : nil } ?? 0 + } + if var audio = info.audio { + let audioURL = directory.appending(path: audio.file) + if !FileManager.default.fileExists(atPath: audioURL.path) { + info.audio = nil + } else if audio.duration <= 0 { + audio.duration = (try? await AVURLAsset(url: audioURL).load(.duration).seconds).flatMap { $0.isFinite ? $0 : nil } ?? 0 + info.audio = audio.duration > 0 ? audio : nil + } + } + return info.duration > 0 ? info : nil + } + + /// `store` is the account the recording belongs to, fixed before any await so a sign-out can't redirect it. + private func assemble(projectId: String, in store: StudioStore, screen: StudioTrackInfo?, camera: StudioTrackInfo?, title: String, opens: Bool = true) async { + let directory = store.directory(for: projectId) + let screen = await resolve(screen, in: directory) + let camera = await resolve(camera, in: directory) + let aligned = StudioTrackInfo.align(screen: screen, camera: camera) + let presets = store.presets + var project = StudioProject(id: projectId, title: title, screen: aligned.screen, camera: aligned.camera, look: presets.load().defaultPreset.look) + guard !project.isEmpty, project.duration >= 1 else { + store.delete(projectId) + Usage.track("recording_discarded", ["mode": "studio", "reason": "too_short"]) + session?.toasts.show("Recording was too short", systemImage: "record.circle", style: .info) + return + } + // A camera on its own is better shown large; a screen gets Cap's framing either way. + if project.screen == nil { project.look.camera.layout = .cameraOnly } + presets.provideImage(for: project.look, in: directory) + do { + try store.save(project) + } catch { + Self.log.error("Studio project didn't save: \(String(describing: error), privacy: .public)") + session?.toasts.show(error: error) + return + } + Usage.track("recording_stopped", ["mode": "studio", "duration_seconds": .int(Int(project.duration.rounded())), "screen": .bool(project.screen != nil), "camera": .bool(project.camera != nil)]) + // Signed out, or into another account, while this was put together: it waits in its own account. + guard store.root == self.store.root else { return } + projects = store.all() + guard opens else { return } + UIImpactFeedbackGenerator(style: .light).impactOccurred() + editing = project + } + + // MARK: Recovery + + @ObservationIgnored private var lastRecovery: Date = .distantPast + @ObservationIgnored private var recovering = false + + /// Recordings whose app or extension was killed before the project was saved. Their files are fragmented MP4s, + /// so everything up to the last second plays; they come back as drafts. A folder still being written is left for + /// a later pass, which runs again whenever the drafts are next shown. + func recoverInterrupted(now: Date = .now) async { + guard !recovering, now.timeIntervalSince(lastRecovery) > 15 else { return } + recovering = true + lastRecovery = now + defer { recovering = false } + let store = store + let fileManager = FileManager.default + let ids = (try? fileManager.contentsOfDirectory(atPath: store.root.path)) ?? [] + for id in ids where id != recordingProjectId && id != armedProjectId { + let directory = store.directory(for: id) + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: directory.path, isDirectory: &isDirectory), isDirectory.boolValue, + store.load(id) == nil + else { continue } + let files = (try? fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: [.contentModificationDateKey])) ?? [] + // Still being written by a recording that's running. + let newest = files.compactMap { try? $0.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate }.max() ?? .distantPast + guard now.timeIntervalSince(newest) > 60 else { continue } + let screen = StudioTrackInfo.load("screen", in: directory) + let camera = StudioTrackInfo.load("camera", in: directory) + guard screen != nil || camera != nil else { + store.delete(id) + continue + } + Self.log.info("Recovering interrupted Studio recording \(id, privacy: .public)") + await assemble(projectId: id, in: store, screen: screen, camera: camera, title: "Recovered recording", opens: false) + } + } + + + /// `store` is the account the project was opened in, which stays its home even if the session ends meanwhile. + func save(_ project: StudioProject, in store: StudioStore? = nil) { + let target = store ?? self.store + do { + try target.save(project) + guard target.root == self.store.root else { return } + if let index = projects.firstIndex(where: { $0.id == project.id }) { projects[index] = project } else { reload() } + } catch { + Self.log.error("Studio project didn't save: \(String(describing: error), privacy: .public)") + } + } + + func delete(_ project: StudioProject) { + store.delete(project.id) + projects.removeAll { $0.id == project.id } + Usage.track("studio_deleted") + } + + /// From a notification. Cap may have been closed when the recording stopped, so the project can still be just + /// the extension's tracks; they're put together first. + func open(projectId: String) async { + guard owner != nil else { + pendingOpen = projectId + return + } + let store = store + if let project = store.load(projectId) { + editing = project + return + } + let directory = store.directory(for: projectId) + let screen = StudioTrackInfo.load("screen", in: directory) + let camera = StudioTrackInfo.load("camera", in: directory) + guard screen != nil || camera != nil else { return } + await assemble(projectId: projectId, in: store, screen: screen, camera: camera, title: Self.title(for: .now, screen: screen != nil)) + } + + // MARK: Presets + + @discardableResult + func savePreset(named name: String, look: StudioLook, from directory: URL) -> StudioPreset? { + var library = presetLibrary + let preset = StudioPreset(name: library.uniqueName(name), look: look) + if case let .image(file) = look.background { + do { + try store.presets.keepImage(named: file, from: directory.appending(path: file)) + } catch { + session?.toasts.show(error: error) + return nil + } + } + library.saved.insert(preset, at: 0) + guard storePresets(library) else { return nil } + Usage.track("studio_preset_saved") + return preset + } + + func updatePreset(_ id: String, look: StudioLook, from directory: URL) { + var library = presetLibrary + guard let index = library.saved.firstIndex(where: { $0.id == id }) else { return } + // Without its photo the preset would point at nothing, so it stays as it was. + if case let .image(file) = look.background { + do { + try store.presets.keepImage(named: file, from: directory.appending(path: file)) + } catch { + session?.toasts.show(error: error) + return + } + } + library.saved[index].look = look + storePresets(library) + } + + func renamePreset(_ id: String, to name: String) { + var library = presetLibrary + guard let index = library.saved.firstIndex(where: { $0.id == id }) else { return } + let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, trimmed != library.saved[index].name else { return } + var others = library + others.saved.remove(at: index) + library.saved[index].name = others.uniqueName(trimmed) + storePresets(library) + } + + func deletePreset(_ id: String) { + var library = presetLibrary + library.saved.removeAll { $0.id == id } + if library.defaultId == id { library.defaultId = nil } + storePresets(library) + } + + func makeDefault(_ id: String) { + var library = presetLibrary + library.defaultId = id == StudioPreset.builtIn[0].id ? nil : id + if storePresets(library) { + session?.toasts.show("New recordings start in \(library.defaultPreset.name)", systemImage: "pin.fill", style: .info) + } + } + + @discardableResult + private func storePresets(_ library: StudioPresetLibrary) -> Bool { + do { + try store.presets.save(library) + presetLibrary = library + return true + } catch { + Self.log.error("Studio presets didn't save: \(String(describing: error), privacy: .public)") + session?.toasts.show(error: error) + return false + } + } + + static func title(for date: Date, screen: Bool) -> String { + (screen ? "Studio Recording " : "Studio Camera ") + date.formatted(.dateTime.day().month(.abbreviated).year().hour().minute()) + } + + // MARK: Thumbnails + + @ObservationIgnored private var thumbnails: [String: UIImage] = [:] + + func thumbnail(for project: StudioProject) async -> UIImage? { + if let image = thumbnails[project.id] { return image } + guard let track = project.screen ?? project.camera else { return nil } + let generator = AVAssetImageGenerator(asset: AVURLAsset(url: store.url(track.file, in: project.id))) + generator.appliesPreferredTrackTransform = true + generator.maximumSize = CGSize(width: 240, height: 240) + guard let image = try? await generator.image(at: CMTime(seconds: min(1, track.duration / 2), preferredTimescale: 600)).image else { return nil } + let result = UIImage(cgImage: image) + thumbnails[project.id] = result + return result + } +} + +enum StudioAssets { + struct Track: Identifiable, Hashable { + let id: String + let title: String + var url: URL? { Bundle.main.url(forResource: id, withExtension: "m4a") } + } + + static let music: [Track] = [ + Track(id: "lofi-beats-mirostar", title: "Lofi Beats"), + Track(id: "raindrops-lofi-sleep-bluelike", title: "Raindrops"), + Track(id: "sunday-mood-lofi-cafe-upbeat-bluelike", title: "Sunday Mood"), + Track(id: "good-night-lofi-cozy-chill-fassounds", title: "Good Night"), + Track(id: "ambient-trap-empty-streets-dreamstate-openmindaudio", title: "Empty Streets"), + Track(id: "lofi-study-calm-peaceful-chill-hop-fassounds", title: "Study"), + Track(id: "lofi-cinematic-pulsebox", title: "Cinematic"), + Track(id: "lofi-hip-hop-leberch", title: "Hip Hop"), + Track(id: "cassette-retrositive", title: "Cassette"), + Track(id: "lofi-smooth-pulsebox", title: "Smooth"), + ] + + static let wallpapers = ["blue-1", "blue-3", "blue-5", "purple-2", "purple-4", "orange-1", "orange-5", "orange-8", "dark-2", "dark-5"] + + static func wallpaperURL(_ name: String) -> URL? { + Bundle.main.url(forResource: name, withExtension: "heic") + } + + private static var thumbnailCache: [String: UIImage] = [:] + + /// Small and drawn in every strip that shows a background, so each is read from disk once. + static func wallpaperThumbnail(_ name: String) -> UIImage? { + if let cached = thumbnailCache[name] { return cached } + let image = wallpaperThumbnailURL(name).flatMap { UIImage(contentsOfFile: $0.path) } + thumbnailCache[name] = image + return image + } + + static func wallpaperThumbnailURL(_ name: String) -> URL? { + Bundle.main.url(forResource: "\(name)-thumb", withExtension: "jpg") + } + + /// Decoded straight to the size it's drawn at: a 2400 px wallpaper never needs to be held at full size. + nonisolated static func loadImage(_ url: URL, maxPixels: Int = 2400) -> CGImage? { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else { return nil } + return CGImageSourceCreateThumbnailAtIndex(source, 0, [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceThumbnailMaxPixelSize: maxPixels, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceShouldCacheImmediately: true, + ] as CFDictionary) + } +} + +#if DEBUG +/// Debug-only: `CAP_DEBUG_ROUTE=studio:` opens `screen.mp4` and `camera.mp4` from a folder as a Studio +/// project, so the simulator, which can't record either, can still exercise and measure the editor. +@MainActor +enum StudioDebug { + static func openFixture(_ directory: URL) { + Task { + let store = StudioSession.shared.store + let id = "fixture" + store.delete(id) + let target = store.directory(for: id) + try? FileManager.default.createDirectory(at: target, withIntermediateDirectories: true) + let cameraOffset = Double(ProcessInfo.processInfo.environment["CAP_DEBUG_STUDIO_CAMERA_OFFSET"] ?? "") ?? 0 + var tracks: [String: StudioProject.Track] = [:] + for name in ["screen", "camera"] { + let source = directory.appending(path: "\(name).mp4") + guard FileManager.default.fileExists(atPath: source.path) else { continue } + try? FileManager.default.copyItem(at: source, to: target.appending(path: "\(name).mp4")) + let duration = (try? await AVURLAsset(url: source).load(.duration).seconds) ?? 0 + tracks[name] = StudioProject.Track(file: "\(name).mp4", offset: name == "camera" ? cameraOffset : 0, duration: duration) + } + var project = StudioProject(id: id, title: "Studio fixture", createdAt: .now, screen: tracks["screen"], camera: tracks["camera"]) + if project.screen == nil { project.look.camera.layout = .cameraOnly } + try? store.save(project) + StudioSession.shared.reload() + StudioSession.shared.editing = project + } + } +} +#endif diff --git a/apps/mobile-ios/Cap/Features/Studio/StudioViews.swift b/apps/mobile-ios/Cap/Features/Studio/StudioViews.swift new file mode 100644 index 00000000000..bfea18e394e --- /dev/null +++ b/apps/mobile-ios/Cap/Features/Studio/StudioViews.swift @@ -0,0 +1,90 @@ +import CapKit +import SwiftUI + +struct StudioEditorCover: View { + let project: StudioProject + let session: SessionModel + @State private var model: EditorModel + + init(project: StudioProject, session: SessionModel) { + self.project = project + self.session = session + _model = State(initialValue: EditorModel(project: project, session: session)) + } + + var body: some View { + EditorView(model: model) { _ in + StudioSession.shared.editing = nil + StudioSession.shared.reload() + } + .environment(session) + } +} + +struct StudioDrafts: View { + let studio: StudioSession + @State private var deleting: StudioProject? + + var body: some View { + ScrollView(.horizontal) { + HStack(spacing: 10) { + ForEach(studio.projects) { project in + Button { + studio.editing = project + } label: { + StudioDraftCard(project: project, studio: studio) + } + .buttonStyle(.plain) + .contextMenu { + Button("Delete", systemImage: "trash", role: .destructive) { deleting = project } + } + .accessibilityLabel("\(project.title), \(CapFormat.timestamp(project.timeline.outputDuration))") + .accessibilityHint("Opens the editor") + .accessibilityAction(named: "Delete") { deleting = project } + } + } + .padding(.horizontal, 16) + } + .scrollIndicators(.hidden) + .frame(height: StudioDraftCard.height) + .confirmationDialog("Delete this recording?", isPresented: Binding(get: { deleting != nil }, set: { if !$0 { deleting = nil } }), titleVisibility: .visible) { + Button("Delete", role: .destructive) { + if let deleting { studio.delete(deleting) } + deleting = nil + } + Button("Cancel", role: .cancel) { deleting = nil } + } message: { + Text("It's only on this iPhone, so it can't be recovered.") + } + .accessibilityIdentifier("studio-drafts") + } +} + +private struct StudioDraftCard: View { + static let height: CGFloat = 112 + let project: StudioProject + let studio: StudioSession + @State private var image: UIImage? + + var body: some View { + ZStack(alignment: .bottomLeading) { + if let image { + Image(uiImage: image).resizable().scaledToFill() + } else { + Color.white.opacity(0.1) + } + LinearGradient(colors: [.clear, .black.opacity(0.65)], startPoint: .center, endPoint: .bottom) + HStack(spacing: 4) { + Image(systemName: project.screen != nil ? (project.camera != nil ? "rectangle.inset.bottomright.filled" : "iphone") : "person.crop.square") + .font(.caption2.weight(.semibold)) + Text(CapFormat.timestamp(project.timeline.outputDuration)) + .font(.caption.weight(.semibold).monospacedDigit()) + } + .padding(8) + } + .frame(width: 76, height: Self.height) + .clipShape(RoundedRectangle(cornerRadius: 14, style: .continuous)) + .overlay(RoundedRectangle(cornerRadius: 14, style: .continuous).strokeBorder(.white.opacity(0.15), lineWidth: 1)) + .task(id: project.id) { image = await studio.thumbnail(for: project) } + } +} diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/ambient-trap-empty-streets-dreamstate-openmindaudio.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/ambient-trap-empty-streets-dreamstate-openmindaudio.m4a new file mode 100644 index 00000000000..05aefd6fe80 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/ambient-trap-empty-streets-dreamstate-openmindaudio.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/cassette-retrositive.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/cassette-retrositive.m4a new file mode 100644 index 00000000000..5076f7ab700 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/cassette-retrositive.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/good-night-lofi-cozy-chill-fassounds.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/good-night-lofi-cozy-chill-fassounds.m4a new file mode 100644 index 00000000000..0c4623bd43f Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/good-night-lofi-cozy-chill-fassounds.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-beats-mirostar.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-beats-mirostar.m4a new file mode 100644 index 00000000000..53a09355f3a Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-beats-mirostar.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-cinematic-pulsebox.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-cinematic-pulsebox.m4a new file mode 100644 index 00000000000..5c9df6ac1fe Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-cinematic-pulsebox.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-hip-hop-leberch.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-hip-hop-leberch.m4a new file mode 100644 index 00000000000..8a31d51201b Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-hip-hop-leberch.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-smooth-pulsebox.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-smooth-pulsebox.m4a new file mode 100644 index 00000000000..a9b95980d52 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-smooth-pulsebox.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-study-calm-peaceful-chill-hop-fassounds.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-study-calm-peaceful-chill-hop-fassounds.m4a new file mode 100644 index 00000000000..529e65aaf2d Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/lofi-study-calm-peaceful-chill-hop-fassounds.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/raindrops-lofi-sleep-bluelike.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/raindrops-lofi-sleep-bluelike.m4a new file mode 100644 index 00000000000..c25c49c5b8a Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/raindrops-lofi-sleep-bluelike.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Music/sunday-mood-lofi-cafe-upbeat-bluelike.m4a b/apps/mobile-ios/Cap/Resources/Studio/Music/sunday-mood-lofi-cafe-upbeat-bluelike.m4a new file mode 100644 index 00000000000..702f38401ec Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Music/sunday-mood-lofi-cafe-upbeat-bluelike.m4a differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1-thumb.jpg new file mode 100644 index 00000000000..b8a67963c53 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1.heic new file mode 100644 index 00000000000..2d5f6241fd3 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-1.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3-thumb.jpg new file mode 100644 index 00000000000..f7722883f43 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3.heic new file mode 100644 index 00000000000..477d50600ae Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-3.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5-thumb.jpg new file mode 100644 index 00000000000..298c7f8c7ba Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5.heic new file mode 100644 index 00000000000..930be7cf7a5 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/blue-5.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2-thumb.jpg new file mode 100644 index 00000000000..f931f77dc5d Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2.heic new file mode 100644 index 00000000000..5bc0f77be7a Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-2.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5-thumb.jpg new file mode 100644 index 00000000000..6650bb10843 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5.heic new file mode 100644 index 00000000000..e47d2f95baa Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/dark-5.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1-thumb.jpg new file mode 100644 index 00000000000..4b31dbf301e Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1.heic new file mode 100644 index 00000000000..1f3c620dab0 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-1.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5-thumb.jpg new file mode 100644 index 00000000000..f734943e35b Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5.heic new file mode 100644 index 00000000000..a6b951819e2 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-5.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8-thumb.jpg new file mode 100644 index 00000000000..399e1a81b6e Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8.heic new file mode 100644 index 00000000000..8c9e285fbd5 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/orange-8.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2-thumb.jpg new file mode 100644 index 00000000000..e52fae0f0fe Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2.heic new file mode 100644 index 00000000000..444c4cddf51 Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-2.heic differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4-thumb.jpg b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4-thumb.jpg new file mode 100644 index 00000000000..ecc1ae12ccd Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4-thumb.jpg differ diff --git a/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4.heic b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4.heic new file mode 100644 index 00000000000..ed00919c77a Binary files /dev/null and b/apps/mobile-ios/Cap/Resources/Studio/Wallpapers/purple-4.heic differ diff --git a/apps/mobile-ios/CapBroadcast/SampleHandler.swift b/apps/mobile-ios/CapBroadcast/SampleHandler.swift index 768fc6bf076..291e810b483 100644 --- a/apps/mobile-ios/CapBroadcast/SampleHandler.swift +++ b/apps/mobile-ios/CapBroadcast/SampleHandler.swift @@ -27,7 +27,9 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { private var api: CapAPI? private var limit: TimeInterval? private var job: RecordingJob? - private var writer: SegmentWriter? + private var writer: (any RecordingSink)? + /// Set when Cap asked for a Studio recording: the screen is kept on the phone for editing, not uploaded. + private var studio: (store: StudioStore, projectId: String)? private var audioSource: RPSampleBufferType? private var lastFrame: CMSampleBuffer? private var lastFrameAt: CMTime = .invalid @@ -40,6 +42,13 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { private var repeatedFrames = 0 override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) { + if let request = StudioScreenRequest.take(in: StudioStore.requests()) { + let store = StudioStore.shared(owner: request.ownerId) + queue.sync { studio = (store, request.projectId) } + startIdleTimer() + listenForStop() + return + } guard let environment = Self.environment() else { finish(message: "Cap couldn't start recording. Reinstall Cap and try again.") return @@ -123,6 +132,12 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { writer?.appendVideo(buffer) lastFrame = buffer lastFrameAt = presentation + if studio != nil, (writer as? StudioTrackWriter)?.failed == true { + complete { [weak self] in + self?.finish(message: "Recording stopped because this iPhone is out of space. What was recorded is saved in Cap.") + } + return + } if let limit, let writer, writer.duration >= limit { limitReached() } @@ -130,6 +145,27 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { /// Created on the first frame, when the screen's size is known. private func start(with buffer: CMSampleBuffer) -> Bool { + if let studio { + guard studio.store.hasRoomToRecord() else { + finish(message: "Free up at least 1 GB on this iPhone to record in Studio Mode.") + return false + } + do { + let directory = studio.store.directory(for: studio.projectId) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + writer = try StudioTrackWriter(url: directory.appending(path: Self.studioScreenFile), settings: .init(kind: .screen, audio: true)) { started in + // Noted as soon as picture or sound starts: if this process is ended mid-recording, Cap can still recover it. + var pending = started + pending.duration = 0 + pending.audio?.duration = 0 + try? pending.save(as: "screen", in: directory) + } + return true + } catch { + finish(message: "Cap couldn't start recording your screen.") + return false + } + } guard let uploads, let credentials, let pixels = CMSampleBufferGetImageBuffer(buffer) else { return false } // H.264 wants even dimensions. let width = CVPixelBufferGetWidth(pixels) & ~1 @@ -219,6 +255,10 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { idleTimer = nil notify_cancel(stopToken) lastFrame = nil + if let studio { + completeStudio(studio, handOff: handOff, done) + return + } guard let writer, let job, let uploads else { done() return @@ -286,6 +326,43 @@ final class SampleHandler: RPBroadcastSampleHandler, @unchecked Sendable { } private static let notificationId = "so.cap.mobile.screen-recording" + private static let studioScreenFile = "screen.mp4" + + /// Studio recordings finish on the phone: close the file, note when it started, and tell Cap it's there. + private func completeStudio(_ studio: (store: StudioStore, projectId: String), handOff: Bool, _ done: @escaping @Sendable () -> Void) { + guard let writer = writer as? StudioTrackWriter else { + done() + return + } + self.writer = nil + let directory = studio.store.directory(for: studio.projectId) + let counts = (frames: frames, repeated: repeatedFrames, started: started) + writer.finish(at: CMClockGetTime(CMClockGetHostTimeClock())) { result in + let duration = (try? result.get()) ?? 0 + ProcessStats.measure(since: counts.started, process: "broadcast-studio", recordedSeconds: duration, extra: [ + "frames": Double(counts.frames), + "repeatedFrames": Double(counts.repeated), + ]).save() + if var info = writer.info { + if duration > 0 { info.duration = duration } + try? info.save(as: "screen", in: directory) + } + notify_post(StudioScreenRequest.finishedNotification) + if !handOff { + Task { await Self.notifyStudioSaved(projectId: studio.projectId) } + } + done() + } + } + + private static func notifyStudioSaved(projectId: String) async { + let content = UNMutableNotificationContent() + content.title = "Your Studio recording is ready" + content.body = "Tap to edit it in Cap." + content.userInfo = ["studioProject": projectId] + content.interruptionLevel = .active + try? await UNUserNotificationCenter.current().add(UNNotificationRequest(identifier: notificationId, content: content, trigger: nil)) + } private static func notifySaved(_ entry: RecordingHandoff.Entry?) async { let content = UNMutableNotificationContent() diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/ProcessStats.swift b/apps/mobile-ios/CapKit/Sources/CapKit/ProcessStats.swift index ca084fcb1f7..09bd182ba22 100644 --- a/apps/mobile-ios/CapKit/Sources/CapKit/ProcessStats.swift +++ b/apps/mobile-ios/CapKit/Sources/CapKit/ProcessStats.swift @@ -59,7 +59,7 @@ public struct ProcessStats: Codable, Sendable { return seconds(usage.ru_utime) + seconds(usage.ru_stime) } - static func footprint() -> (current: Double, peak: Double) { + public static func footprint() -> (current: Double, peak: Double) { var info = task_vm_info_data_t() var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) let result = withUnsafeMutablePointer(to: &info) { diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/Recording.swift b/apps/mobile-ios/CapKit/Sources/CapKit/Recording.swift index 6191377350b..3f0ba418a86 100644 --- a/apps/mobile-ios/CapKit/Sources/CapKit/Recording.swift +++ b/apps/mobile-ios/CapKit/Sources/CapKit/Recording.swift @@ -158,6 +158,8 @@ public struct RecordingCreated: Codable, Sendable, Hashable { public enum RecordingLimits { /// Matches `mobileFreeRecordingDurationSeconds` on the server. public static let freeDuration: TimeInterval = 5 * 60 + /// Matches `mobileRecordingDurationToleranceSeconds` on the server. + public static let freeTolerance: TimeInterval = 5 public static let segmentDuration: TimeInterval = 2 } diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/Studio.swift b/apps/mobile-ios/CapKit/Sources/CapKit/Studio.swift new file mode 100644 index 00000000000..f66a0e5d732 --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/Studio.swift @@ -0,0 +1,646 @@ +import CoreGraphics +import Foundation + +public struct StudioColor: Codable, Sendable, Hashable { + public var red: UInt8 + public var green: UInt8 + public var blue: UInt8 + + public init(_ red: UInt8, _ green: UInt8, _ blue: UInt8) { + self.red = red + self.green = green + self.blue = blue + } + + public var components: (red: Double, green: Double, blue: Double) { + (Double(red) / 255, Double(green) / 255, Double(blue) / 255) + } +} + +public enum StudioBackground: Codable, Sendable, Hashable { + case none + case color(StudioColor) + case gradient(from: StudioColor, to: StudioColor, angle: Double) + case wallpaper(String) + case image(String) + /// Soft blooms of colour, drawn at any size. See `StudioGlow`. + case glow(String) + + public var isNone: Bool { self == .none } + + public static let glows: [StudioBackground] = StudioGlow.all.map { .glow($0.id) } + + /// Cap desktop's gradient presets, in the same order. + public static let gradients: [StudioBackground] = [ + (StudioColor(71, 133, 255), StudioColor(255, 71, 102)), + (StudioColor(15, 52, 67), StudioColor(52, 232, 158)), + (StudioColor(34, 193, 195), StudioColor(253, 187, 45)), + (StudioColor(29, 253, 251), StudioColor(195, 29, 253)), + (StudioColor(69, 104, 220), StudioColor(176, 106, 179)), + (StudioColor(106, 130, 251), StudioColor(252, 92, 125)), + (StudioColor(131, 58, 180), StudioColor(253, 29, 29)), + (StudioColor(249, 212, 35), StudioColor(255, 78, 80)), + (StudioColor(255, 94, 0), StudioColor(255, 42, 104)), + (StudioColor(255, 0, 150), StudioColor(0, 204, 255)), + (StudioColor(0, 242, 96), StudioColor(5, 117, 230)), + (StudioColor(238, 205, 163), StudioColor(239, 98, 159)), + (StudioColor(44, 62, 80), StudioColor(52, 152, 219)), + (StudioColor(168, 239, 255), StudioColor(238, 205, 163)), + (StudioColor(74, 0, 224), StudioColor(143, 0, 255)), + (StudioColor(252, 74, 26), StudioColor(247, 183, 51)), + ].map { .gradient(from: $0.0, to: $0.1, angle: 135) } + + public static let colors: [StudioBackground] = [ + StudioColor(255, 255, 255), StudioColor(0, 0, 0), StudioColor(28, 28, 30), StudioColor(242, 242, 247), + StudioColor(71, 133, 255), StudioColor(255, 69, 0), StudioColor(255, 140, 0), StudioColor(255, 215, 0), + StudioColor(50, 205, 50), StudioColor(0, 206, 209), StudioColor(75, 0, 130), StudioColor(128, 0, 128), + ].map { .color($0) } +} + +/// A background in the style of the system wallpapers: a few large blooms of colour over a base, blurred together. +/// Drawn from these numbers, so it's sharp at any size and costs nothing to ship. +public struct StudioGlow: Sendable, Hashable, Identifiable { + public struct Bloom: Sendable, Hashable { + /// Centre in unit coordinates, origin top left. + public var x: Double + public var y: Double + /// As a share of the frame's longer side. + public var radius: Double + public var color: StudioColor + } + + public let id: String + public let base: StudioColor + public let blooms: [Bloom] + + init(_ id: String, base: StudioColor, _ blooms: [(Double, Double, Double, StudioColor)]) { + self.id = id + self.base = base + self.blooms = blooms.map { Bloom(x: $0.0, y: $0.1, radius: $0.2, color: $0.3) } + } + + /// How soft the blooms are, as a share of the frame's shorter side. + public static let softness: Double = 0.09 + + public static func named(_ id: String) -> StudioGlow? { + all.first { $0.id == id } + } + + private static func c(_ r: UInt8, _ g: UInt8, _ b: UInt8) -> StudioColor { StudioColor(r, g, b) } + + public static let all: [StudioGlow] = [ + StudioGlow("sky", base: c(206, 226, 255), [ + (0.12, 0.08, 0.62, c(110, 168, 255)), (0.92, 0.28, 0.55, c(168, 214, 255)), + (0.28, 0.96, 0.7, c(92, 140, 246)), (0.88, 0.9, 0.48, c(196, 186, 255)), + ]), + StudioGlow("mint", base: c(204, 243, 228), [ + (0.18, 0.16, 0.6, c(104, 214, 176)), (0.88, 0.34, 0.55, c(150, 232, 222)), + (0.36, 0.92, 0.68, c(70, 192, 196)), (0.94, 0.96, 0.42, c(196, 240, 168)), + ]), + StudioGlow("peach", base: c(255, 228, 208), [ + (0.1, 0.1, 0.6, c(255, 166, 138)), (0.94, 0.38, 0.55, c(255, 202, 160)), + (0.3, 0.92, 0.68, c(255, 136, 150)), (0.84, 0.96, 0.46, c(255, 214, 128)), + ]), + StudioGlow("lavender", base: c(232, 224, 255), [ + (0.14, 0.18, 0.6, c(184, 156, 255)), (0.92, 0.14, 0.52, c(255, 188, 238)), + (0.48, 0.88, 0.72, c(156, 166, 255)), (0.96, 0.8, 0.42, c(218, 178, 255)), + ]), + StudioGlow("sunset", base: c(255, 186, 124), [ + (0.1, 0.08, 0.58, c(255, 118, 82)), (0.92, 0.24, 0.56, c(255, 88, 124)), + (0.38, 0.96, 0.7, c(255, 202, 92)), (0.86, 0.86, 0.46, c(226, 80, 150)), + ]), + StudioGlow("dusk", base: c(38, 20, 68), [ + (0.1, 0.14, 0.62, c(236, 98, 146)), (0.9, 0.2, 0.56, c(142, 86, 250)), + (0.5, 0.98, 0.74, c(252, 150, 86)), (0.82, 0.7, 0.4, c(226, 82, 176)), + ]), + StudioGlow("aurora", base: c(6, 22, 30), [ + (0.14, 0.3, 0.58, c(28, 192, 146)), (0.72, 0.12, 0.58, c(38, 132, 214)), + (0.5, 0.84, 0.66, c(112, 58, 196)), (0.96, 0.62, 0.4, c(20, 212, 184)), + ]), + StudioGlow("ocean", base: c(6, 36, 78), [ + (0.18, 0.14, 0.62, c(22, 116, 200)), (0.86, 0.44, 0.56, c(12, 164, 186)), + (0.3, 0.92, 0.68, c(30, 66, 168)), (0.96, 0.96, 0.4, c(58, 196, 218)), + ]), + StudioGlow("midnight", base: c(8, 10, 28), [ + (0.2, 0.16, 0.66, c(30, 64, 190)), (0.86, 0.42, 0.56, c(86, 40, 182)), + (0.36, 0.96, 0.7, c(12, 104, 192)), (0.92, 0.94, 0.38, c(132, 58, 210)), + ]), + StudioGlow("graphite", base: c(40, 42, 48), [ + (0.18, 0.16, 0.62, c(104, 110, 124)), (0.86, 0.58, 0.58, c(26, 28, 34)), + (0.38, 0.96, 0.6, c(80, 86, 100)), (0.94, 0.08, 0.4, c(130, 134, 146)), + ]), + ] +} + +public enum StudioAspect: String, Codable, Sendable, CaseIterable, Identifiable { + case auto + case portrait + case fourFive + case square + case landscape + + public var id: String { rawValue } + + public var title: String { + switch self { + case .auto: "Original" + case .portrait: "9:16" + case .fourFive: "4:5" + case .square: "1:1" + case .landscape: "16:9" + } + } + + /// Width over height, or nil to follow the recording. + public var ratio: Double? { + switch self { + case .auto: nil + case .portrait: 9.0 / 16 + case .fourFive: 4.0 / 5 + case .square: 1 + case .landscape: 16.0 / 9 + } + } +} + +public enum StudioCorner: String, Codable, Sendable, CaseIterable { + case topLeading + case topTrailing + case bottomLeading + case bottomTrailing +} + +public enum StudioCameraShape: String, Codable, Sendable, CaseIterable, Identifiable { + case circle + case rounded + case square + + public var id: String { rawValue } +} + +public enum StudioCameraLayout: String, Codable, Sendable, CaseIterable, Identifiable { + case bubble + /// Screen above the camera (side by side when the frame is wide). + case split + case cameraOnly + case screenOnly + + public var id: String { rawValue } + + public var title: String { + switch self { + case .bubble: "Bubble" + case .split: "Split" + case .cameraOnly: "Camera" + case .screenOnly: "Screen" + } + } +} + +public struct StudioCameraLook: Codable, Sendable, Hashable { + public var layout: StudioCameraLayout = .bubble + public var corner: StudioCorner = .bottomTrailing + /// Diameter as a share of the frame's shorter side. + public var size: Double = 0.32 + public var shape: StudioCameraShape = .circle + + public init() {} + + public static let sizeRange: ClosedRange = 0.18...0.6 +} + +public struct StudioLook: Codable, Sendable, Hashable { + public var background: StudioBackground + /// Space around the video as a share of the frame's shorter side. + public var padding: Double + /// As a share of the video's shorter side. + public var rounding: Double + public var shadow: Double + public var aspect: StudioAspect + public var camera: StudioCameraLook + + public init(background: StudioBackground = .none, padding: Double = 0, rounding: Double = 0, shadow: Double = 0, aspect: StudioAspect = .auto, camera: StudioCameraLook = .init()) { + self.background = background + self.padding = padding + self.rounding = rounding + self.shadow = shadow + self.aspect = aspect + self.camera = camera + } + + public static let untouched = StudioLook() + /// What a new Studio recording starts with. + public static let studioDefault = StudioLook(background: StudioBackground.gradients[0], padding: 0.06, rounding: 0.06, shadow: 0.6) + + public static let paddingRange: ClosedRange = 0...0.2 + public static let roundingRange: ClosedRange = 0...0.2 + + /// True when rendering can't change a single pixel of the main video. + public func isIdentity(hasCamera: Bool) -> Bool { + background.isNone && aspect == .auto && padding == 0 && rounding == 0 && (!hasCamera || camera.layout == .screenOnly) + } + + /// Without a background the frame is the video itself, so padding, corners and shadow have nothing to show on. + public var effectivePadding: Double { background.isNone ? 0 : padding } + public var effectiveRounding: Double { background.isNone ? 0 : rounding } + public var effectiveShadow: Double { background.isNone ? 0 : shadow } +} + +public struct StudioMusic: Codable, Sendable, Hashable { + public enum Source: Codable, Sendable, Hashable { + case builtIn(String) + case file(String) + } + + public var source: Source + public var title: String + public var volume: Double + /// Seconds into the track where the video starts. + public var offset: Double + + public init(source: Source, title: String, volume: Double = 0.35, offset: Double = 0) { + self.source = source + self.title = title + self.volume = volume + self.offset = offset + } + + public static let fadeOut: Double = 1.5 +} + +/// One recording made in Studio Mode. It stays on the phone until it's shared, so it can be edited with nothing +/// re-encoded and nothing uploaded twice. +public struct StudioProject: Codable, Sendable, Hashable, Identifiable { + public struct Track: Codable, Sendable, Hashable { + public var file: String + /// Seconds from the project's start to the track's first frame. Screen and camera are recorded by different + /// processes, so each starts when its own first frame arrives. + public var offset: Double + public var duration: Double + /// Seconds from the track's first frame where it recorded nothing. Hidden rather than frozen. + public var gaps: [EditRange]? + /// Sound recorded beside the picture, in its own file. + public var audio: Audio? + + public struct Audio: Codable, Sendable, Hashable { + public var file: String + /// Seconds from the track's first frame to its first sound; negative when the sound started first. + public var offset: Double + public var duration: Double + + public init(file: String, offset: Double, duration: Double) { + self.file = file + self.offset = offset + self.duration = duration + } + } + + public init(file: String, offset: Double, duration: Double, gaps: [EditRange]? = nil, audio: Audio? = nil) { + self.file = file + self.offset = offset + self.duration = duration + self.gaps = gaps + self.audio = audio + } + + public var recorded: [EditRange] { + var ranges: [EditRange] = [] + var cursor = 0.0 + for gap in (gaps ?? []).sorted(by: { $0.start < $1.start }) where gap.end > cursor { + if gap.start > cursor { ranges.append(EditRange(start: cursor, end: min(gap.start, duration))) } + cursor = max(cursor, gap.end) + } + if duration > cursor { ranges.append(EditRange(start: cursor, end: duration)) } + return ranges.filter { $0.duration > 0.001 } + } + } + + public var id: String + public var title: String + public var createdAt: Date + public var screen: Track? + public var camera: Track? + public var look: StudioLook + public var music: StudioMusic? + public var voiceVolume: Double + /// Kept ranges in project time, the same model the Cap editor uses. + public var clips: [EditRange]? + public var capId: String? + + public init(id: String = UUID().uuidString.lowercased(), title: String, createdAt: Date = .now, screen: Track? = nil, camera: Track? = nil, look: StudioLook = .studioDefault, music: StudioMusic? = nil, voiceVolume: Double = 1, clips: [EditRange]? = nil, capId: String? = nil) { + self.id = id + self.title = title + self.createdAt = createdAt + self.screen = screen + self.camera = camera + self.look = look + self.music = music + self.voiceVolume = voiceVolume + self.clips = clips + self.capId = capId + } + + public var duration: Double { + max(screen.map { $0.offset + $0.duration } ?? 0, camera.map { $0.offset + $0.duration } ?? 0) + } + + public var isEmpty: Bool { screen == nil && camera == nil } + + public var timeline: EditTimeline { + if let clips { EditTimeline(sourceDuration: duration, clips: clips) } else { EditTimeline(sourceDuration: duration) } + } +} + +/// Where each piece of a frame goes, in pixels with the origin at the top left. Shared by the preview, the export and +/// the tests, so the three can't disagree. +public struct StudioLayout: Sendable, Equatable { + public enum Shape: Sendable, Equatable { + case rounded(radius: Double) + case circle + } + + public struct Placement: Sendable, Equatable { + /// Where the visible part goes. + public var frame: CGRect + public var shape: Shape + } + + public let canvas: CGSize + public let screen: Placement? + public let camera: Placement? + + /// The longest edge of an exported video. 1080p on the long side keeps phone recordings sharp at a sensible size. + public static let exportLongEdge: Double = 1920 + /// Small enough to composite fast while the editor plays, larger than any phone shows it. + public static let previewLongEdge: Double = 1280 + + /// The frame size for a look. `primary` is the shape of the screen (or the Cap's video), `camera` of the camera. + public static func canvasSize(look: StudioLook, primary: CGSize?, camera: CGSize?, longEdge: Double) -> CGSize { + let content = contentShape(look: look, primary: primary, camera: camera) + let natural: Double = content.height > 0 ? Double(content.width / content.height) : 9.0 / 16 + let ratio: Double = look.aspect.ratio ?? natural + // Never upscale the recording: a 720p video exports at 720p. + let sourceLong = Double(max(content.width, content.height)) + let long: Double = sourceLong > 0 ? min(longEdge, max(sourceLong, longEdge / 2)) : longEdge + var width: Double = ratio >= 1 ? long : long * ratio + var height: Double = ratio >= 1 ? long / ratio : long + // H.264 wants even sizes. + width = (width / 2).rounded() * 2 + height = (height / 2).rounded() * 2 + return CGSize(width: max(2, width), height: max(2, height)) + } + + private static func contentShape(look: StudioLook, primary: CGSize?, camera: CGSize?) -> CGSize { + switch look.camera.layout { + case .cameraOnly: camera ?? primary ?? .zero + case .split where primary != nil && camera != nil: + // Stacked, so the frame is as wide as the screen and twice as tall as half of it. + primary.map { CGSize(width: $0.width, height: $0.height) } ?? .zero + default: primary ?? camera ?? .zero + } + } + + public init(look: StudioLook, canvas: CGSize, primary: CGSize?, camera: CGSize?) { + self.canvas = canvas + let short: CGFloat = min(canvas.width, canvas.height) + let inset: CGFloat = CGFloat(look.effectivePadding) * short + let area = CGRect(origin: .zero, size: canvas).insetBy(dx: inset, dy: inset) + let roundingShare = CGFloat(look.effectiveRounding) + + func rounded(_ rect: CGRect) -> Shape { + .rounded(radius: Double(roundingShare * min(rect.width, rect.height))) + } + + var layout = look.camera.layout + if camera == nil { layout = .screenOnly } + if primary == nil { layout = .cameraOnly } + + switch layout { + case .screenOnly: + let frame = primary.map { Self.fit($0, in: area) } ?? .zero + screen = Placement(frame: frame, shape: rounded(frame)) + self.camera = nil + case .cameraOnly: + // Camera fills the space: a face is better cropped than letterboxed. + screen = nil + self.camera = Placement(frame: area, shape: rounded(area)) + case .split: + let gap: CGFloat = max(inset / 2, short * 0.015) + let wide = canvas.width > canvas.height + let first: CGRect + let second: CGRect + if wide { + let width: CGFloat = (area.width - gap) / 2 + first = CGRect(x: area.minX, y: area.minY, width: width, height: area.height) + second = CGRect(x: area.minX + width + gap, y: area.minY, width: width, height: area.height) + } else { + // The screen keeps its shape; the camera takes what's left, but never less than a third. + let topHeight: CGFloat = area.height * 2 / 3 - gap / 2 + let screenFit = primary.map { Self.fit($0, in: CGRect(x: area.minX, y: area.minY, width: area.width, height: topHeight)) } ?? .zero + let screenHeight: CGFloat = max(screenFit.height, 1) + first = CGRect(x: area.minX, y: area.minY, width: area.width, height: screenHeight) + second = CGRect(x: area.minX, y: area.minY + screenHeight + gap, width: area.width, height: max(1, area.height - screenHeight - gap)) + } + let screenFrame = primary.map { Self.fit($0, in: first) } ?? first + screen = Placement(frame: screenFrame, shape: rounded(screenFrame)) + // A split always has some rounding on the camera, or the two halves read as one video. + let cameraShape = Shape.rounded(radius: Double(max(roundingShare, 0.04) * min(second.width, second.height))) + self.camera = Placement(frame: second, shape: cameraShape) + case .bubble: + let frame = primary.map { Self.fit($0, in: area) } ?? .zero + screen = Placement(frame: frame, shape: rounded(frame)) + let side: CGFloat = CGFloat(min(max(look.camera.size, StudioCameraLook.sizeRange.lowerBound), StudioCameraLook.sizeRange.upperBound)) * short + let margin: CGFloat = max(short * 0.035, inset * 0.6) + // Over the frame, inside the padding when there is some, so the bubble sits on the video's corner. + let bounds = look.effectivePadding > 0 ? frame.insetBy(dx: -inset * 0.4, dy: -inset * 0.4) : CGRect(origin: .zero, size: canvas) + let x: CGFloat + let y: CGFloat + switch look.camera.corner { + case .topLeading, .bottomLeading: x = bounds.minX + margin + case .topTrailing, .bottomTrailing: x = bounds.maxX - margin - side + } + switch look.camera.corner { + case .topLeading, .topTrailing: y = bounds.minY + margin + case .bottomLeading, .bottomTrailing: y = bounds.maxY - margin - side + } + let bubble = CGRect(x: min(max(0, x), canvas.width - side), y: min(max(0, y), canvas.height - side), width: side, height: side) + let shape: Shape = switch look.camera.shape { + case .circle: .circle + case .rounded: .rounded(radius: Double(side * 0.22)) + case .square: .rounded(radius: Double(side * 0.04)) + } + self.camera = Placement(frame: bubble, shape: shape) + } + } + + public static func fit(_ size: CGSize, in area: CGRect) -> CGRect { + guard size.width > 0, size.height > 0, area.width > 0, area.height > 0 else { return area } + let scale = min(area.width / size.width, area.height / size.height) + let width = (size.width * scale).rounded() + let height = (size.height * scale).rounded() + return CGRect(x: (area.midX - width / 2).rounded(), y: (area.midY - height / 2).rounded(), width: width, height: height) + } + + public static func fill(_ size: CGSize, into rect: CGRect) -> (scale: Double, origin: CGPoint) { + guard size.width > 0, size.height > 0 else { return (1, rect.origin) } + let scale = max(rect.width / size.width, rect.height / size.height) + let origin = CGPoint(x: rect.midX - size.width * scale / 2, y: rect.midY - size.height * scale / 2) + return (scale, origin) + } + + public static func nearestCorner(to point: CGPoint, in canvas: CGSize) -> StudioCorner { + let left = point.x < canvas.width / 2 + let top = point.y < canvas.height / 2 + switch (top, left) { + case (true, true): return .topLeading + case (true, false): return .topTrailing + case (false, true): return .bottomLeading + case (false, false): return .bottomTrailing + } + } +} + +/// Studio projects live in the app group, so the screen recording extension can write into them. +public final class StudioStore: @unchecked Sendable { + public let root: URL + private let lock = NSLock() + + public init(root: URL) { + self.root = root + try? FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + var values = URLResourceValues() + values.isExcludedFromBackup = true + var url = root + try? url.setResourceValues(values) + } + + private static func base(appGroup: String) -> URL { + let container = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: appGroup) + ?? FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + return container.appending(path: "Studio", directoryHint: .isDirectory) + } + + /// One user's drafts. Each account has its own folder, so signing in as someone else never shows them. + public static func shared(owner: String, appGroup: String = "group.so.cap.mobile") -> StudioStore { + StudioStore(root: base(appGroup: appGroup).appending(path: "Users/\(folderName(for: owner))", directoryHint: .isDirectory)) + } + + /// Where the app leaves requests for the screen recording extension, outside every user's drafts. + public static func requests(appGroup: String = "group.so.cap.mobile") -> StudioStore { + StudioStore(root: base(appGroup: appGroup)) + } + + /// User ids are server data; only plain characters reach the file system. + static func folderName(for owner: String) -> String { + let allowed = owner.unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) && $0.isASCII || $0 == "_" || $0 == "-" } + let name = String(String.UnicodeScalarView(allowed)).prefix(80) + return name.isEmpty ? "unknown" : String(name) + } + + public func directory(for id: String) -> URL { + root.appending(path: id, directoryHint: .isDirectory) + } + + public func url(_ file: String, in id: String) -> URL { + directory(for: id).appending(path: file) + } + + private func manifest(for id: String) -> URL { + directory(for: id).appending(path: "project.json") + } + + public func save(_ project: StudioProject) throws { + lock.lock() + defer { lock.unlock() } + try FileManager.default.createDirectory(at: directory(for: project.id), withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + try encoder.encode(project).write(to: manifest(for: project.id), options: [.atomic, .completeFileProtectionUntilFirstUserAuthentication]) + } + + public func load(_ id: String) -> StudioProject? { + lock.lock() + defer { lock.unlock() } + guard let data = try? Data(contentsOf: manifest(for: id)) else { return nil } + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return try? decoder.decode(StudioProject.self, from: data) + } + + /// Newest first. Skips folders without a readable manifest, such as a recording still being written. + public func all() -> [StudioProject] { + let ids = (try? FileManager.default.contentsOfDirectory(atPath: root.path)) ?? [] + return ids.compactMap(load).filter { !$0.isEmpty }.sorted { $0.createdAt > $1.createdAt } + } + + public func delete(_ id: String) { + lock.lock() + defer { lock.unlock() } + try? FileManager.default.removeItem(at: directory(for: id)) + } + + /// Studio files are large: an hour of screen is a few gigabytes. + public func hasRoomToRecord(minimum: Int64 = 1_000_000_000) -> Bool { + #if os(iOS) + let free = (try? root.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]).volumeAvailableCapacityForImportantUsage) ?? Int64.max + #else + let free = Int64((try? root.resourceValues(forKeys: [.volumeAvailableCapacityKey]).volumeAvailableCapacity) ?? Int.max) + #endif + return free >= minimum + } + + public func size(of id: String) -> Int64 { + let directory = directory(for: id) + let files = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: [.fileSizeKey])) ?? [] + return files.reduce(0) { $0 + Int64((try? $1.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0) } + } +} + +/// The app asks the screen recording extension to record into a Studio project rather than upload. Written by the +/// app just before the system sheet opens, taken once by the extension. +public enum StudioScreenRequest { + public struct Request: Codable, Sendable, Equatable { + public var projectId: String + /// Whose drafts the recording joins. + public var ownerId: String + public var requestedAt: Date + + public init(projectId: String, ownerId: String, requestedAt: Date = .now) { + self.projectId = projectId + self.ownerId = ownerId + self.requestedAt = requestedAt + } + } + + /// A request older than this is from a sheet that was closed without recording. + public static let lifetime: TimeInterval = 10 * 60 + public static let finishedNotification = "so.cap.mobile.studio.screen-finished" + + private static func file(in store: StudioStore) -> URL { + store.root.appending(path: "screen-request.json") + } + + public static func set(_ request: Request?, in store: StudioStore) { + let url = file(in: store) + guard let request, let data = try? JSONEncoder().encode(request) else { + try? FileManager.default.removeItem(at: url) + return + } + try? data.write(to: url, options: .atomic) + } + + public static func peek(in store: StudioStore, now: Date = .now) -> Request? { + guard let data = try? Data(contentsOf: file(in: store)), + let request = try? JSONDecoder().decode(Request.self, from: data), + now.timeIntervalSince(request.requestedAt) < lifetime, now >= request.requestedAt.addingTimeInterval(-60) + else { return nil } + return request + } + + /// Returns the request at most once. + public static func take(in store: StudioStore, now: Date = .now) -> Request? { + let request = peek(in: store, now: now) + set(nil, in: store) + return request + } +} diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/StudioExporter.swift b/apps/mobile-ios/CapKit/Sources/CapKit/StudioExporter.swift new file mode 100644 index 00000000000..d89c60f3c3b --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/StudioExporter.swift @@ -0,0 +1,282 @@ +@preconcurrency import AVFoundation +import Foundation +import Synchronization + +/// Renders a composition to an H.264 MP4 at a bitrate chosen for sharing. `AVAssetExportSession` has no bitrate +/// control and its high-quality presets make files several times larger than a link needs. +public final class StudioExporter: @unchecked Sendable { + public enum Failure: Error, LocalizedError { + case cantRead + case cantWrite + case cancelled + + public var errorDescription: String? { + switch self { + case .cantRead, .cantWrite: "Cap couldn't render this video. Try again." + case .cancelled: "Rendering was cancelled." + } + } + } + + public struct Result: Sendable { + public let url: URL + public let duration: Double + public let seconds: Double + public let bytes: Int64 + public let size: CGSize + } + + /// About 0.1 bits per pixel per frame: crisp for screen text and faces at 30 fps. + public static func bitrate(for size: CGSize, fps: Double = 30) -> Int { + Int(min(16_000_000, max(2_000_000, size.width * size.height * fps * 0.1))) + } + + private let cancelled = Flag() + /// `cancel()` comes from another thread while the export sets this, so it's only touched under the lock. + private let lock = NSLock() + private var pumps: Pumps? + /// Tests only: runs just before the file is closed, to cancel at that exact moment. + nonisolated(unsafe) var willFinishWriting: (@Sendable () -> Void)? + + public init() {} + + /// Only asks the export to stop. The reader and writer aren't safe to use from two threads at once, so + /// cancelling them is left to the export, once every pump has stopped touching them. + public func cancel() { + _ = cancelled.raise() + lock.lock() + let pumps = pumps + lock.unlock() + pumps?.finishAll() + } + + private func hold(_ pumps: Pumps?) { + lock.lock() + self.pumps = pumps + lock.unlock() + } + + public func export(_ built: StudioComposition, to url: URL, progress: (@Sendable (Double) -> Void)? = nil) async throws -> Result { + let started = CFAbsoluteTimeGetCurrent() + try? FileManager.default.removeItem(at: url) + let asset = built.composition + let reader = try AVAssetReader(asset: asset) + let writer = try AVAssetWriter(outputURL: url, fileType: .mp4) + writer.shouldOptimizeForNetworkUse = true + + let videoTracks = try await asset.loadTracks(withMediaType: .video) + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + guard !videoTracks.isEmpty, let videoComposition = built.videoComposition else { throw Failure.cantRead } + + // The compositor draws BGRA; asking for the encoder's own YUV here halves what each queued frame holds. + let videoOutput = AVAssetReaderVideoCompositionOutput(videoTracks: videoTracks, videoSettings: [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelBufferIOSurfacePropertiesKey as String: [String: String](), + ]) + videoOutput.videoComposition = videoComposition + videoOutput.alwaysCopiesSampleData = false + guard reader.canAdd(videoOutput) else { throw Failure.cantRead } + reader.add(videoOutput) + + let size = built.renderSize + let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: Int(size.width), + AVVideoHeightKey: Int(size.height), + AVVideoCompressionPropertiesKey: [ + AVVideoAverageBitRateKey: Self.bitrate(for: size), + AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel, + AVVideoMaxKeyFrameIntervalDurationKey: 2, + AVVideoExpectedSourceFrameRateKey: 30, + ] as [String: Any], + ]) + videoInput.expectsMediaDataInRealTime = false + guard writer.canAdd(videoInput) else { throw Failure.cantWrite } + writer.add(videoInput) + + var audioPair: (AVAssetReaderAudioMixOutput, AVAssetWriterInput)? + if !audioTracks.isEmpty { + let audioOutput = AVAssetReaderAudioMixOutput(audioTracks: audioTracks, audioSettings: [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: 48_000, + AVNumberOfChannelsKey: 2, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false, + ]) + audioOutput.audioMix = built.audioMix + audioOutput.alwaysCopiesSampleData = false + let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 48_000, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 160_000, + ]) + audioInput.expectsMediaDataInRealTime = false + if reader.canAdd(audioOutput), writer.canAdd(audioInput) { + reader.add(audioOutput) + writer.add(audioInput) + audioPair = (audioOutput, audioInput) + } + } + + guard reader.startReading() else { throw reader.error ?? Failure.cantRead } + guard writer.startWriting() else { throw writer.error ?? Failure.cantWrite } + writer.startSession(atSourceTime: .zero) + + let total = max(built.duration, 0.001) + let cancelled = cancelled + nonisolated(unsafe) let pumpReader = reader + // A writer that is cancelled or fails stops asking for data, so a pump waiting for room would never hear + // again. Every pump can be ended from outside: by `cancel()`, or by the other pump failing. + let pumps = Pumps() + hold(pumps) + if cancelled.isRaised { pumps.finishAll() } + @Sendable func pump(_ output: AVAssetReaderOutput, into input: AVAssetWriterInput, label: String, reportsProgress: Bool) async { + nonisolated(unsafe) let output = output + nonisolated(unsafe) let input = input + let queue = DispatchQueue(label: "so.cap.mobile.studio.export.\(label)") + await withCheckedContinuation { (continuation: CheckedContinuation) in + guard let handle = pumps.add(continuation, on: queue) else { return } + input.requestMediaDataWhenReady(on: queue) { + // A pump only ends on its own queue, between passes, so once it has ended nothing here + // touches the reader or writer again. + if handle.isFinished { return } + while input.isReadyForMoreMediaData { + if cancelled.isRaised || pumpReader.status != .reading { + pumps.finishAll() + return + } + guard let sample = output.copyNextSampleBuffer() else { + input.markAsFinished() + handle.finish() + return + } + if reportsProgress { + let time = CMSampleBufferGetPresentationTimeStamp(sample).seconds + if time.isFinite { progress?(min(1, time / total)) } + } + if !input.append(sample) { + pumps.finishAll() + return + } + } + } + } + } + + // An encoder that fails on its own leaves both inputs waiting with nothing to report it. + nonisolated(unsafe) let watchedWriter = writer + let watchdog = Task.detached { + while !Task.isCancelled { + try? await Task.sleep(for: .milliseconds(250)) + if watchedWriter.status == .failed || watchedWriter.status == .cancelled { + pumps.finishAll() + return + } + } + } + defer { watchdog.cancel() } + + // Each output and input is only ever touched by its own pump. + nonisolated(unsafe) let videoFeed = (videoOutput as AVAssetReaderOutput, videoInput) + nonisolated(unsafe) let audioFeed = audioPair.map { ($0.0 as AVAssetReaderOutput, $0.1) } + await withTaskGroup(of: Void.self) { group in + group.addTask { await pump(videoFeed.0, into: videoFeed.1, label: "video", reportsProgress: true) } + if let audioFeed { + group.addTask { await pump(audioFeed.0, into: audioFeed.1, label: "audio", reportsProgress: false) } + } + } + hold(nil) + if writer.status == .failed { + reader.cancelReading() + throw writer.error ?? Failure.cantWrite + } + + if cancelled.isRaised { + reader.cancelReading() + writer.cancelWriting() + try? FileManager.default.removeItem(at: url) + throw Failure.cancelled + } + if reader.status == .failed { + writer.cancelWriting() + throw reader.error ?? Failure.cantRead + } + willFinishWriting?() + await writer.finishWriting() + // A cancel while the file was being closed still wins: the finished file is never handed on. + if cancelled.isRaised { + try? FileManager.default.removeItem(at: url) + throw Failure.cancelled + } + guard writer.status == .completed else { throw writer.error ?? Failure.cantWrite } + progress?(1) + let bytes = (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? Int64) ?? 0 + return Result(url: url, duration: built.duration, seconds: CFAbsoluteTimeGetCurrent() - started, bytes: bytes, size: size) + } +} + +final class Flag: Sendable { + private let value = Atomic(false) + + /// True for the call that set it. + func raise() -> Bool { + !value.exchange(true, ordering: .acquiringAndReleasing) + } + + var isRaised: Bool { value.load(ordering: .acquiring) } +} + +/// The exporter's feeding loops, each ended exactly once from whichever side gets there first. +final class Pumps: @unchecked Sendable { + final class Handle: @unchecked Sendable { + private let done = Flag() + fileprivate let continuation: CheckedContinuation + fileprivate let queue: DispatchQueue + + fileprivate init(_ continuation: CheckedContinuation, queue: DispatchQueue) { + self.continuation = continuation + self.queue = queue + } + + var isFinished: Bool { done.isRaised } + + /// From the pump's own queue. + func finish() { + if done.raise() { continuation.resume() } + } + + /// From anywhere: waits for a pass the pump is in the middle of to return first. + fileprivate func finishOnQueue() { + queue.async { self.finish() } + } + } + + private let lock = NSLock() + private var handles: [Handle] = [] + private var ended = false + + /// Nil, with the continuation already resumed, once everything has been ended. + func add(_ continuation: CheckedContinuation, on queue: DispatchQueue) -> Handle? { + let handle = Handle(continuation, queue: queue) + lock.lock() + let isEnded = ended + if !isEnded { handles.append(handle) } + lock.unlock() + if isEnded { + handle.finish() + return nil + } + return handle + } + + func finishAll() { + lock.lock() + ended = true + let all = handles + lock.unlock() + for handle in all { handle.finishOnQueue() } + } +} diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/StudioPresets.swift b/apps/mobile-ios/CapKit/Sources/CapKit/StudioPresets.swift new file mode 100644 index 00000000000..7e6eb37b0d0 --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/StudioPresets.swift @@ -0,0 +1,145 @@ +import Foundation + +/// A named look: background, frame and camera layout, applied in one tap. +public struct StudioPreset: Codable, Sendable, Hashable, Identifiable { + public var id: String + public var name: String + public var look: StudioLook + + public init(id: String = UUID().uuidString.lowercased(), name: String, look: StudioLook) { + self.id = id + self.name = name + self.look = look + } + + /// Cap's own presets can be used and made the default, but not changed or deleted. + public var isBuiltIn: Bool { id.hasPrefix(Self.builtInPrefix) } + + static let builtInPrefix = "cap." + + private static func look(_ background: StudioBackground, aspect: StudioAspect = .auto, padding: Double = 0.08, rounding: Double = 0.07, shadow: Double = 0.7, layout: StudioCameraLayout = .bubble, corner: StudioCorner = .bottomTrailing, size: Double = 0.32, shape: StudioCameraShape = .circle) -> StudioLook { + var camera = StudioCameraLook() + camera.layout = layout + camera.corner = corner + camera.size = size + camera.shape = shape + return StudioLook(background: background, padding: padding, rounding: rounding, shadow: shadow, aspect: aspect, camera: camera) + } + + public static let builtIn: [StudioPreset] = [ + StudioPreset(id: "cap.studio", name: "Studio", look: .studioDefault), + StudioPreset(id: "cap.sky", name: "Sky", look: look(.glow("sky"), padding: 0.08, rounding: 0.08, shadow: 0.45)), + StudioPreset(id: "cap.dusk", name: "Dusk", look: look(.glow("dusk"), corner: .bottomLeading)), + StudioPreset(id: "cap.ocean", name: "Ocean", look: look(.wallpaper("blue-3"))), + StudioPreset(id: "cap.ember", name: "Ember", look: look(.wallpaper("orange-5"))), + StudioPreset(id: "cap.nebula", name: "Nebula", look: look(.wallpaper("purple-4"), corner: .bottomLeading, shape: .rounded)), + StudioPreset(id: "cap.graphite", name: "Graphite", look: look(.wallpaper("dark-5"), padding: 0.07, rounding: 0.06, shadow: 0.85)), + StudioPreset(id: "cap.mint", name: "Mint", look: look(.glow("mint"), padding: 0.08, rounding: 0.08, shadow: 0.45, shape: .rounded)), + StudioPreset(id: "cap.paper", name: "Paper", look: look(.color(StudioColor(236, 236, 241)), padding: 0.07, rounding: 0.05, shadow: 0.35, shape: .rounded)), + StudioPreset(id: "cap.vertical", name: "Vertical", look: look(.wallpaper("dark-2"), aspect: .portrait, padding: 0.05, rounding: 0.05, shadow: 0.6, layout: .split)), + StudioPreset(id: "cap.square", name: "Square", look: look(StudioBackground.gradients[1], aspect: .square, padding: 0.06, rounding: 0.06, shadow: 0.6, size: 0.3)), + StudioPreset(id: "cap.widescreen", name: "Widescreen", look: look(.wallpaper("purple-2"), aspect: .landscape, padding: 0.06, rounding: 0.05, shadow: 0.7, size: 0.3)), + ] +} + +/// Someone's saved presets and the one new recordings start with. +public struct StudioPresetLibrary: Codable, Sendable, Hashable { + public var saved: [StudioPreset] + public var defaultId: String? + + public init(saved: [StudioPreset] = [], defaultId: String? = nil) { + self.saved = saved + self.defaultId = defaultId + } + + /// Yours first, newest first, then Cap's. + public var all: [StudioPreset] { saved + StudioPreset.builtIn } + + /// Falls back to Cap's Studio look when the chosen preset has been deleted. + public var defaultPreset: StudioPreset { + all.first { $0.id == defaultId } ?? StudioPreset.builtIn[0] + } + + /// The preset a look came from, if it hasn't been changed since. + public func preset(matching look: StudioLook) -> StudioPreset? { + all.first { $0.look == look } + } + + /// "My Look", then "My Look 2" and so on, so two presets never share a name. + public func uniqueName(_ name: String) -> String { + let base = name.trimmingCharacters(in: .whitespacesAndNewlines) + let root = base.isEmpty ? "My Look" : String(base.prefix(40)) + let taken = Set(all.map { $0.name.lowercased() }) + guard taken.contains(root.lowercased()) else { return root } + var index = 2 + while taken.contains("\(root) \(index)".lowercased()) { index += 1 } + return "\(root) \(index)" + } +} + +/// Presets sit beside a person's drafts rather than among them: every folder in the drafts is a recording, and one +/// with no tracks in it is cleared away as a failed recording. +public final class StudioPresetStore: @unchecked Sendable { + public let root: URL + private let lock = NSLock() + + public init(root: URL) { + self.root = root + } + + private var file: URL { root.appending(path: "presets.json") } + + public func load() -> StudioPresetLibrary { + lock.lock() + defer { lock.unlock() } + guard let data = try? Data(contentsOf: file) else { return StudioPresetLibrary() } + return (try? JSONDecoder().decode(StudioPresetLibrary.self, from: data)) ?? StudioPresetLibrary() + } + + public func save(_ library: StudioPresetLibrary) throws { + lock.lock() + defer { lock.unlock() } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + try JSONEncoder().encode(library).write(to: file, options: [.atomic, .completeFileProtectionUntilFirstUserAuthentication]) + removeUnusedImages(library) + } + + public func imageURL(_ name: String) -> URL { + root.appending(path: name) + } + + /// A photo background is copied in when its preset is saved, because the recording it came from can be deleted. + public func keepImage(named name: String, from source: URL) throws { + let target = imageURL(name) + guard !FileManager.default.fileExists(atPath: target.path) else { return } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + try FileManager.default.copyItem(at: source, to: target) + } + + /// Copies a preset's photo into the folder that's using it. Nothing to do for any other background. + public func provideImage(for look: StudioLook, in directory: URL) { + guard case let .image(name) = look.background else { return } + let target = directory.appending(path: name) + guard !FileManager.default.fileExists(atPath: target.path) else { return } + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try? FileManager.default.copyItem(at: imageURL(name), to: target) + } + + private func removeUnusedImages(_ library: StudioPresetLibrary) { + let used = Set(library.saved.compactMap { preset -> String? in + if case let .image(name) = preset.look.background { name } else { nil } + }) + let files = (try? FileManager.default.contentsOfDirectory(atPath: root.path)) ?? [] + for name in files where name != "presets.json" && !used.contains(name) { + try? FileManager.default.removeItem(at: root.appending(path: name)) + } + } +} + +extension StudioStore { + /// `Users/.presets`, next to `Users/`. Account folder names never contain a dot, so it can't be taken by + /// another account. + public var presets: StudioPresetStore { + StudioPresetStore(root: root.deletingLastPathComponent().appending(path: "\(root.lastPathComponent).presets", directoryHint: .isDirectory)) + } +} diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/StudioRenderer.swift b/apps/mobile-ios/CapKit/Sources/CapKit/StudioRenderer.swift new file mode 100644 index 00000000000..c10f3b0ad40 --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/StudioRenderer.swift @@ -0,0 +1,743 @@ +@preconcurrency import AVFoundation +import CoreImage +import CoreImage.CIFilterBuiltins +import Foundation +import ImageIO +import Metal +import QuartzCore +import Synchronization + +/// What one source contributes to a frame: its shape as recorded, and how it was rotated. +public struct StudioSourceInfo: Sendable, Equatable { + public var naturalSize: CGSize + public var transform: CGAffineTransform + + public init(naturalSize: CGSize, transform: CGAffineTransform = .identity) { + self.naturalSize = naturalSize + self.transform = transform + } + + public var displaySize: CGSize { + let rect = CGRect(origin: .zero, size: naturalSize).applying(transform) + return CGSize(width: abs(rect.width), height: abs(rect.height)) + } + + /// Video tracks only ever rotate by quarter turns. + var orientation: CGImagePropertyOrientation { + let t = transform + if t.a == 0, t.b == 1, t.c == -1, t.d == 0 { return .right } + if t.a == 0, t.b == -1, t.c == 1, t.d == 0 { return .left } + if t.a == -1, t.d == -1 { return .down } + return .up + } +} + +/// Everything about a frame that isn't the video itself. Changing the look while the editor is up only swaps this; +/// nothing is rebuilt and nothing is decoded again. +public final class StudioScene: @unchecked Sendable { + public struct State: Sendable { + public var look: StudioLook + public var screen: StudioSourceInfo? + public var camera: StudioSourceInfo? + public var backgroundImage: CGImage? + } + + private let state: Mutex<(value: State, generation: Int)> + /// Newest last. + private let frames = Mutex<[Frame]>([]) + public let renderer: StudioRenderer + + public init(look: StudioLook, screen: StudioSourceInfo?, camera: StudioSourceInfo?, backgroundImage: CGImage? = nil, renderer: StudioRenderer = .shared) { + state = Mutex((State(look: look, screen: screen, camera: camera, backgroundImage: backgroundImage), 0)) + self.renderer = renderer + } + + public var current: (state: State, generation: Int) { + state.withLock { ($0.value, $0.generation) } + } + + public func update(_ change: (inout State) -> Void) { + state.withLock { + change(&$0.value) + $0.generation += 1 + } + } + + /// A copy that never changes, for exporting while the editor carries on. + public func frozen() -> StudioScene { + let value = current.state + return StudioScene(look: value.look, screen: value.screen, camera: value.camera, backgroundImage: value.backgroundImage, renderer: renderer) + } + + // MARK: Last frames + + struct Frame: @unchecked Sendable { + let time: CMTime + let screen: CVPixelBuffer? + let camera: CVPixelBuffer? + let size: CGSize + } + + /// The compositor keeps the last few source frames, so a paused preview can be redrawn with a new look without + /// asking the decoder for anything. + /// Keeps the most recently composed frames, whatever their times: after a seek back, the frame just drawn is + /// the one a redraw needs, even though frames from later in the video were composed before it. + func remember(_ frame: Frame) { + frames.withLock { frames in + frames.removeAll { $0.time == frame.time } + frames.append(frame) + if frames.count > Self.rememberedFrames { frames.removeFirst(frames.count - Self.rememberedFrames) } + } + } + + static let rememberedFrames = 6 + + public func forgetFrames() { + frames.withLock { $0.removeAll() } + } + + public var rememberedTimes: [Double] { + frames.withLock { $0.map(\.time.seconds) } + } + + /// The source frames composed within `tolerance` of `time`, for drawing the same moment in another look. + public func sources(near time: CMTime, tolerance: Double) -> (screen: CVPixelBuffer?, camera: CVPixelBuffer?)? { + let frame = frames.withLock { frames in + frames.min { abs(CMTimeSubtract($0.time, time).seconds) < abs(CMTimeSubtract($1.time, time).seconds) } + } + guard let frame, abs(CMTimeSubtract(frame.time, time).seconds) <= tolerance else { return nil } + return (frame.screen, frame.camera) + } + + /// Redraws the remembered frame closest to `time` with the current look. Nil if nothing near it was decoded. + public func renderStill(near time: CMTime, tolerance: Double = 0.25) -> CVPixelBuffer? { + let frame = frames.withLock { frames in + frames.min { abs(CMTimeSubtract($0.time, time).seconds) < abs(CMTimeSubtract($1.time, time).seconds) } + } + guard let frame, abs(CMTimeSubtract(frame.time, time).seconds) <= tolerance else { return nil } + return renderer.render(scene: self, screen: frame.screen, camera: frame.camera, size: frame.size) + } +} + +/// Draws Studio frames with Core Image on the GPU. One shared instance: its caches are what make frames cheap. +public final class StudioRenderer: @unchecked Sendable { + public static let shared = StudioRenderer() + + public let context: CIContext + private let colorSpace = CGColorSpace(name: CGColorSpace.sRGB)! + private let cache = Mutex(nil) + /// The last glow drawn, kept apart from the frame cache: dragging padding or corners redraws the frame's + /// still parts on every step, and the glow under them hasn't changed. + private let glowCache = Mutex<(id: String, size: CGSize, image: CIImage)?>(nil) + private let poolLock = NSLock() + private var poolSize = CGSize.zero + private var pool: CVPixelBufferPool? + + /// The parts of a frame that don't move, drawn once per look and size: background, shadows. + private struct Cache { + let key: CacheKey + let base: CIImage + let layout: StudioLayout + let screenMask: CIImage? + let cameraMask: CIImage? + } + + private struct CacheKey: Equatable { + let look: StudioLook + let size: CGSize + let screen: CGSize? + let camera: CGSize? + let image: ObjectIdentifier? + } + + private init(context: CIContext) { + self.context = context + } + + /// Shares this renderer's GPU context and compiled kernels but keeps its own cache, so drawing other looks (the + /// preset thumbnails) never evicts the background and masks the live preview is using. + public func sibling() -> StudioRenderer { + StudioRenderer(context: context) + } + + public init(device: MTLDevice? = MTLCreateSystemDefaultDevice()) { + let options: [CIContextOption: Any] = [ + .cacheIntermediates: false, + .workingColorSpace: CGColorSpace(name: CGColorSpace.sRGB)!, + .outputColorSpace: CGColorSpace(name: CGColorSpace.sRGB)!, + // Half floats would double the bandwidth for no visible gain on 8-bit video. + .workingFormat: CIFormat.RGBA8, + .name: "Cap Studio", + ] + if let device { + context = CIContext(mtlDevice: device, options: options) + } else { + context = CIContext(options: options) + } + } + + /// Core Image compiles its kernels the first time a graph is drawn, which costs hundreds of milliseconds. Drawing + /// one throwaway frame of the same shape while the video loads moves that cost off the first frame people see, + /// and leaves the background and masks for this look already cached. + public func prewarm(scene: StudioScene, size: CGSize) { + let state = scene.current.state + func dummy(_ info: StudioSourceInfo?) -> CVPixelBuffer? { + guard info != nil else { return nil } + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 64, 64, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, [kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any]()] as CFDictionary, &buffer) + return buffer + } + _ = render(scene: scene, screen: dummy(state.screen), camera: dummy(state.camera), size: size) + } + + /// Renders into a pixel buffer from the renderer's own pool. For stills and tests; the compositor renders into + /// buffers AVFoundation hands it. + public func render(scene: StudioScene, screen: CVPixelBuffer?, camera: CVPixelBuffer?, size: CGSize) -> CVPixelBuffer? { + guard let output = makeBuffer(size: size) else { return nil } + render(scene: scene, screen: screen, camera: camera, into: output) + return output + } + + public func render(scene: StudioScene, screen: CVPixelBuffer?, camera: CVPixelBuffer?, into output: CVPixelBuffer) { + let size = CGSize(width: CVPixelBufferGetWidth(output), height: CVPixelBufferGetHeight(output)) + context.render(compose(scene: scene, screen: screen, camera: camera, size: size), to: output, bounds: CGRect(origin: .zero, size: size), colorSpace: colorSpace) + } + + public func prewarmImages(scene: StudioScene) { + let state = scene.current.state + let look = StudioScene(look: state.look, screen: nil, camera: nil, renderer: self) + _ = image(scene: look, screen: nil, camera: nil, size: CGSize(width: 32, height: 32)) + } + + /// For small pictures shown in the interface, at any size, without going through the video buffer pool. + public func image(scene: StudioScene, screen: CVPixelBuffer?, camera: CVPixelBuffer?, size: CGSize) -> CGImage? { + context.createCGImage(compose(scene: scene, screen: screen, camera: camera, size: size), from: CGRect(origin: .zero, size: size), format: .RGBA8, colorSpace: colorSpace) + } + + private func compose(scene: StudioScene, screen: CVPixelBuffer?, camera: CVPixelBuffer?, size: CGSize) -> CIImage { + let state = scene.current.state + let cache = cached(state: state, size: size) + var image = cache.base + if let placement = cache.layout.screen, let screen, let info = state.screen { + image = composite(screen, info: info, placement: placement, mask: cache.screenMask, canvas: size, over: image) + } + if let placement = cache.layout.camera, let camera, let info = state.camera { + image = composite(camera, info: info, placement: placement, mask: cache.cameraMask, canvas: size, over: image) + } + return image + } + + /// The look and size the cached frame parts were drawn for. For tests. + var cachedLook: (look: StudioLook, size: CGSize)? { + cache.withLock { $0.map { ($0.key.look, $0.key.size) } } + } + + private func composite(_ buffer: CVPixelBuffer, info: StudioSourceInfo, placement: StudioLayout.Placement, mask: CIImage?, canvas: CGSize, over below: CIImage) -> CIImage { + var source = CIImage(cvPixelBuffer: buffer) + if info.orientation != .up { source = source.oriented(info.orientation) } + let extent = source.extent + let slot = Self.flip(placement.frame, in: canvas) + let fill = StudioLayout.fill(extent.size, into: slot) + let placed = source + .transformed(by: CGAffineTransform(translationX: -extent.minX, y: -extent.minY) + .concatenating(CGAffineTransform(scaleX: fill.scale, y: fill.scale)) + .concatenating(CGAffineTransform(translationX: fill.origin.x, y: fill.origin.y))) + .cropped(to: slot) + guard let mask else { return placed.composited(over: below) } + return placed.applyingFilter("CIBlendWithAlphaMask", parameters: [ + kCIInputBackgroundImageKey: below, + kCIInputMaskImageKey: mask, + ]) + } + + private func cached(state: StudioScene.State, size: CGSize) -> Cache { + let key = CacheKey(look: state.look, size: size, screen: state.screen?.displaySize, camera: state.camera?.displaySize, image: state.backgroundImage.map(ObjectIdentifier.init)) + if let cache = cache.withLock({ $0 }), cache.key == key { return cache } + let layout = StudioLayout(look: state.look, canvas: size, primary: state.screen?.displaySize, camera: state.camera?.displaySize) + let screenMask = layout.screen.flatMap { mask(for: $0, canvas: size) } + let cameraMask = layout.camera.flatMap { mask(for: $0, canvas: size) } + var base = background(state.look.background, image: state.backgroundImage, size: size) + let strength = state.look.effectiveShadow + if strength > 0 { + for placement in [layout.screen, layout.camera].compactMap({ $0 }) { + base = shadow(for: placement, strength: strength, canvas: size).composited(over: base) + } + } + // Drawn once into a buffer of its own, so each frame samples one texture instead of re-running blurs. + let buffer = makeStaticBuffer(size: size) + if let buffer { + context.render(base.cropped(to: CGRect(origin: .zero, size: size)), to: buffer, bounds: CGRect(origin: .zero, size: size), colorSpace: colorSpace) + } + let flattened = buffer.map { CIImage(cvPixelBuffer: $0) } ?? base + let made = Cache(key: key, base: flattened, layout: layout, screenMask: screenMask, cameraMask: cameraMask) + cache.withLock { $0 = made } + return made + } + + private func mask(for placement: StudioLayout.Placement, canvas: CGSize) -> CIImage? { + let rect = Self.flip(placement.frame, in: canvas) + let radius: Double = switch placement.shape { + case let .rounded(radius): radius + case .circle: min(rect.width, rect.height) / 2 + } + guard radius >= 0.5 else { return nil } + let generator = CIFilter.roundedRectangleGenerator() + generator.extent = rect + generator.radius = Float(radius) + generator.color = CIColor.white + guard let shape = generator.outputImage else { return nil } + // The mask is drawn once into a texture, with a soft one-pixel edge, so curves stay smooth when scaled. + let softened = shape.applyingGaussianBlur(sigma: 0.6).cropped(to: CGRect(origin: .zero, size: canvas)) + guard let buffer = makeStaticBuffer(size: canvas) else { return softened } + context.render(softened.composited(over: CIImage(color: .clear).cropped(to: CGRect(origin: .zero, size: canvas))), to: buffer, bounds: CGRect(origin: .zero, size: canvas), colorSpace: colorSpace) + return CIImage(cvPixelBuffer: buffer) + } + + private func shadow(for placement: StudioLayout.Placement, strength: Double, canvas: CGSize) -> CIImage { + let rect = Self.flip(placement.frame, in: canvas) + let short = min(canvas.width, canvas.height) + let radius: Double = switch placement.shape { + case let .rounded(radius): radius + case .circle: min(rect.width, rect.height) / 2 + } + let generator = CIFilter.roundedRectangleGenerator() + generator.extent = rect.offsetBy(dx: 0, dy: -short * 0.012 * strength) + generator.radius = Float(radius) + generator.color = CIColor(red: 0, green: 0, blue: 0, alpha: 0.55 * strength) + let blur = short * 0.035 * (0.4 + strength) + return (generator.outputImage ?? CIImage.empty()).applyingGaussianBlur(sigma: blur).cropped(to: CGRect(origin: .zero, size: canvas)) + } + + private func background(_ background: StudioBackground, image: CGImage?, size: CGSize) -> CIImage { + let canvas = CGRect(origin: .zero, size: size) + switch background { + case .none: + return CIImage(color: .black).cropped(to: canvas) + case let .color(color): + let c = color.components + return CIImage(color: CIColor(red: c.red, green: c.green, blue: c.blue)).cropped(to: canvas) + case let .gradient(from, to, angle): + // The angle runs like CSS: 90 is left to right, 180 top to bottom. + let radians = (angle - 90) * .pi / 180 + let half = (abs(cos(radians)) * size.width + abs(sin(radians)) * size.height) / 2 + let centre = CGPoint(x: size.width / 2, y: size.height / 2) + let dx = cos(radians) * half + let dy = -sin(radians) * half + let gradient = CIFilter.linearGradient() + gradient.point0 = CGPoint(x: centre.x - dx, y: centre.y - dy) + gradient.point1 = CGPoint(x: centre.x + dx, y: centre.y + dy) + let a = from.components + let b = to.components + gradient.color0 = CIColor(red: a.red, green: a.green, blue: a.blue) + gradient.color1 = CIColor(red: b.red, green: b.green, blue: b.blue) + return (gradient.outputImage ?? CIImage(color: .black)).cropped(to: canvas) + case let .glow(id): + if let cached = glowCache.withLock({ $0 }), cached.id == id, cached.size == size { return cached.image } + guard let glow = StudioGlow.named(id) else { return CIImage(color: .black).cropped(to: canvas) } + let drawn = Self.glow(glow, size: size) + guard let buffer = makeStaticBuffer(size: size) else { return drawn } + context.render(drawn, to: buffer, bounds: canvas, colorSpace: colorSpace) + let image = CIImage(cvPixelBuffer: buffer) + glowCache.withLock { $0 = (id, size, image) } + return image + case .wallpaper, .image: + guard let image else { return CIImage(color: CIColor(red: 0.1, green: 0.1, blue: 0.12)).cropped(to: canvas) } + let picture = CIImage(cgImage: image) + let fill = StudioLayout.fill(picture.extent.size, into: canvas) + return picture + .transformed(by: CGAffineTransform(scaleX: fill.scale, y: fill.scale).concatenating(CGAffineTransform(translationX: fill.origin.x, y: fill.origin.y))) + .cropped(to: canvas) + } + } + + /// The blooms are laid down and blurred at an eighth of the size, then scaled up: smooth at any size for the + /// cost of a thumbnail, and only ever drawn once per look into the cached background. + static func glow(_ glow: StudioGlow, size: CGSize) -> CIImage { + let scale: CGFloat = 8 + let small = CGSize(width: max(8, (size.width / scale).rounded(.up)), height: max(8, (size.height / scale).rounded(.up))) + let area = CGRect(origin: .zero, size: small) + let long = max(small.width, small.height) + func color(_ c: StudioColor, alpha: CGFloat = 1) -> CIColor { + let parts = c.components + return CIColor(red: parts.red, green: parts.green, blue: parts.blue, alpha: alpha) + } + var image = CIImage(color: color(glow.base)).cropped(to: area) + for bloom in glow.blooms { + let gradient = CIFilter.radialGradient() + gradient.center = CGPoint(x: bloom.x * small.width, y: (1 - bloom.y) * small.height) + gradient.radius0 = 0 + gradient.radius1 = Float(bloom.radius * long) + gradient.color0 = color(bloom.color, alpha: 0.95) + gradient.color1 = color(bloom.color, alpha: 0) + if let output = gradient.outputImage { image = output.cropped(to: area).composited(over: image) } + } + let sigma = StudioGlow.softness * min(small.width, small.height) + let blurred = image.clampedToExtent().applyingGaussianBlur(sigma: sigma).cropped(to: area) + let scaled = blurred + .transformed(by: CGAffineTransform(scaleX: size.width / small.width, y: size.height / small.height)) + .cropped(to: CGRect(origin: .zero, size: size)) + // About one level of noise, so slow gradients don't step into bands in 8-bit video. + return scaled.applyingFilter("CIDither", parameters: [kCIInputIntensityKey: 0.06]).cropped(to: CGRect(origin: .zero, size: size)) + } + + /// Layout is top-left; Core Image is bottom-left. + static func flip(_ rect: CGRect, in canvas: CGSize) -> CGRect { + CGRect(x: rect.minX, y: canvas.height - rect.maxY, width: rect.width, height: rect.height) + } + + // MARK: Buffers + + nonisolated(unsafe) static let bufferAttributes: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferIOSurfacePropertiesKey as String: [:] as [String: Any], + kCVPixelBufferMetalCompatibilityKey as String: true, + ] + + private func makeBuffer(size: CGSize) -> CVPixelBuffer? { + poolLock.lock() + if poolSize != size || pool == nil { + var attributes = Self.bufferAttributes + attributes[kCVPixelBufferWidthKey as String] = Int(size.width) + attributes[kCVPixelBufferHeightKey as String] = Int(size.height) + var created: CVPixelBufferPool? + CVPixelBufferPoolCreate(kCFAllocatorDefault, [kCVPixelBufferPoolMinimumBufferCountKey: 3] as CFDictionary, attributes as CFDictionary, &created) + pool = created + poolSize = size + } + let current = pool + poolLock.unlock() + guard let current else { return nil } + var buffer: CVPixelBuffer? + CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, current, &buffer) + return buffer + } + + private func makeStaticBuffer(size: CGSize) -> CVPixelBuffer? { + var attributes = Self.bufferAttributes + attributes[kCVPixelBufferWidthKey as String] = Int(size.width) + attributes[kCVPixelBufferHeightKey as String] = Int(size.height) + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, Int(size.width), Int(size.height), kCVPixelFormatType_32BGRA, attributes as CFDictionary, &buffer) + return buffer + } +} + +// MARK: - Compositor + +/// Carries the scene to the compositor. AVFoundation creates compositors itself, so this is the only way in. +public final class StudioInstruction: NSObject, AVVideoCompositionInstructionProtocol, @unchecked Sendable { + public let timeRange: CMTimeRange + public let enablePostProcessing = false + public let containsTweening = true + public let requiredSourceTrackIDs: [NSValue]? + public let passthroughTrackID = kCMPersistentTrackID_Invalid + public let scene: StudioScene + public let screenTrack: CMPersistentTrackID? + public let cameraTrack: CMPersistentTrackID? + public let remembersFrames: Bool + + public init(timeRange: CMTimeRange, scene: StudioScene, screenTrack: CMPersistentTrackID?, cameraTrack: CMPersistentTrackID?, remembersFrames: Bool) { + self.timeRange = timeRange + self.scene = scene + self.screenTrack = screenTrack + self.cameraTrack = cameraTrack + self.remembersFrames = remembersFrames + requiredSourceTrackIDs = [screenTrack, cameraTrack].compactMap { $0 }.map { NSNumber(value: $0) } + } +} + +public final class StudioCompositor: NSObject, AVVideoCompositing, @unchecked Sendable { + + static let renderTimes = Mutex<[Double]>([]) + + public static func takeRenderTimes() -> [Double] { + renderTimes.withLock { times in + defer { times.removeAll() } + return times + } + } + + public var sourcePixelBufferAttributes: [String: any Sendable]? { + [ + kCVPixelBufferPixelFormatTypeKey as String: [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, kCVPixelFormatType_32BGRA], + kCVPixelBufferIOSurfacePropertiesKey as String: [String: String](), + kCVPixelBufferMetalCompatibilityKey as String: true, + ] + } + + public var requiredPixelBufferAttributesForRenderContext: [String: any Sendable] { + [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferIOSurfacePropertiesKey as String: [String: String](), + kCVPixelBufferMetalCompatibilityKey as String: true, + ] + } + + public func renderContextChanged(_ newRenderContext: AVVideoCompositionRenderContext) {} + + /// Up to three frames render at once, so decoding, compositing and encoding overlap. AVFoundation asks for frames + /// as fast as it can during an export; past three it waits here, which keeps memory to a few frames. + private static let slots = 3 + private let inFlight = DispatchSemaphore(value: StudioCompositor.slots) + private let queue = DispatchQueue(label: "so.cap.mobile.studio.compositor", qos: .userInteractive, attributes: .concurrent) + + public func startRequest(_ request: AVAsynchronousVideoCompositionRequest) { + inFlight.wait() + queue.async { [self] in + defer { inFlight.signal() } + guard let instruction = request.videoCompositionInstruction as? StudioInstruction, + let output = request.renderContext.newPixelBuffer() + else { + request.finish(with: NSError(domain: "so.cap.mobile.studio", code: 1)) + return + } + let started = CACurrentMediaTime() + let screen = instruction.screenTrack.flatMap { request.sourceFrame(byTrackID: $0) } + let camera = instruction.cameraTrack.flatMap { request.sourceFrame(byTrackID: $0) } + let scene = instruction.scene + scene.renderer.render(scene: scene, screen: screen, camera: camera, into: output) + if instruction.remembersFrames { + let size = CGSize(width: CVPixelBufferGetWidth(output), height: CVPixelBufferGetHeight(output)) + scene.remember(.init(time: request.compositionTime, screen: screen, camera: camera, size: size)) + } + let took = (CACurrentMediaTime() - started) * 1000 + Self.renderTimes.withLock { times in + times.append(took) + if times.count > 2000 { times.removeFirst(times.count - 2000) } + } + request.finish(withComposedVideoFrame: output) + } + } + + /// Requests already started finish; at most three, so there is little to gain from dropping them. + public func cancelAllPendingVideoCompositionRequests() {} +} + +// MARK: - Composition + +/// A source on the timeline: one video track (and its sound) starting `offset` seconds into the project. +public struct StudioTrackSource: @unchecked Sendable { + public let asset: AVAsset + public let video: AVAssetTrack + public let audio: AVAssetTrack? + public let duration: Double + public let offset: Double + public let info: StudioSourceInfo + /// Stretches of the track's own time with no pictures, which show as nothing rather than a frozen frame. + public let gaps: [EditRange] + /// Sound from its own file: where it starts relative to the picture, and how long it runs. + public let separateAudio: (asset: AVAsset, offset: Double, duration: Double)? + + public init(asset: AVAsset, video: AVAssetTrack, audio: AVAssetTrack?, duration: Double, offset: Double, info: StudioSourceInfo, gaps: [EditRange] = [], separateAudio: (asset: AVAsset, offset: Double, duration: Double)? = nil) { + self.asset = asset + self.video = video + self.audio = audio + self.duration = duration + self.offset = offset + self.info = info + self.gaps = gaps + self.separateAudio = separateAudio + } + + /// `audio` is the sound file recorded beside the picture, with its start relative to the first frame. + public static func load(url: URL, offset: Double, gaps: [EditRange] = [], audio: (url: URL, offset: Double, duration: Double)? = nil, precise: Bool = true) async throws -> StudioTrackSource { + let asset = AVURLAsset(url: url, options: [AVURLAssetPreferPreciseDurationAndTimingKey: precise]) + let source = try await load(asset: asset, offset: offset) + var soundTrack = source.audio + var separate: (asset: AVAsset, offset: Double, duration: Double)? + if let audio, FileManager.default.fileExists(atPath: audio.url.path) { + let soundAsset = AVURLAsset(url: audio.url, options: [AVURLAssetPreferPreciseDurationAndTimingKey: precise]) + if let track = try? await soundAsset.loadTracks(withMediaType: .audio).first { + soundTrack = track + separate = (soundAsset, audio.offset, audio.duration) + } + } + return StudioTrackSource(asset: source.asset, video: source.video, audio: soundTrack, duration: source.duration, offset: source.offset, info: source.info, gaps: gaps, separateAudio: separate) + } + + var pictured: [EditRange] { + StudioProject.Track(file: "", offset: offset, duration: duration, gaps: gaps).recorded + } + + public static func load(asset: AVAsset, offset: Double) async throws -> StudioTrackSource { + async let videos = asset.loadTracks(withMediaType: .video) + async let audios = asset.loadTracks(withMediaType: .audio) + async let length = asset.load(.duration) + guard let video = try await videos.first else { throw StudioComposition.Failure.noVideo } + let audio = try await audios.first + let duration = try await length.seconds + async let size = video.load(.naturalSize) + async let transform = video.load(.preferredTransform) + let info = StudioSourceInfo(naturalSize: try await size, transform: (try? await transform) ?? .identity) + return StudioTrackSource(asset: asset, video: video, audio: audio, duration: duration, offset: offset, info: info) + } +} + +public struct StudioMusicSource: @unchecked Sendable { + /// Held because a track only weakly references its asset; without it inserts fail with -12780. + public let asset: AVAsset + public let track: AVAssetTrack + public let duration: Double + public let volume: Double + public let offset: Double + + public init(asset: AVAsset, track: AVAssetTrack, duration: Double, volume: Double, offset: Double) { + self.asset = asset + self.track = track + self.duration = duration + self.volume = volume + self.offset = offset + } + + public static func load(url: URL, volume: Double, offset: Double) async throws -> StudioMusicSource { + let asset = AVURLAsset(url: url) + async let tracks = asset.loadTracks(withMediaType: .audio) + async let length = asset.load(.duration) + guard let track = try await tracks.first else { throw StudioComposition.Failure.noAudio } + return StudioMusicSource(asset: asset, track: track, duration: try await length.seconds, volume: volume, offset: offset) + } +} + +/// Builds the edited video as an `AVComposition`: only references to the recorded files, so a rebuild takes +/// milliseconds and nothing is encoded until export. +public struct StudioComposition: @unchecked Sendable { + public enum Failure: Error { case noVideo, noAudio } + + public static let screenTrackID: CMPersistentTrackID = 1 + public static let cameraTrackID: CMPersistentTrackID = 2 + + public let composition: AVMutableComposition + public let videoComposition: AVVideoComposition? + public let audioMix: AVAudioMix? + public let renderSize: CGSize + public let duration: Double + + /// `clips` are ranges of project time to keep, in order. `screen` is the main video: a screen recording, or the + /// Cap being edited. + public static func build( + clips: [EditRange], + screen: StudioTrackSource?, + camera: StudioTrackSource?, + music: StudioMusicSource?, + voiceVolume: Double, + scene: StudioScene, + longEdge: Double, + frameRate: Int32 = 30, + remembersFrames: Bool, + passthrough: Bool = false + ) throws -> StudioComposition { + let composition = AVMutableComposition() + let screenVideo = screen.flatMap { _ in composition.addMutableTrack(withMediaType: .video, preferredTrackID: screenTrackID) } + let cameraVideo = camera.flatMap { _ in composition.addMutableTrack(withMediaType: .video, preferredTrackID: cameraTrackID) } + let screenAudio = screen?.audio.flatMap { _ in composition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) } + let cameraAudio = camera?.audio.flatMap { _ in composition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) } + + var cursor = CMTime.zero + for clip in clips { + let start = CMTime(seconds: clip.start, preferredTimescale: 600) + let end = CMTime(seconds: clip.end, preferredTimescale: 600) + guard end > start else { continue } + if let screen, let screenVideo { try insert(screen, video: screenVideo, audio: screenAudio, from: start, to: end, at: cursor) } + if let camera, let cameraVideo { try insert(camera, video: cameraVideo, audio: cameraAudio, from: start, to: end, at: cursor) } + cursor = cursor + CMTimeSubtract(end, start) + } + let total = cursor + guard total > .zero, screenVideo != nil || cameraVideo != nil else { throw Failure.noVideo } + + var mixes: [AVMutableAudioMixInputParameters] = [] + for track in [screenAudio, cameraAudio].compactMap({ $0 }) where voiceVolume != 1 { + let parameters = AVMutableAudioMixInputParameters(track: track) + parameters.setVolume(Float(voiceVolume), at: .zero) + mixes.append(parameters) + } + if let music, music.duration > 0.5, let musicTrack = composition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) { + // Loops to cover the whole video, starting `offset` into the song. + var at = CMTime.zero + var from = CMTime(seconds: min(max(0, music.offset), music.duration - 0.5), preferredTimescale: 600) + let songEnd = CMTime(seconds: music.duration, preferredTimescale: 600) + while at < total { + let length = CMTimeMinimum(CMTimeSubtract(songEnd, from), CMTimeSubtract(total, at)) + guard length > .zero else { break } + try musicTrack.insertTimeRange(CMTimeRange(start: from, duration: length), of: music.track, at: at) + at = at + length + from = .zero + } + let parameters = AVMutableAudioMixInputParameters(track: musicTrack) + let fade = CMTime(seconds: min(StudioMusic.fadeOut, total.seconds / 3), preferredTimescale: 600) + parameters.setVolume(Float(music.volume), at: .zero) + parameters.setVolumeRamp(fromStartVolume: Float(music.volume), toEndVolume: 0, timeRange: CMTimeRange(start: CMTimeSubtract(total, fade), duration: fade)) + mixes.append(parameters) + } + let audioMix: AVMutableAudioMix? = mixes.isEmpty ? nil : { + let mix = AVMutableAudioMix() + mix.inputParameters = mixes + return mix + }() + + let state = scene.current.state + // An untouched video with nothing over it plays straight from the decoder, with no compositing at all. + if passthrough, cameraVideo == nil, let screen, let screenVideo { + screenVideo.preferredTransform = screen.info.transform + return StudioComposition(composition: composition, videoComposition: nil, audioMix: audioMix, renderSize: screen.info.displaySize, duration: total.seconds) + } + let renderSize = StudioLayout.canvasSize(look: state.look, primary: state.screen?.displaySize, camera: state.camera?.displaySize, longEdge: longEdge) + let instruction = StudioInstruction( + timeRange: CMTimeRange(start: .zero, duration: total), + scene: scene, + screenTrack: screenVideo.map { _ in screenTrackID }, + cameraTrack: cameraVideo.map { _ in cameraTrackID }, + remembersFrames: remembersFrames + ) + let videoComposition: AVVideoComposition + if #available(iOS 26, macOS 26, *) { + var configuration = AVVideoComposition.Configuration() + configuration.customVideoCompositorClass = StudioCompositor.self + configuration.frameDuration = CMTime(value: 1, timescale: frameRate) + configuration.renderSize = renderSize + configuration.instructions = [instruction] + videoComposition = AVVideoComposition(configuration: configuration) + } else { + let mutable = AVMutableVideoComposition() + mutable.customVideoCompositorClass = StudioCompositor.self + mutable.frameDuration = CMTime(value: 1, timescale: frameRate) + mutable.renderSize = renderSize + mutable.instructions = [instruction] + videoComposition = mutable + } + return StudioComposition(composition: composition, videoComposition: videoComposition, audioMix: audioMix, renderSize: renderSize, duration: total.seconds) + } + + /// Inserts the part of a source that overlaps project time `start.. overlapStart { + let pieceRange = CMTimeRange(start: CMTime(seconds: piece.start, preferredTimescale: 600), end: CMTime(seconds: piece.end, preferredTimescale: 600)) + let clipped = range.intersection(pieceRange) + guard clipped.duration > .zero else { continue } + try video.insertTimeRange(clipped, of: source.video, at: at + CMTimeSubtract(clipped.start, range.start)) + } + // Sound runs straight through: silence in a gap is real silence, and the voice must never shift. + if let audio, let track = source.audio { + if let separate = source.separateAudio { + // Its own file, which can start before or after the picture: place it by its own start. + let soundStart = CMTime(seconds: source.offset + separate.offset, preferredTimescale: 48_000) + let soundEnd = CMTime(seconds: source.offset + separate.offset + separate.duration, preferredTimescale: 48_000) + let from = CMTimeMaximum(start, soundStart) + let to = CMTimeMinimum(end, soundEnd) + if to > from { + let soundRange = CMTimeRange(start: CMTimeSubtract(from, soundStart), end: CMTimeSubtract(to, soundStart)) + try? audio.insertTimeRange(soundRange, of: track, at: cursor + CMTimeSubtract(from, start)) + } + } else { + try? audio.insertTimeRange(range, of: track, at: at) + } + } + } +} diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/StudioTrackWriter.swift b/apps/mobile-ios/CapKit/Sources/CapKit/StudioTrackWriter.swift new file mode 100644 index 00000000000..f381427e571 --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/StudioTrackWriter.swift @@ -0,0 +1,438 @@ +@preconcurrency import AVFoundation +import Foundation +import Synchronization + +/// What a recorder appends to: `SegmentWriter` uploads as it goes, `StudioTrackWriter` keeps one file on the phone. +public protocol RecordingSink: AnyObject, Sendable { + func appendVideo(_ buffer: CMSampleBuffer) + func appendAudio(_ buffer: CMSampleBuffer) + var duration: Double { get } + func finish(at stop: CMTime?, _ completion: @escaping @Sendable (Result) -> Void) + func cancel() +} + +extension SegmentWriter: RecordingSink {} + +/// Writes one track of a Studio recording to the phone, at a higher quality than Instant Mode uploads and with a +/// keyframe every second so the editor can land on any frame quickly. +/// +/// Picture and sound go to separate files. In one file the writer keeps them interleaved, so when sound stops coming +/// (the microphone muted mid-recording, an audio route change, an interruption) it stops taking pictures a second +/// later and the recording silently freezes. Apart, each runs on its own, and each notes the host clock time of its +/// first sample so they line up again in the editor. +/// +/// Append from one serial queue. `clock` is the clock samples are timed on when it isn't the host clock, such as a +/// capture session's. +public final class StudioTrackWriter: RecordingSink, @unchecked Sendable { + public struct Settings: Sendable { + public enum Kind: Sendable { + case screen + case camera + } + + public var kind: Kind + public var fps: Int + public var audio: Bool + + public init(kind: Kind, fps: Int = 30, audio: Bool) { + self.kind = kind + self.fps = fps + self.audio = audio + } + + /// HEVC at these rates keeps screen text and faces clean through an edit and a second encode, at about + /// 2.7 GB an hour for a full-resolution iPhone screen and 1.8 GB for the 720p camera. + public func bitrate(width: Int, height: Int) -> Int { + switch kind { + case .screen: min(6_000_000, max(2_500_000, width * height * 2)) + case .camera: min(6_000_000, max(2_000_000, width * height * 4)) + } + } + } + + public enum Failure: Error, LocalizedError { + case cantWrite + + public var errorDescription: String? { "Cap couldn't save the recording. Free up some space and try again." } + } + + /// A pause longer than this in the frames is a gap (the camera was interrupted), not a slow frame. + public static let gapThreshold: Double = 0.5 + /// Missing sound longer than this is filled with silence, so what comes after stays where it was said. + static let audioGapThreshold: Double = 0.1 + + public let url: URL + public let audioURL: URL + private let settings: Settings + private let writer: AVAssetWriter + private var video: AVAssetWriterInput? + private let audioWriter: AVAssetWriter? + private let audio: AVAssetWriterInput? + private let frameDuration: CMTime + + // Append-queue state. + private var started: CMTime? + private var lastVideo: CMTime = .invalid + private var lastSample: CMSampleBuffer? + private var audioStarted: CMTime? + private var audioEnd: CMTime = .invalid + private var finished = false + /// Off once either file fails: nothing more is appended, but `finish` still closes whatever is healthy. + private var accepting = true + private let state = Mutex(State()) + private let clock: CMClock? + private let onStart: (@Sendable (StudioTrackInfo) -> Void)? + + private struct State { + var start: CMTime? + var hostStart: Double? + var end: CMTime = .zero + var gaps: [EditRange] = [] + var audioHostStart: Double? + var audioDuration: Double = 0 + var failed = false + } + + /// `onStart` is called when the picture starts and again when the sound does, with what's known so far, so a + /// process that dies mid-recording has still noted where its track begins. + public init(url: URL, settings: Settings, clock: CMClock? = nil, onStart: (@Sendable (StudioTrackInfo) -> Void)? = nil) throws { + self.url = url + audioURL = url.deletingPathExtension().appendingPathExtension("audio.m4a") + self.settings = settings + self.clock = clock + self.onStart = onStart + try? FileManager.default.removeItem(at: url) + try? FileManager.default.removeItem(at: audioURL) + writer = try AVAssetWriter(outputURL: url, fileType: .mp4) + // A fragment every second: if the app or extension is killed mid-recording, all but the last second plays. + writer.movieFragmentInterval = CMTime(value: 1, timescale: 1) + writer.shouldOptimizeForNetworkUse = false + frameDuration = CMTime(value: 1, timescale: CMTimeScale(settings.fps)) + if settings.audio { + let audioWriter = try AVAssetWriter(outputURL: audioURL, fileType: .m4a) + audioWriter.movieFragmentInterval = CMTime(value: 1, timescale: 1) + let input = AVAssetWriterInput(mediaType: .audio, outputSettings: [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 48_000, + AVNumberOfChannelsKey: 1, + AVEncoderBitRateKey: 128_000, + ]) + input.expectsMediaDataInRealTime = true + guard audioWriter.canAdd(input) else { throw Failure.cantWrite } + audioWriter.add(input) + self.audioWriter = audioWriter + audio = input + } else { + audioWriter = nil + audio = nil + } + } + + private func makeVideoInput(for buffer: CMSampleBuffer) -> AVAssetWriterInput? { + guard let pixels = CMSampleBufferGetImageBuffer(buffer) else { return nil } + // HEVC wants even sizes. + let width = CVPixelBufferGetWidth(pixels) & ~1 + let height = CVPixelBufferGetHeight(pixels) & ~1 + let input = AVAssetWriterInput(mediaType: .video, outputSettings: [ + // Every phone iOS 26 runs on encodes HEVC in hardware. + AVVideoCodecKey: AVVideoCodecType.hevc, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + AVVideoCompressionPropertiesKey: [ + AVVideoAverageBitRateKey: settings.bitrate(width: width, height: height), + AVVideoExpectedSourceFrameRateKey: settings.fps, + AVVideoMaxKeyFrameIntervalDurationKey: 1, + // No reordered frames: decoding any one frame never needs a later one, so seeks are cheaper. + AVVideoAllowFrameReorderingKey: false, + ] as [String: Any], + ]) + input.expectsMediaDataInRealTime = true + guard writer.canAdd(input) else { return nil } + writer.add(input) + return input + } + + /// For feeding a file faster than real time, as tests do. + var isReadyForVideo: Bool { video?.isReadyForMoreMediaData ?? true } + var isReadyForAudio: Bool { audioStarted == nil || audio?.isReadyForMoreMediaData ?? true } + + /// Host clock seconds of the first frame, once there is one. Screen and camera tracks are lined up by this. + public var hostStart: Double? { state.withLock { $0.hostStart } } + + /// Stretches with no frames, in seconds from the first frame. + public var gaps: [EditRange] { state.withLock { $0.gaps } } + + /// The picture stopped taking frames, for example because the disk filled. What was written before still plays. + public var failed: Bool { state.withLock { $0.failed } } + + public var info: StudioTrackInfo? { + state.withLock { value in Self.info(value, file: url.lastPathComponent, audioFile: audioURL.lastPathComponent) } + } + + private static func info(_ value: State, file: String, audioFile: String) -> StudioTrackInfo? { + guard let start = value.start, let hostStart = value.hostStart else { return nil } + var info = StudioTrackInfo(file: file, hostStart: hostStart, duration: max(0, CMTimeSubtract(value.end, start).seconds), gaps: value.gaps.isEmpty ? nil : value.gaps) + if let audioHostStart = value.audioHostStart { + info.audio = .init(file: audioFile, hostStart: audioHostStart, duration: value.audioDuration) + } + return info + } + + static func hostSeconds(_ time: CMTime, clock: CMClock?) -> Double { + let host = CMClockGetHostTimeClock() + guard let clock, clock !== host else { return time.seconds } + return CMSyncConvertTime(time, from: clock, to: host).seconds + } + + public var duration: Double { + state.withLock { value in + guard let start = value.start else { return 0 } + return max(0, CMTimeSubtract(value.end, start).seconds) + } + } + + private func reportStart() { + guard let onStart, let info else { return } + onStart(info) + } + + public func appendVideo(_ buffer: CMSampleBuffer) { + guard !finished, accepting else { return } + let time = CMSampleBufferGetPresentationTimeStamp(buffer) + guard time.isValid, !lastVideo.isValid || time > lastVideo else { return } + if started == nil { + guard let input = makeVideoInput(for: buffer), writer.startWriting() else { + markFailed() + return + } + video = input + writer.startSession(atSourceTime: time) + started = time + let hostStart = Self.hostSeconds(time, clock: clock) + state.withLock { + $0.start = time + $0.hostStart = hostStart + $0.end = CMTimeAdd(time, frameDuration) + } + reportStart() + } + guard writer.status == .writing, let video else { + if writer.status == .failed { markFailed() } + return + } + // A real-time input that isn't ready drops the frame; the next one carries on at its own time. + guard video.isReadyForMoreMediaData else { return } + if video.append(buffer) { + // Only a camera can be interrupted. A still screen sends no frames and is held by repeats that can be + // late, and it must keep showing. + if case .camera = settings.kind, lastVideo.isValid, let started, CMTimeSubtract(time, lastVideo).seconds > Self.gapThreshold { + let gap = EditRange(start: CMTimeSubtract(CMTimeAdd(lastVideo, frameDuration), started).seconds, end: CMTimeSubtract(time, started).seconds) + state.withLock { $0.gaps.append(gap) } + } + lastVideo = time + lastSample = buffer + let end = CMTimeAdd(time, frameDuration) + state.withLock { $0.end = end } + } else if writer.status == .failed { + markFailed() + } + } + + func markFailed() { + accepting = false + state.withLock { $0.failed = true } + } + + public func appendAudio(_ buffer: CMSampleBuffer) { + guard !finished, accepting, let audio, let audioWriter else { return } + let time = CMSampleBufferGetPresentationTimeStamp(buffer) + let length = CMSampleBufferGetDuration(buffer) + guard time.isValid else { return } + if audioStarted == nil { + // A sound file that can't be written stops the recording like a picture that can't, keeping what's saved. + guard audioWriter.startWriting() else { + markFailed() + return + } + audioWriter.startSession(atSourceTime: time) + audioStarted = time + audioEnd = time + let hostStart = Self.hostSeconds(time, clock: clock) + state.withLock { $0.audioHostStart = hostStart } + reportStart() + } + guard audioWriter.status == .writing else { + if audioWriter.status == .failed { markFailed() } + return + } + // Overlapping or repeated sound would push everything after it later. + if audioEnd.isValid, length.isValid, CMTimeAdd(time, length) <= audioEnd { return } + if audioEnd.isValid, CMTimeSubtract(time, audioEnd).seconds > Self.audioGapThreshold { + fillSilence(from: audioEnd, to: time, like: buffer) + } + guard audio.isReadyForMoreMediaData else { return } + guard audio.append(buffer) else { + if audioWriter.status == .failed { markFailed() } + return + } + audioEnd = length.isValid ? CMTimeAdd(time, length) : time + if let audioStarted { + let duration = CMTimeSubtract(audioEnd, audioStarted).seconds + state.withLock { $0.audioDuration = duration } + } + } + + /// Silence in the same format as the sound around it, in one-second pieces. + private func fillSilence(from start: CMTime, to end: CMTime, like buffer: CMSampleBuffer) { + guard let audio, let format = CMSampleBufferGetFormatDescription(buffer), + let description = CMAudioFormatDescriptionGetStreamBasicDescription(format)?.pointee, + description.mFormatID == kAudioFormatLinearPCM, description.mSampleRate > 0, description.mBytesPerFrame > 0 + else { return } + let rate = description.mSampleRate + let planes = description.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 ? Int(description.mChannelsPerFrame) : 1 + var cursor = start + // At most ten minutes: anything longer is a clock jump, not a pause. + var remaining = min(Int((CMTimeSubtract(end, start).seconds * rate).rounded()), Int(rate * 600)) + while remaining > 0 { + let frames = min(remaining, Int(rate)) + let bytes = frames * Int(description.mBytesPerFrame) * planes + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock(allocator: nil, memoryBlock: nil, blockLength: bytes, blockAllocator: nil, customBlockSource: nil, offsetToData: 0, dataLength: bytes, flags: kCMBlockBufferAssureMemoryNowFlag, blockBufferOut: &block) == noErr, let block, + CMBlockBufferFillDataBytes(with: 0, blockBuffer: block, offsetIntoDestination: 0, dataLength: bytes) == noErr + else { return } + var silence: CMSampleBuffer? + guard CMAudioSampleBufferCreateReadyWithPacketDescriptions(allocator: nil, dataBuffer: block, formatDescription: format, sampleCount: frames, presentationTimeStamp: cursor, packetDescriptions: nil, sampleBufferOut: &silence) == noErr, let silence, + audio.isReadyForMoreMediaData, audio.append(silence) + else { return } + cursor = CMTimeAdd(cursor, CMTime(value: CMTimeValue(frames), timescale: CMTimeScale(rate))) + audioEnd = cursor + remaining -= frames + } + } + + /// `stop` carries a still screen, which sends no new frames, to the moment recording stopped. + public func finish(at stop: CMTime? = nil, _ completion: @escaping @Sendable (Result) -> Void) { + guard !finished else { + completion(.success(duration)) + return + } + finished = true + // A failed file is left as it is: its fragments up to the failure still play, and cancelling would delete it. + guard started != nil, let video, writer.status == .writing else { + let failed = writer.status == .failed + let error = writer.error + finishAudio { audioError in + if failed { + completion(.failure(error ?? Failure.cantWrite)) + } else if let audioError { + completion(.failure(audioError)) + } else { + completion(.success(self.duration)) + } + } + return + } + if let stop, let lastSample, lastVideo.isValid, CMTimeSubtract(stop, lastVideo).seconds > frameDuration.seconds * 2, + let held = SegmentWriter.retime(lastSample, by: CMTimeSubtract(stop, lastVideo)), video.isReadyForMoreMediaData, video.append(held) { + state.withLock { $0.end = CMTimeAdd(stop, frameDuration) } + } + lastSample = nil + let end = state.withLock { $0.end } + video.markAsFinished() + writer.endSession(atSourceTime: end) + nonisolated(unsafe) let writer = writer + writer.finishWriting { [self] in + let status = writer.status + let error = writer.error + finishAudio { audioError in + if status == .completed, audioError == nil { + completion(.success(self.duration)) + } else { + completion(.failure(error ?? audioError ?? Failure.cantWrite)) + } + } + } + } + + /// Reports the sound file's error, if closing it failed. Its fragments up to then still play. + private func finishAudio(_ done: @escaping @Sendable ((any Error)?) -> Void) { + guard let audioWriter, let audio, audioStarted != nil, audioWriter.status == .writing else { + let failed = audioWriter?.status == .failed + done(failed ? (audioWriter?.error ?? Failure.cantWrite) : nil) + return + } + audio.markAsFinished() + if audioEnd.isValid { audioWriter.endSession(atSourceTime: audioEnd) } + nonisolated(unsafe) let closing = audioWriter + closing.finishWriting { done(closing.status == .completed ? nil : (closing.error ?? Failure.cantWrite)) } + } + + public func cancel() { + finished = true + lastSample = nil + if writer.status == .writing { writer.cancelWriting() } + if let audioWriter, audioWriter.status == .writing { audioWriter.cancelWriting() } + try? FileManager.default.removeItem(at: url) + try? FileManager.default.removeItem(at: audioURL) + } +} + +/// Written next to a track by whichever process recorded it, so the app can line tracks up afterwards. +public struct StudioTrackInfo: Codable, Sendable, Equatable { + public struct Audio: Codable, Sendable, Equatable { + public var file: String + /// Host clock seconds of the first sound. + public var hostStart: Double + /// Zero until finished, as for the picture. + public var duration: Double + + public init(file: String, hostStart: Double, duration: Double) { + self.file = file + self.hostStart = hostStart + self.duration = duration + } + } + + public var file: String + /// Host clock seconds of the first frame. + public var hostStart: Double + /// Zero until the track is finished: a track whose process died is measured from its file instead. + public var duration: Double + /// Seconds from the first frame where nothing was recorded, such as a camera iOS paused. + public var gaps: [EditRange]? + public var audio: Audio? + + public init(file: String, hostStart: Double, duration: Double, gaps: [EditRange]? = nil, audio: Audio? = nil) { + self.file = file + self.hostStart = hostStart + self.duration = duration + self.gaps = gaps + self.audio = audio + } + + public static func url(for track: String, in directory: URL) -> URL { + directory.appending(path: "\(track).json") + } + + public func save(as track: String, in directory: URL) throws { + try JSONEncoder().encode(self).write(to: Self.url(for: track, in: directory), options: .atomic) + } + + public static func load(_ track: String, in directory: URL) -> StudioTrackInfo? { + guard let data = try? Data(contentsOf: url(for: track, in: directory)) else { return nil } + return try? JSONDecoder().decode(StudioTrackInfo.self, from: data) + } + + /// Lines up whatever was recorded: the earliest picture starts the project at zero, and each track's sound sits + /// where it was heard relative to its picture. + public static func align(screen: StudioTrackInfo?, camera: StudioTrackInfo?) -> (screen: StudioProject.Track?, camera: StudioProject.Track?) { + let usable = [screen, camera].compactMap { $0 }.filter { $0.duration > 0 } + guard let origin = usable.map(\.hostStart).min() else { return (nil, nil) } + func track(_ info: StudioTrackInfo?) -> StudioProject.Track? { + guard let info, info.duration > 0 else { return nil } + let audio = info.audio.flatMap { $0.duration > 0 ? StudioProject.Track.Audio(file: $0.file, offset: $0.hostStart - info.hostStart, duration: $0.duration) : nil } + return StudioProject.Track(file: info.file, offset: max(0, info.hostStart - origin), duration: info.duration, gaps: info.gaps, audio: audio) + } + return (track(screen), track(camera)) + } +} diff --git a/apps/mobile-ios/CapKit/Sources/CapKit/StudioWaveform.swift b/apps/mobile-ios/CapKit/Sources/CapKit/StudioWaveform.swift new file mode 100644 index 00000000000..3e842c47eee --- /dev/null +++ b/apps/mobile-ios/CapKit/Sources/CapKit/StudioWaveform.swift @@ -0,0 +1,49 @@ +@preconcurrency import AVFoundation +import Foundation + +public enum StudioWaveform { + /// Loudest sample in each of `count` equal slices, 0...1. Decoded at a low rate in mono: a few milliseconds of + /// work per minute of music. + public static func peaks(url: URL, count: Int) async -> [Float] { + guard count > 0 else { return [] } + let asset = AVURLAsset(url: url) + guard let track = try? await asset.loadTracks(withMediaType: .audio).first, + let duration = try? await asset.load(.duration).seconds, duration > 0, + let reader = try? AVAssetReader(asset: asset) + else { return [] } + let rate = 8000.0 + let output = AVAssetReaderTrackOutput(track: track, outputSettings: [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false, + AVNumberOfChannelsKey: 1, + AVSampleRateKey: rate, + ]) + output.alwaysCopiesSampleData = false + guard reader.canAdd(output) else { return [] } + reader.add(output) + guard reader.startReading() else { return [] } + let total = max(1, Int(duration * rate)) + let perBucket = max(1, total / count) + var peaks = [Float](repeating: 0, count: count) + var index = 0 + while let buffer = output.copyNextSampleBuffer(), let block = CMSampleBufferGetDataBuffer(buffer) { + var length = 0 + var pointer: UnsafeMutablePointer? + guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length, dataPointerOut: &pointer) == noErr, let pointer else { continue } + let samples = length / 2 + pointer.withMemoryRebound(to: Int16.self, capacity: samples) { values in + for offset in 0.. peaks[bucket] { peaks[bucket] = level } + index += 1 + } + } + } + let loudest = peaks.max() ?? 0 + return loudest > 0 ? peaks.map { $0 / loudest } : peaks + } +} diff --git a/apps/mobile-ios/CapKit/Tests/CapKitTests/RecordingTests.swift b/apps/mobile-ios/CapKit/Tests/CapKitTests/RecordingTests.swift index d9a9ed13c0f..db66070b570 100644 --- a/apps/mobile-ios/CapKit/Tests/CapKitTests/RecordingTests.swift +++ b/apps/mobile-ios/CapKit/Tests/CapKitTests/RecordingTests.swift @@ -243,9 +243,12 @@ final class CompletionLog: Sendable { // A file may only be gone once job.json on disk already says it was uploaded, or a crash in // between would find a pending segment with nothing to upload. + // Files are looked at before job.json: a file gone by then must be marked uploaded in any later save, whereas + // reading job.json first can race an upload finishing between the two reads. func checkInvariant() throws { + let missing = segments.filter { !FileManager.default.fileExists(atPath: directory.appending(path: $0.fileName).path(percentEncoded: false)) } let saved = try? CapJSON.decoder().decode(RecordingJob.self, from: Data(contentsOf: directory.appending(path: "job.json"))) - for segment in segments where !FileManager.default.fileExists(atPath: directory.appending(path: segment.fileName).path(percentEncoded: false)) { + for segment in missing { let state = saved?.segments.first { $0.subpath == segment.subpath }?.state #expect(state == .uploaded, "\(segment.fileName) was deleted before its upload was saved") } diff --git a/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioPresetsTests.swift b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioPresetsTests.swift new file mode 100644 index 00000000000..be69ad2cdb6 --- /dev/null +++ b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioPresetsTests.swift @@ -0,0 +1,222 @@ +@testable import CapKit +import CoreImage +import Foundation +import Testing + +@Suite struct StudioPresetsTests { + private func temporaryStore() -> StudioStore { + StudioStore(root: FileManager.default.temporaryDirectory.appending(path: "presets-\(UUID().uuidString)/Users/user_a", directoryHint: .isDirectory)) + } + + @Test func capPresetsAreDistinct() { + let presets = StudioPreset.builtIn + #expect(presets.first?.look == .studioDefault) + #expect(Set(presets.map(\.id)).count == presets.count) + #expect(Set(presets.map(\.name)).count == presets.count) + #expect(Set(presets.map(\.look)).count == presets.count) + #expect(presets.allSatisfy { $0.isBuiltIn }) + #expect(!StudioPreset(name: "Mine", look: .studioDefault).isBuiltIn) + // Every preset frames the video: none is the plain recording. + #expect(presets.allSatisfy { !$0.look.background.isNone && $0.look.padding > 0 }) + } + + @Test func savedPresetsAndTheDefaultRoundTrip() throws { + let store = temporaryStore().presets + #expect(store.load() == StudioPresetLibrary()) + #expect(store.load().defaultPreset.id == "cap.studio") + var look = StudioLook.studioDefault + look.aspect = .square + let mine = StudioPreset(name: "Launch", look: look) + try store.save(StudioPresetLibrary(saved: [mine], defaultId: mine.id)) + let loaded = store.load() + #expect(loaded.saved == [mine]) + #expect(loaded.defaultPreset == mine) + #expect(loaded.all.first == mine) + #expect(loaded.preset(matching: look) == mine) + // A default that's been deleted falls back to Cap's Studio look. + try store.save(StudioPresetLibrary(saved: [], defaultId: mine.id)) + #expect(store.load().defaultPreset.id == "cap.studio") + } + + @Test func presetsAreNeverMistakenForARecording() throws { + let drafts = temporaryStore() + try drafts.presets.save(StudioPresetLibrary(saved: [StudioPreset(name: "Mine", look: .studioDefault)])) + // Recovery treats every folder in the drafts as a recording and clears ones with no tracks. + let inDrafts = (try? FileManager.default.contentsOfDirectory(atPath: drafts.root.path)) ?? [] + #expect(inDrafts.isEmpty) + #expect(!drafts.presets.root.path.hasPrefix(drafts.root.path + "/")) + #expect(drafts.all().isEmpty) + // Another account's presets are its own. + let other = StudioStore(root: drafts.root.deletingLastPathComponent().appending(path: "user_b", directoryHint: .isDirectory)) + #expect(other.presets.load().saved.isEmpty) + } + + @Test func photoBackgroundsOutliveTheirRecording() throws { + let drafts = temporaryStore() + let project = drafts.directory(for: "p1") + try FileManager.default.createDirectory(at: project, withIntermediateDirectories: true) + try Data([1, 2, 3]).write(to: project.appending(path: "background-1.jpg")) + let presets = drafts.presets + try presets.keepImage(named: "background-1.jpg", from: project.appending(path: "background-1.jpg")) + var look = StudioLook.studioDefault + look.background = .image("background-1.jpg") + try presets.save(StudioPresetLibrary(saved: [StudioPreset(name: "Photo", look: look)])) + drafts.delete("p1") + // Applied to a new recording, the photo is copied in beside it. + let next = drafts.directory(for: "p2") + presets.provideImage(for: look, in: next) + #expect(FileManager.default.fileExists(atPath: next.appending(path: "background-1.jpg").path)) + // Once no preset uses it, it goes. + try presets.save(StudioPresetLibrary()) + #expect(!FileManager.default.fileExists(atPath: presets.imageURL("background-1.jpg").path)) + } + + @Test func namesNeverRepeat() { + let library = StudioPresetLibrary(saved: [StudioPreset(name: "My Look", look: .untouched), StudioPreset(name: "My Look 2", look: .untouched)]) + #expect(library.uniqueName("my look") == "my look 3") + #expect(library.uniqueName(" Ocean ") == "Ocean 2") + #expect(library.uniqueName("") == "My Look 3") + #expect(library.uniqueName("Launch") == "Launch") + } + + /// Thumbnails for every preset come from a renderer of their own: the preview's cached background must still be + /// the preview's afterwards, or its next frame would pay to redraw it. + @Test func thumbnailsLeaveThePreviewAlone() throws { + let renderer = StudioRenderer() + let screenSize = CGSize(width: 1179, height: 2556) + let cameraSize = CGSize(width: 720, height: 1280) + let screenInfo = StudioSourceInfo(naturalSize: screenSize) + let cameraInfo = StudioSourceInfo(naturalSize: cameraSize) + let screen = StudioRenderingTests.solid(screenSize, CIColor(red: 0.9, green: 0.9, blue: 0.9), renderer: renderer) + let camera = StudioRenderingTests.solid(cameraSize, CIColor(red: 0.5, green: 0.4, blue: 0.3), renderer: renderer) + let preview = StudioScene(look: .studioDefault, screen: screenInfo, camera: cameraInfo, renderer: renderer) + let canvas = StudioLayout.canvasSize(look: .studioDefault, primary: screenSize, camera: cameraSize, longEdge: StudioLayout.previewLongEdge) + _ = renderer.render(scene: preview, screen: screen, camera: camera, size: canvas) + #expect(renderer.cachedLook?.look == .studioDefault) + + let thumbnails = renderer.sibling() + #expect(thumbnails.context === renderer.context) + var times: [Double] = [] + let first = CFAbsoluteTimeGetCurrent() + _ = thumbnails.image(scene: StudioScene(look: .studioDefault, screen: screenInfo, camera: cameraInfo, renderer: thumbnails), screen: screen, camera: camera, size: CGSize(width: 64, height: 64)) + let warm = (CFAbsoluteTimeGetCurrent() - first) * 1000 + for _ in 0..<3 { + for preset in StudioPreset.builtIn { + let size = StudioLayout.canvasSize(look: preset.look, primary: screenSize, camera: cameraSize, longEdge: 360) + let scene = StudioScene(look: preset.look, screen: screenInfo, camera: cameraInfo, renderer: thumbnails) + let start = CFAbsoluteTimeGetCurrent() + let image = try #require(thumbnails.image(scene: scene, screen: screen, camera: camera, size: size)) + times.append((CFAbsoluteTimeGetCurrent() - start) * 1000) + #expect(image.width == Int(size.width) && image.height == Int(size.height)) + } + } + #expect(renderer.cachedLook?.look == .studioDefault) + #expect(renderer.cachedLook?.size == canvas) + + times.sort() + let p50 = times[times.count / 2] + let worst = times.last ?? 0 + print("STUDIO_BENCH preset thumbnail first \(String(format: "%.1f", warm)) ms, then p50 \(String(format: "%.2f", p50)) ms max \(String(format: "%.2f", worst)) ms over \(times.count)") + // About 2.5 ms on an M4 when it's quiet. They're drawn off the main thread, so a frame at 60 fps is the bound. + if ProcessInfo.processInfo.environment["CI"] == nil { + #expect(p50 < 16) + } + } + + @Test func thumbnailShowsThePresetsLayout() throws { + let renderer = StudioRenderer().sibling() + let screenSize = CGSize(width: 590, height: 1278) + let cameraSize = CGSize(width: 360, height: 640) + let preset = try #require(StudioPreset.builtIn.first { $0.id == "cap.vertical" }) + let size = StudioLayout.canvasSize(look: preset.look, primary: screenSize, camera: cameraSize, longEdge: 360) + #expect(abs(size.width / size.height - 9.0 / 16) < 0.01) + let scene = StudioScene(look: preset.look, screen: StudioSourceInfo(naturalSize: screenSize), camera: StudioSourceInfo(naturalSize: cameraSize), renderer: renderer) + let image = try #require(renderer.image(scene: scene, screen: StudioRenderingTests.solid(screenSize, CIColor(red: 1, green: 0, blue: 0), renderer: renderer), camera: StudioRenderingTests.solid(cameraSize, CIColor(red: 0, green: 1, blue: 0), renderer: renderer), size: size)) + let layout = StudioLayout(look: preset.look, canvas: size, primary: screenSize, camera: cameraSize) + let screenFrame = try #require(layout.screen?.frame) + let cameraFrame = try #require(layout.camera?.frame) + // Split: screen on top, camera underneath. + #expect(screenFrame.maxY <= cameraFrame.minY) + let pixels = try #require(image.dataProvider?.data as Data?) + func pixel(_ x: CGFloat, _ y: CGFloat) -> (r: Int, g: Int, b: Int) { + let offset = Int(y) * image.bytesPerRow + Int(x) * 4 + return (Int(pixels[offset]), Int(pixels[offset + 1]), Int(pixels[offset + 2])) + } + let top = pixel(screenFrame.midX, screenFrame.midY) + #expect(top.r > 230 && top.g < 30) + let bottom = pixel(cameraFrame.midX, cameraFrame.midY) + #expect(bottom.g > 230 && bottom.r < 30) + } +} + +@Suite struct StudioGlowTests { + @Test func everyGlowRoundTrips() throws { + #expect(Set(StudioGlow.all.map(\.id)).count == StudioGlow.all.count) + for background in StudioBackground.glows { + var look = StudioLook.studioDefault + look.background = background + #expect(try JSONDecoder().decode(StudioLook.self, from: JSONEncoder().encode(look)) == look) + } + } + + /// Each bloom shows its own colour where it sits, the base shows where none reach, and the background is drawn + /// once per look, so its cost only has to stay well under a frame. + @Test func bloomsLandWhereTheyAreDefined() throws { + let renderer = StudioRenderer() + let size = CGSize(width: 1080, height: 1920) + var times: [Double] = [] + for glow in StudioGlow.all { + let start = CFAbsoluteTimeGetCurrent() + let image = StudioRenderer.glow(glow, size: size) + let cg = try #require(renderer.context.createCGImage(image, from: CGRect(origin: .zero, size: size), format: .RGBA8, colorSpace: CGColorSpace(name: CGColorSpace.sRGB)!)) + times.append((CFAbsoluteTimeGetCurrent() - start) * 1000) + let data = try #require(cg.dataProvider?.data as Data?) + func pixel(_ x: Double, _ y: Double) -> (Double, Double, Double) { + let px = min(cg.width - 1, max(0, Int(x * Double(cg.width)))) + let py = min(cg.height - 1, max(0, Int(y * Double(cg.height)))) + let o = py * cg.bytesPerRow + px * 4 + return (Double(data[o]), Double(data[o + 1]), Double(data[o + 2])) + } + func distance(_ a: (Double, Double, Double), _ c: StudioColor) -> Double { + let b = (Double(c.red), Double(c.green), Double(c.blue)) + return ((a.0 - b.0) * (a.0 - b.0) + (a.1 - b.1) * (a.1 - b.1) + (a.2 - b.2) * (a.2 - b.2)).squareRoot() + } + // The bloom that stands out most from the base must pull the colour where it sits towards its own. + let base = (Double(glow.base.red), Double(glow.base.green), Double(glow.base.blue)) + let boldest = try #require(glow.blooms.max { distance(base, $0.color) < distance(base, $1.color) }) + let atBloom = pixel(boldest.x, boldest.y) + #expect(distance(atBloom, boldest.color) < distance(base, boldest.color) * 0.8, "\(glow.id)") + // Smooth: neighbouring pixels a few apart never jump. + let a = pixel(0.5, 0.5) + let b = pixel(0.5 + 4.0 / 1080, 0.5) + #expect(abs(a.0 - b.0) < 6 && abs(a.1 - b.1) < 6 && abs(a.2 - b.2) < 6, "\(glow.id)") + } + times.sort() + print("STUDIO_BENCH glow 1080x1920 p50 \(String(format: "%.2f", times[times.count / 2])) ms max \(String(format: "%.2f", times.last ?? 0)) ms") + } + + /// Padding drags redraw the frame's still parts each step; the glow beneath is drawn once. + @Test func dragsDontRedrawTheGlow() throws { + let renderer = StudioRenderer() + let screenSize = CGSize(width: 1179, height: 2556) + let screen = StudioRenderingTests.solid(screenSize, CIColor(red: 0.9, green: 0.9, blue: 0.9), renderer: renderer) + var look = StudioLook.studioDefault + look.background = .glow("dusk") + let scene = StudioScene(look: look, screen: StudioSourceInfo(naturalSize: screenSize), camera: nil, renderer: renderer) + let canvas = StudioLayout.canvasSize(look: look, primary: screenSize, camera: nil, longEdge: StudioLayout.previewLongEdge) + _ = renderer.render(scene: scene, screen: screen, camera: nil, size: canvas) + var times: [Double] = [] + for step in 1...30 { + scene.update { $0.look.padding = 0.05 + Double(step) * 0.002 } + let start = CFAbsoluteTimeGetCurrent() + _ = renderer.render(scene: scene, screen: screen, camera: nil, size: canvas) + times.append((CFAbsoluteTimeGetCurrent() - start) * 1000) + } + times.sort() + let p50 = times[times.count / 2] + print("STUDIO_BENCH glow padding drag step p50 \(String(format: "%.2f", p50)) ms max \(String(format: "%.2f", times.last ?? 0)) ms") + if ProcessInfo.processInfo.environment["CI"] == nil { + #expect(p50 < 16) + } + } +} diff --git a/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioSyncTests.swift b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioSyncTests.swift new file mode 100644 index 00000000000..946e8961617 --- /dev/null +++ b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioSyncTests.swift @@ -0,0 +1,444 @@ +@preconcurrency import AVFoundation +@testable import CapKit +import CoreImage +import Foundation +import Synchronization +import Testing + +/// Screen and camera are recorded by different processes on different clocks; these prove they come out of an export +/// together. Each track flashes white (and the screen beeps) at the same moments on the host clock, with the kinds of +/// trouble real recordings have: tracks starting at different times, 29.97 against 30 fps, timestamp jitter, dropped +/// frames, a camera gap and cuts. The export is then measured frame by frame and sample by sample. +@Suite(.serialized) struct StudioSyncTests { + static let screenSize = CGSize(width: 360, height: 780) + static let cameraSize = CGSize(width: 180, height: 320) + static let screenColor = CIColor(red: 0.1, green: 0.2, blue: 0.75) + static let cameraColor = CIColor(red: 0.1, green: 0.75, blue: 0.2) + + struct Track { + var hostStart: Double + var seconds: Double + var frameInterval: Double + var jitter: Double + var dropEvery: Int? + var gap: EditRange? + var color: CIColor + var size: CGSize + var beeps: Bool + /// The microphone goes quiet (no samples at all) over this stretch, as when it's muted mid-recording. + var audioDropout: EditRange? = nil + /// No sound after this many seconds, while the picture carries on. + var audioStopsAt: Double? = nil + } + + /// Deterministic pseudo-random numbers, so a failure reproduces. + struct Noise { + var state: UInt64 + mutating func next() -> Double { + state = state &* 6364136223846793005 &+ 1442695040888963407 + return Double(state >> 11) / Double(1 << 53) + } + } + + static func flashes(_ times: [Double], at host: Double, interval: Double) -> Bool { + times.contains { host >= $0 && host < $0 + interval } + } + + /// Feeds frames and audio in time order, the way a recorder delivers them. + /// Returns the track's info and the host time of the first frame of each flash, as it was actually stamped. + static func record(_ track: Track, flashesAt times: [Double], to url: URL, kind: StudioTrackWriter.Settings.Kind) async throws -> (info: StudioTrackInfo, flashFrames: [Double]) { + var flashFrames: [Double] = [] + var lastWasFlash = false + let writer = try StudioTrackWriter(url: url, settings: .init(kind: kind, audio: track.beeps)) + let renderer = StudioRenderer.shared + var noise = Noise(state: UInt64(track.hostStart * 1000)) + var format: CMVideoFormatDescription? + var audioFormat: CMAudioFormatDescription? + var asbd = AudioStreamBasicDescription(mSampleRate: 48_000, mFormatID: kAudioFormatLinearPCM, mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2, mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0) + CMAudioFormatDescriptionCreate(allocator: nil, asbd: &asbd, layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &audioFormat) + let audioChunk = 960 + var nextAudio = 0 + var index = 0 + while true { + let ideal = Double(index) * track.frameInterval + guard ideal < track.seconds else { break } + let host = track.hostStart + ideal + (noise.next() - 0.5) * 2 * track.jitter + index += 1 + // Audio up to this frame first, as a recorder interleaves it. + while track.beeps, Double(nextAudio) / 48_000 < ideal { + let local = Double(nextAudio) / 48_000 + if let stop = track.audioStopsAt, local >= stop { break } + if let dropout = track.audioDropout, local >= dropout.start, local < dropout.end { + nextAudio += audioChunk + continue + } + let start = track.hostStart + local + var samples = [Int16](repeating: 0, count: audioChunk) + for i in 0..= $0 && t < $0 + 0.08 }) { samples[i] = Int16(sin(t * 2 * .pi * 1000) * 20000) } + } + var block: CMBlockBuffer? + CMBlockBufferCreateWithMemoryBlock(allocator: nil, memoryBlock: nil, blockLength: audioChunk * 2, blockAllocator: nil, customBlockSource: nil, offsetToData: 0, dataLength: audioChunk * 2, flags: 0, blockBufferOut: &block) + samples.withUnsafeBytes { _ = CMBlockBufferReplaceDataBytes(with: $0.baseAddress!, blockBuffer: block!, offsetIntoDestination: 0, dataLength: audioChunk * 2) } + var sample: CMSampleBuffer? + CMAudioSampleBufferCreateReadyWithPacketDescriptions(allocator: nil, dataBuffer: block!, formatDescription: audioFormat!, sampleCount: audioChunk, presentationTimeStamp: CMTime(seconds: start, preferredTimescale: 48_000), packetDescriptions: nil, sampleBufferOut: &sample) + while !writer.isReadyForAudio { try await Task.sleep(for: .milliseconds(1)) } + writer.appendAudio(sample!) + nextAudio += audioChunk + } + if let gap = track.gap, ideal >= gap.start, ideal < gap.end { continue } + if let every = track.dropEvery, index % every == 0, !flashes(times, at: track.hostStart + ideal, interval: track.frameInterval) { continue } + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, Int(track.size.width), Int(track.size.height), kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + let isFlash = flashes(times, at: track.hostStart + ideal, interval: track.frameInterval) + if isFlash, !lastWasFlash { flashFrames.append(host) } + lastWasFlash = isFlash + let color = isFlash ? CIColor(red: 1, green: 1, blue: 1) : track.color + renderer.context.render(CIImage(color: color).cropped(to: CGRect(origin: .zero, size: track.size)), to: buffer!, bounds: CGRect(origin: .zero, size: track.size), colorSpace: CGColorSpace(name: CGColorSpace.sRGB)!) + if format == nil { CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescriptionOut: &format) } + var timing = CMSampleTimingInfo(duration: CMTime(seconds: track.frameInterval, preferredTimescale: 90_000), presentationTimeStamp: CMTime(seconds: host, preferredTimescale: 90_000), decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescription: format!, sampleTiming: &timing, sampleBufferOut: &sample) + while !writer.isReadyForVideo { try await Task.sleep(for: .milliseconds(1)) } + writer.appendVideo(sample!) + } + let duration = try await withCheckedThrowingContinuation { continuation in + writer.finish(at: nil) { continuation.resume(with: $0) } + } + var info = try #require(writer.info) + info.duration = duration + return (info, flashFrames) + } + + struct Measured { + var screenFlashes: [Double] = [] + var cameraFlashes: [Double] = [] + var beeps: [Double] = [] + var cameraHidden: [Double] = [] + } + + static func measure(_ url: URL, screen: CGPoint, camera: CGPoint) async throws -> Measured { + let asset = AVURLAsset(url: url) + let reader = try AVAssetReader(asset: asset) + let videoTrack = try #require(try await asset.loadTracks(withMediaType: .video).first) + let video = AVAssetReaderTrackOutput(track: videoTrack, outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) + reader.add(video) + var audio: AVAssetReaderTrackOutput? + if let audioTrack = try await asset.loadTracks(withMediaType: .audio).first { + let output = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: [AVFormatIDKey: kAudioFormatLinearPCM, AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false, AVLinearPCMIsNonInterleaved: false, AVNumberOfChannelsKey: 1, AVSampleRateKey: 48_000]) + reader.add(output) + audio = output + } + reader.startReading() + var measured = Measured() + var screenLit = false + var cameraLit = false + while let sample = video.copyNextSampleBuffer() { + let time = CMSampleBufferGetPresentationTimeStamp(sample).seconds + guard let pixels = CMSampleBufferGetImageBuffer(sample) else { continue } + let s = StudioRenderingTests.pixel(pixels, Int(screen.x), Int(screen.y)) + let c = StudioRenderingTests.pixel(pixels, Int(camera.x), Int(camera.y)) + let screenWhite = s.r > 200 && s.g > 200 && s.b > 200 + let cameraWhite = c.r > 200 && c.g > 200 && c.b > 200 + if screenWhite, !screenLit { measured.screenFlashes.append(time) } + if cameraWhite, !cameraLit { measured.cameraFlashes.append(time) } + // Neither the camera's green nor a flash: the bubble isn't there. + if !(c.g > 150 && c.r < 100) && !cameraWhite { measured.cameraHidden.append(time) } + screenLit = screenWhite + cameraLit = cameraWhite + } + if let audio { + var lastLoud = -1.0 + while let sample = audio.copyNextSampleBuffer(), let block = CMSampleBufferGetDataBuffer(sample) { + let start = CMSampleBufferGetPresentationTimeStamp(sample).seconds + var length = 0 + var pointer: UnsafeMutablePointer? + CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length, dataPointerOut: &pointer) + guard let pointer else { continue } + pointer.withMemoryRebound(to: Int16.self, capacity: length / 2) { values in + for i in 0..<(length / 2) where abs(Int32(values[i])) > 6000 { + let t = start + Double(i) / 48_000 + // A new beep after a stretch of quiet. + if t - lastLoud > 0.2 { measured.beeps.append(t) } + lastLoud = t + } + } + } + } + return measured + } + + @Test func screenCameraAndVoiceStayTogetherThroughCutsAndGaps() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-sync-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + // Host-clock moments both tracks flash at. The camera starts 0.73 s before the screen. + let base = 5000.0 + let flashTimes = [base + 1.5, base + 5.47, base + 11.03, base + 16.6, base + 28.1] + // The microphone is muted for two and a half seconds part way, so the voice after it must still land on its picture. + let screenTrack = Track(hostStart: base + 0.73, seconds: 30, frameInterval: 1.0 / 30, jitter: 0.004, dropEvery: 31, gap: nil, color: Self.screenColor, size: Self.screenSize, beeps: true, audioDropout: EditRange(start: 18, end: 20.5)) + // 29.97 fps, more jitter, every 23rd frame dropped, and nothing for four seconds (a call, say). + let cameraTrack = Track(hostStart: base, seconds: 31, frameInterval: 1001.0 / 30000, jitter: 0.008, dropEvery: 23, gap: EditRange(start: 21, end: 24), color: Self.cameraColor, size: Self.cameraSize, beeps: false) + let (screenInfo, screenFlashFrames) = try await Self.record(screenTrack, flashesAt: flashTimes, to: directory.appending(path: "screen.mp4"), kind: .screen) + let (cameraInfo, cameraFlashFrames) = try await Self.record(cameraTrack, flashesAt: flashTimes, to: directory.appending(path: "camera.mp4"), kind: .camera) + #expect(abs(screenInfo.hostStart - (base + 0.73)) < 0.01) + let cameraGap = try #require(cameraInfo.gaps?.first) + #expect(abs(cameraGap.start - 21) < 0.1 && abs(cameraGap.end - 24) < 0.1) + + let aligned = StudioTrackInfo.align(screen: screenInfo, camera: cameraInfo) + let screenProject = try #require(aligned.screen) + let cameraProject = try #require(aligned.camera) + #expect(abs(screenProject.offset - 0.73) < 0.01) + let screenAudio = try #require(screenProject.audio) + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: screenProject.offset, gaps: screenProject.gaps ?? [], audio: (directory.appending(path: screenAudio.file), screenAudio.offset, screenAudio.duration)) + let camera = try await StudioTrackSource.load(url: directory.appending(path: "camera.mp4"), offset: cameraProject.offset, gaps: cameraProject.gaps ?? []) + + var look = StudioLook.studioDefault + look.shadow = 0 + look.camera.size = 0.4 + let scene = StudioScene(look: look, screen: screen.info, camera: camera.info) + // Cuts on either side of flashes and across the camera gap: project time 3-4.5, 13-14 and 22-23 removed. + let project = StudioProject(title: "Sync", screen: screenProject, camera: cameraProject) + let clips = [EditRange(start: 0, end: 3), EditRange(start: 4.5, end: 13), EditRange(start: 14, end: 22), EditRange(start: 23, end: project.duration)] + let timeline = EditTimeline(sourceDuration: project.duration, clips: clips) + let built = try StudioComposition.build(clips: timeline.clips, screen: screen, camera: camera, music: nil, voiceVolume: 1, scene: scene, longEdge: 960, remembersFrames: false) + let exported = try await StudioExporter().export(built, to: directory.appending(path: "export.mp4")) + + let layout = StudioLayout(look: look, canvas: built.renderSize, primary: screen.info.displaySize, camera: camera.info.displaySize) + let screenFrame = try #require(layout.screen?.frame) + let bubble = try #require(layout.camera?.frame) + // A point on the screen well away from the bubble, and the bubble's centre. + let screenPoint = CGPoint(x: screenFrame.minX + screenFrame.width * 0.3, y: screenFrame.minY + screenFrame.height * 0.3) + let measured = try await Self.measure(exported.url, screen: screenPoint, camera: CGPoint(x: bubble.midX, y: bubble.midY)) + + // Where each flash belongs in the edited video. The project starts at the earlier track's first frame. + let origin = min(screenInfo.hostStart, cameraInfo.hostStart) + let expected = flashTimes.map { timeline.outputTime(forSource: $0 - origin) } + print("STUDIO_SYNC expected \(expected.map { String(format: "%.3f", $0) })") + print("STUDIO_SYNC screen \(measured.screenFlashes.map { String(format: "%.3f", $0) })") + print("STUDIO_SYNC camera \(measured.cameraFlashes.map { String(format: "%.3f", $0) })") + print("STUDIO_SYNC beeps \(measured.beeps.map { String(format: "%.3f", $0) })") + #expect(measured.screenFlashes.count == expected.count) + #expect(measured.cameraFlashes.count == expected.count) + #expect(measured.beeps.count == expected.count) + // Exactly where each flash frame must appear: a recorded frame shows from the first 30 fps output frame at or + // after its own timestamp, mapped through the cuts. Anything else is a sync error, not rounding. + func shownAt(_ host: Double) -> Double { + let output = timeline.outputTime(forSource: host - origin) + return (output * 30 - 1e-6).rounded(.up) / 30 + } + for (index, time) in expected.enumerated() { + guard index < measured.screenFlashes.count, index < measured.cameraFlashes.count, index < measured.beeps.count, + index < screenFlashFrames.count, index < cameraFlashFrames.count + else { break } + let screenError = measured.screenFlashes[index] - shownAt(screenFlashFrames[index]) + let cameraError = measured.cameraFlashes[index] - shownAt(cameraFlashFrames[index]) + let voiceError = measured.beeps[index] - time + print("STUDIO_SYNC flash \(index) at \(String(format: "%.3f", time)) s: screen frame \(String(format: "%+.1f", screenError * 1000)) ms, camera frame \(String(format: "%+.1f", cameraError * 1000)) ms, voice \(String(format: "%+.1f", voiceError * 1000)) ms off its exact place") + #expect(abs(screenError) < 0.002) + #expect(abs(cameraError) < 0.002) + // AAC adds a few milliseconds of fixed delay at most. + #expect(abs(voiceError) < 0.006) + } + // In the camera's gap (21-24 s; the camera starts the project), the bubble is gone, not frozen. + let gapMiddle = timeline.outputTime(forSource: 23.5) + #expect(measured.cameraHidden.contains { abs($0 - gapMiddle) < 0.1 }) + // And it's back after. + let afterGap = timeline.outputTime(forSource: 26) + #expect(!measured.cameraHidden.contains { abs($0 - afterGap) < 0.05 }) + #expect(abs(exported.duration - timeline.outputDuration) < 0.05) + } + + /// The app or extension can be killed mid-recording. The file must still play up to its last fragment. + @Test func anUnfinishedTrackStillPlays() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-crash-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let url = directory.appending(path: "camera.mp4") + let started = Mutex(nil) + do { + let writer = try StudioTrackWriter(url: url, settings: .init(kind: .camera, audio: false)) { info in started.withLock { $0 = info.hostStart } } + var format: CMVideoFormatDescription? + for index in 0..<(30 * 7) { + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 180, 320, kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + if format == nil { CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescriptionOut: &format) } + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: 30), presentationTimeStamp: CMTime(value: CMTimeValue(9000 + index), timescale: 30), decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescription: format!, sampleTiming: &timing, sampleBufferOut: &sample) + while !writer.isReadyForVideo { try await Task.sleep(for: .milliseconds(1)) } + writer.appendVideo(sample!) + } + // Give the encoder a moment to write its fragments, then drop the writer without finishing, as a crash would. + try await Task.sleep(for: .milliseconds(500)) + _ = writer + } + #expect(started.withLock { $0 } == 300) + let duration = try await AVURLAsset(url: url).load(.duration).seconds + print("STUDIO_SYNC unfinished 7.0 s track plays \(String(format: "%.2f", duration)) s") + #expect(duration >= 3.9) + } + + /// A full-size export holds only a few frames at a time, however long the video. + @Test func exportMemoryStaysFlat() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-memory-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let screenTrack = Track(hostStart: 0, seconds: 8, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: nil, color: Self.screenColor, size: CGSize(width: 1178, height: 2556), beeps: true) + let cameraTrack = Track(hostStart: 0.2, seconds: 8, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: nil, color: Self.cameraColor, size: CGSize(width: 720, height: 1280), beeps: false) + let (screenInfo, _) = try await Self.record(screenTrack, flashesAt: [5, 10], to: directory.appending(path: "screen.mp4"), kind: .screen) + let (cameraInfo, _) = try await Self.record(cameraTrack, flashesAt: [5, 10], to: directory.appending(path: "camera.mp4"), kind: .camera) + let aligned = StudioTrackInfo.align(screen: screenInfo, camera: cameraInfo) + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: aligned.screen!.offset) + let camera = try await StudioTrackSource.load(url: directory.appending(path: "camera.mp4"), offset: aligned.camera!.offset) + let scene = StudioScene(look: .studioDefault, screen: screen.info, camera: camera.info) + let built = try StudioComposition.build(clips: [EditRange(start: 0, end: 8)], screen: screen, camera: camera, music: nil, voiceVolume: 1, scene: scene, longEdge: StudioLayout.exportLongEdge, remembersFrames: false) + let before = ProcessStats.footprint() + let sampler = Task.detached { () -> Double in + var high = 0.0 + for _ in 0..<3000 { + if Task.isCancelled { break } + high = max(high, ProcessStats.footprint().current) + try? await Task.sleep(for: .milliseconds(20)) + } + return high + } + let result = try await StudioExporter().export(built, to: directory.appending(path: "out.mp4")) + sampler.cancel() + let high = await sampler.value + print("STUDIO_BENCH full-size export \(Int(result.size.width))x\(Int(result.size.height)) \(String(format: "%.1f", result.duration)) s in \(String(format: "%.2f", result.seconds)) s (\(String(format: "%.1f", result.duration / result.seconds))x), memory \(Int(before.current)) MB before, \(Int(high)) MB highest during, \(Int(high - before.current)) MB added") + // Measured 133-163 MB added on an M4 with hardware codecs and about 245 MB on CI's virtual Macs, which encode in + // software. Frames piling up over this 240-frame export would add a gigabyte or more, which is what this catches. + #expect(high - before.current < 400) + } + + /// Cancelling a render part way returns at once, whatever each feeding loop was waiting on. + @Test func cancellingARenderReturns() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-cancel-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let track = Track(hostStart: 0, seconds: 6, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: nil, color: Self.screenColor, size: CGSize(width: 720, height: 1560), beeps: true) + _ = try await Self.record(track, flashesAt: [3], to: directory.appending(path: "screen.mp4"), kind: .screen) + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: 0) + let scene = StudioScene(look: .studioDefault, screen: screen.info, camera: nil) + for delay in [0.0, 0.05, 0.3] { + let built = try StudioComposition.build(clips: [EditRange(start: 0, end: 6)], screen: screen, camera: nil, music: nil, voiceVolume: 1, scene: scene, longEdge: 1920, remembersFrames: false) + let exporter = StudioExporter() + let url = directory.appending(path: "out-\(delay).mp4") + let started = CFAbsoluteTimeGetCurrent() + let render = Task { try await exporter.export(built, to: url) } + try await Task.sleep(for: .seconds(delay)) + exporter.cancel() + var cancelled = false + do { + _ = try await render.value + } catch StudioExporter.Failure.cancelled { + cancelled = true + } catch { + cancelled = true + } + let took = CFAbsoluteTimeGetCurrent() - started - delay + print("STUDIO_BENCH cancel after \(delay) s returned in \(String(format: "%.2f", took)) s") + #expect(cancelled) + #expect(took < 3) + #expect(!FileManager.default.fileExists(atPath: url.path)) + } + } + + /// A cancel that lands while the finished file is being closed still cancels: nothing is handed on to share. + @Test func cancellingWhileTheFileClosesReturnsNothing() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-cancel-close-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let track = Track(hostStart: 0, seconds: 2, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: nil, color: Self.screenColor, size: CGSize(width: 720, height: 1560), beeps: true) + _ = try await Self.record(track, flashesAt: [1], to: directory.appending(path: "screen.mp4"), kind: .screen) + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: 0) + let scene = StudioScene(look: .studioDefault, screen: screen.info, camera: nil) + let built = try StudioComposition.build(clips: [EditRange(start: 0, end: 2)], screen: screen, camera: nil, music: nil, voiceVolume: 1, scene: scene, longEdge: 1280, remembersFrames: false) + let exporter = StudioExporter() + exporter.willFinishWriting = { [weak exporter] in exporter?.cancel() } + let url = directory.appending(path: "out.mp4") + await #expect(throws: StudioExporter.Failure.self) { _ = try await exporter.export(built, to: url) } + #expect(!FileManager.default.fileExists(atPath: url.path)) + } + + /// Sound that stops while the picture carries on (a muted microphone, an interruption) must never stop the picture. + @Test func pictureCarriesOnWhenSoundStops() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-nosound-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let track = Track(hostStart: 200, seconds: 10, frameInterval: 1.0 / 30, jitter: 0.002, dropEvery: nil, gap: nil, color: Self.screenColor, size: Self.screenSize, beeps: true, audioStopsAt: 1.5) + let (info, _) = try await Self.record(track, flashesAt: [201, 208], to: directory.appending(path: "screen.mp4"), kind: .screen) + print("STUDIO_SYNC sound stopped at 1.5 s: picture \(String(format: "%.2f", info.duration)) s, sound \(String(format: "%.2f", info.audio?.duration ?? 0)) s") + #expect(info.duration > 9.9) + #expect(abs((info.audio?.duration ?? 0) - 1.5) < 0.1) + let video = try await AVURLAsset(url: directory.appending(path: "screen.mp4")).load(.duration).seconds + #expect(video > 9.9) + } + + /// A screen that stops changing sends nothing for a while; its last picture must hold, not be cut out as a gap. + @Test func aStillScreenIsNotAGap() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-still-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let pause = EditRange(start: 2, end: 3.2) + let (screen, _) = try await Self.record(Track(hostStart: 0, seconds: 5, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: pause, color: Self.screenColor, size: Self.screenSize, beeps: false), flashesAt: [], to: directory.appending(path: "screen.mp4"), kind: .screen) + #expect(screen.gaps == nil) + let (camera, _) = try await Self.record(Track(hostStart: 0, seconds: 5, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: pause, color: Self.cameraColor, size: Self.cameraSize, beeps: false), flashesAt: [], to: directory.appending(path: "camera.mp4"), kind: .camera) + #expect(camera.gaps?.count == 1) + } + + /// When one file fails, recording stops but the healthy file is still closed properly, keeping its picture. + @Test func aFailureStillClosesTheHealthyFile() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-fail-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let url = directory.appending(path: "camera.mp4") + let writer = try StudioTrackWriter(url: url, settings: .init(kind: .camera, audio: false)) + var format: CMVideoFormatDescription? + for index in 0..<(30 * 4) { + if index == 30 * 2 { writer.markFailed() } + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 180, 320, kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + if format == nil { CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescriptionOut: &format) } + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: 30), presentationTimeStamp: CMTime(value: CMTimeValue(600 + index), timescale: 30), decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: buffer!, formatDescription: format!, sampleTiming: &timing, sampleBufferOut: &sample) + while !writer.isReadyForVideo { try await Task.sleep(for: .milliseconds(1)) } + writer.appendVideo(sample!) + } + #expect(writer.failed) + let result = try await withCheckedThrowingContinuation { continuation in writer.finish(at: nil) { continuation.resume(with: $0) } } + // Frames after the failure are refused; the two seconds before it are closed into a complete file. + #expect(abs(result - 2) < 0.05) + let duration = try await AVURLAsset(url: url).load(.duration).seconds + #expect(abs(duration - 2) < 0.05) + } + + @Test func recordedRangesSkipGaps() { + let track = StudioProject.Track(file: "c", offset: 1, duration: 10, gaps: [EditRange(start: 6, end: 7), EditRange(start: 2, end: 3), EditRange(start: 9.5, end: 12)]) + #expect(track.recorded == [EditRange(start: 0, end: 2), EditRange(start: 3, end: 6), EditRange(start: 7, end: 9.5)]) + #expect(StudioProject.Track(file: "c", offset: 0, duration: 4).recorded == [EditRange(start: 0, end: 4)]) + } + + /// An hour-long recording cut into hundreds of pieces still rebuilds in milliseconds. + @Test func longProjectsRebuildFast() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-long-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let short = Track(hostStart: 0, seconds: 3, frameInterval: 1.0 / 30, jitter: 0, dropEvery: nil, gap: nil, color: Self.screenColor, size: Self.screenSize, beeps: false) + _ = try await Self.record(short, flashesAt: [], to: directory.appending(path: "screen.mp4"), kind: .screen) + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: 0) + // Pretend the source is an hour long by cutting the same 3 s into 600 clips. + let clips = (0..<600).map { _ in EditRange(start: 0.5, end: 2.5) } + let scene = StudioScene(look: .studioDefault, screen: screen.info, camera: nil) + let start = CFAbsoluteTimeGetCurrent() + let built = try StudioComposition.build(clips: clips, screen: screen, camera: nil, music: nil, voiceVolume: 1, scene: scene, longEdge: 1280, remembersFrames: true) + let took = (CFAbsoluteTimeGetCurrent() - start) * 1000 + print("STUDIO_BENCH rebuild 600 clips \(String(format: "%.1f", took)) ms") + #expect(abs(built.duration - 1200) < 0.5) + #expect(took < 250) + } +} diff --git a/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioTests.swift b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioTests.swift new file mode 100644 index 00000000000..4f677c11dd3 --- /dev/null +++ b/apps/mobile-ios/CapKit/Tests/CapKitTests/StudioTests.swift @@ -0,0 +1,426 @@ +@preconcurrency import AVFoundation +@testable import CapKit +import CoreImage +import Foundation +import Testing + +@Suite struct StudioLayoutTests { + let phoneScreen = CGSize(width: 1179, height: 2556) + let camera = CGSize(width: 720, height: 1280) + + @Test func untouchedLookFillsTheFrame() { + let canvas = StudioLayout.canvasSize(look: .untouched, primary: phoneScreen, camera: nil, longEdge: StudioLayout.exportLongEdge) + #expect(canvas.height == 1920) + #expect(abs(canvas.width / canvas.height - phoneScreen.width / phoneScreen.height) < 0.01) + let layout = StudioLayout(look: .untouched, canvas: canvas, primary: phoneScreen, camera: nil) + let frame = try! #require(layout.screen?.frame) + #expect(abs(frame.width - canvas.width) <= 1) + #expect(abs(frame.height - canvas.height) <= 1) + #expect(layout.camera == nil) + } + + @Test func neverUpscalesSmallVideo() { + let canvas = StudioLayout.canvasSize(look: .untouched, primary: CGSize(width: 1280, height: 720), camera: nil, longEdge: 1920) + #expect(canvas == CGSize(width: 1280, height: 720)) + } + + @Test func sizesAreEven() { + for aspect in StudioAspect.allCases { + var look = StudioLook.studioDefault + look.aspect = aspect + let canvas = StudioLayout.canvasSize(look: look, primary: CGSize(width: 1179, height: 2557), camera: camera, longEdge: 1920) + #expect(Int(canvas.width) % 2 == 0) + #expect(Int(canvas.height) % 2 == 0) + if let ratio = aspect.ratio { #expect(abs(canvas.width / canvas.height - ratio) < 0.01) } + } + } + + @Test func paddingInsetsTheVideoAndKeepsItsShape() { + var look = StudioLook.studioDefault + look.padding = 0.1 + look.aspect = .square + let canvas = CGSize(width: 1920, height: 1920) + let layout = StudioLayout(look: look, canvas: canvas, primary: phoneScreen, camera: nil) + let frame = try! #require(layout.screen?.frame) + #expect(frame.minY >= 192 - 1) + #expect(frame.maxY <= 1920 - 192 + 1) + #expect(abs(frame.width / frame.height - phoneScreen.width / phoneScreen.height) < 0.01) + #expect(abs(frame.midX - 960) <= 1) + } + + @Test func noBackgroundMeansNoPaddingOrCorners() { + var look = StudioLook.untouched + look.padding = 0.2 + look.rounding = 0.2 + let canvas = CGSize(width: 886, height: 1920) + let layout = StudioLayout(look: look, canvas: canvas, primary: phoneScreen, camera: nil) + #expect(layout.screen?.shape == .rounded(radius: 0)) + #expect(abs((layout.screen?.frame.height ?? 0) - 1920) <= 1) + } + + @Test func bubbleSitsInTheChosenCorner() { + var look = StudioLook.studioDefault + let canvas = CGSize(width: 1080, height: 1920) + for corner in StudioCorner.allCases { + look.camera.corner = corner + let layout = StudioLayout(look: look, canvas: canvas, primary: phoneScreen, camera: camera) + let bubble = try! #require(layout.camera?.frame) + #expect(StudioLayout.nearestCorner(to: CGPoint(x: bubble.midX, y: bubble.midY), in: canvas) == corner) + #expect(CGRect(origin: .zero, size: canvas).contains(bubble)) + #expect(abs(bubble.width - bubble.height) < 0.5) + } + } + + @Test func splitStacksScreenOverCameraWhenTall() { + var look = StudioLook.studioDefault + look.camera.layout = .split + look.aspect = .portrait + let canvas = CGSize(width: 1080, height: 1920) + let layout = StudioLayout(look: look, canvas: canvas, primary: CGSize(width: 1920, height: 1080), camera: camera) + let screen = try! #require(layout.screen?.frame) + let cam = try! #require(layout.camera?.frame) + #expect(screen.maxY < cam.minY) + #expect(cam.height >= canvas.height / 4) + #expect(!screen.intersects(cam)) + } + + @Test func splitIsSideBySideWhenWide() { + var look = StudioLook.studioDefault + look.camera.layout = .split + look.aspect = .landscape + let canvas = CGSize(width: 1920, height: 1080) + let layout = StudioLayout(look: look, canvas: canvas, primary: CGSize(width: 1920, height: 1080), camera: camera) + let screen = try! #require(layout.screen?.frame) + let cam = try! #require(layout.camera?.frame) + #expect(screen.maxX < cam.minX) + } + + @Test func missingCameraFallsBackToScreen() { + var look = StudioLook.studioDefault + look.camera.layout = .cameraOnly + let layout = StudioLayout(look: look, canvas: CGSize(width: 1080, height: 1920), primary: phoneScreen, camera: nil) + #expect(layout.screen != nil) + #expect(layout.camera == nil) + } + + @Test func cameraOnlyFillsTheArea() { + var look = StudioLook.studioDefault + look.camera.layout = .cameraOnly + look.padding = 0 + let canvas = CGSize(width: 1080, height: 1920) + let layout = StudioLayout(look: look, canvas: canvas, primary: phoneScreen, camera: camera) + #expect(layout.screen == nil) + #expect(layout.camera?.frame == CGRect(origin: .zero, size: canvas)) + } + + @Test func identityLook() { + #expect(StudioLook.untouched.isIdentity(hasCamera: false)) + #expect(!StudioLook.untouched.isIdentity(hasCamera: true)) + #expect(!StudioLook.studioDefault.isIdentity(hasCamera: false)) + var cropped = StudioLook.untouched + cropped.aspect = .square + #expect(!cropped.isIdentity(hasCamera: false)) + } +} + +@Suite struct StudioProjectTests { + @Test func tracksLineUpOnTheHostClock() { + let screen = StudioTrackInfo(file: "screen.mp4", hostStart: 1000.4, duration: 30) + let camera = StudioTrackInfo(file: "camera.mp4", hostStart: 1000.1, duration: 29) + let aligned = StudioTrackInfo.align(screen: screen, camera: camera) + #expect(aligned.camera?.offset == 0) + #expect(abs((aligned.screen?.offset ?? 0) - 0.3) < 1e-9) + let project = StudioProject(title: "t", screen: aligned.screen, camera: aligned.camera) + #expect(abs(project.duration - 30.3) < 1e-9) + } + + @Test func emptyTracksAreDropped() { + let aligned = StudioTrackInfo.align(screen: StudioTrackInfo(file: "s", hostStart: 5, duration: 0), camera: StudioTrackInfo(file: "c", hostStart: 6, duration: 4)) + #expect(aligned.screen == nil) + #expect(aligned.camera?.offset == 0) + } + + @Test func storeRoundTripsNewestFirst() throws { + let root = FileManager.default.temporaryDirectory.appending(path: "studio-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let store = StudioStore(root: root) + var old = StudioProject(title: "Old", createdAt: Date(timeIntervalSince1970: 100), camera: .init(file: "camera.mp4", offset: 0, duration: 3)) + old.music = StudioMusic(source: .builtIn("lofi"), title: "Lofi") + old.look.camera.corner = .topLeading + old.clips = [EditRange(start: 0.5, end: 2)] + let new = StudioProject(title: "New", createdAt: Date(timeIntervalSince1970: 200), screen: .init(file: "screen.mp4", offset: 0, duration: 3)) + try store.save(old) + try store.save(new) + try FileManager.default.createDirectory(at: store.directory(for: "half-written"), withIntermediateDirectories: true) + let all = store.all() + #expect(all.map(\.title) == ["New", "Old"]) + #expect(all[1] == old) + #expect(all[1].timeline.clips == [EditRange(start: 0.5, end: 2)]) + store.delete(old.id) + #expect(store.all().map(\.title) == ["New"]) + } + + @Test func screenRequestIsTakenOnceAndExpires() { + let root = FileManager.default.temporaryDirectory.appending(path: "studio-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let store = StudioStore(root: root) + let now = Date() + StudioScreenRequest.set(.init(projectId: "p1", ownerId: "user_a", requestedAt: now), in: store) + #expect(StudioScreenRequest.peek(in: store, now: now.addingTimeInterval(5))?.projectId == "p1") + let taken = StudioScreenRequest.take(in: store, now: now.addingTimeInterval(5)) + #expect(taken?.projectId == "p1") + #expect(taken?.ownerId == "user_a") + #expect(StudioScreenRequest.take(in: store, now: now.addingTimeInterval(5)) == nil) + StudioScreenRequest.set(.init(projectId: "p2", ownerId: "user_a", requestedAt: now), in: store) + #expect(StudioScreenRequest.take(in: store, now: now.addingTimeInterval(StudioScreenRequest.lifetime + 1)) == nil) + } + + @Test func draftsBelongToOneAccount() throws { + let group = "test.studio.\(UUID().uuidString)" + let first = StudioStore.shared(owner: "user_a", appGroup: group) + let second = StudioStore.shared(owner: "user_b", appGroup: group) + defer { + try? FileManager.default.removeItem(at: first.root) + try? FileManager.default.removeItem(at: second.root) + } + try first.save(StudioProject(title: "Mine", camera: .init(file: "camera.mp4", offset: 0, duration: 3))) + #expect(first.all().map(\.title) == ["Mine"]) + #expect(second.all().isEmpty) + #expect(first.root != StudioStore.requests(appGroup: group).root) + #expect(StudioStore.folderName(for: "../../etc") == "etc") + #expect(StudioStore.folderName(for: "") == "unknown") + #expect(StudioStore.folderName(for: "user_9x-Q") == "user_9x-Q") + } + + @Test func lookDecodesFromEveryBackgroundKind() throws { + let backgrounds: [StudioBackground] = [.none, .color(StudioColor(1, 2, 3)), StudioBackground.gradients[3], .wallpaper("sonoma"), .image("bg.jpg")] + for background in backgrounds { + var look = StudioLook.studioDefault + look.background = background + let data = try JSONEncoder().encode(look) + #expect(try JSONDecoder().decode(StudioLook.self, from: data) == look) + } + } +} + +/// Real pixels through the real renderer and compositor, on this Mac's GPU. +@Suite(.serialized) struct StudioRenderingTests { + static func solid(_ size: CGSize, _ color: CIColor, renderer: StudioRenderer) -> CVPixelBuffer { + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, Int(size.width), Int(size.height), kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + renderer.context.render(CIImage(color: color).cropped(to: CGRect(origin: .zero, size: size)), to: buffer!, bounds: CGRect(origin: .zero, size: size), colorSpace: CGColorSpace(name: CGColorSpace.sRGB)!) + return buffer! + } + + /// Top-left origin, like the layout. + static func pixel(_ buffer: CVPixelBuffer, _ x: Int, _ y: Int) -> (r: Int, g: Int, b: Int) { + CVPixelBufferLockBaseAddress(buffer, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } + let row = CVPixelBufferGetBytesPerRow(buffer) + let base = CVPixelBufferGetBaseAddress(buffer)!.assumingMemoryBound(to: UInt8.self) + let p = base + y * row + x * 4 + return (Int(p[2]), Int(p[1]), Int(p[0])) + } + + @Test func drawsBackgroundScreenAndBubbleWhereTheLayoutSays() throws { + let renderer = StudioRenderer() + let screenSize = CGSize(width: 590, height: 1278) + let cameraSize = CGSize(width: 360, height: 640) + var look = StudioLook.studioDefault + look.background = .color(StudioColor(0, 0, 255)) + look.shadow = 0 + look.padding = 0.1 + look.aspect = .portrait + let scene = StudioScene(look: look, screen: StudioSourceInfo(naturalSize: screenSize), camera: StudioSourceInfo(naturalSize: cameraSize), renderer: renderer) + let canvas = CGSize(width: 1080, height: 1920) + let output = try #require(renderer.render(scene: scene, screen: Self.solid(screenSize, CIColor(red: 1, green: 0, blue: 0), renderer: renderer), camera: Self.solid(cameraSize, CIColor(red: 0, green: 1, blue: 0), renderer: renderer), size: canvas)) + let layout = StudioLayout(look: look, canvas: canvas, primary: screenSize, camera: cameraSize) + let screen = try #require(layout.screen?.frame) + let bubble = try #require(layout.camera?.frame) + #expect(Self.pixel(output, 5, 5) == (0, 0, 255)) + let red = Self.pixel(output, Int(screen.midX), Int(screen.minY + 40)) + #expect(red.r > 240 && red.g < 15 && red.b < 15) + let green = Self.pixel(output, Int(bubble.midX), Int(bubble.midY)) + #expect(green.g > 240 && green.r < 15) + // Outside the circle but inside its square: the screen shows through. + let corner = Self.pixel(output, Int(bubble.minX + 4), Int(bubble.minY + 4)) + #expect(corner.g < 40) + // The screen's rounded corner shows background. + let screenCorner = Self.pixel(output, Int(screen.minX + 2), Int(screen.minY + 2)) + #expect(screenCorner.b > 200 && screenCorner.r < 60) + } + + @Test func lookChangesApplyToTheNextFrameWithoutRebuilding() throws { + let renderer = StudioRenderer() + let size = CGSize(width: 400, height: 800) + let scene = StudioScene(look: .untouched, screen: StudioSourceInfo(naturalSize: size), camera: nil, renderer: renderer) + let red = Self.solid(size, CIColor(red: 1, green: 0, blue: 0), renderer: renderer) + let before = try #require(renderer.render(scene: scene, screen: red, camera: nil, size: size)) + #expect(Self.pixel(before, 2, 2).r > 240) + scene.update { $0.look.background = .color(StudioColor(0, 255, 0)); $0.look.padding = 0.1 } + let after = try #require(renderer.render(scene: scene, screen: red, camera: nil, size: size)) + #expect(Self.pixel(after, 2, 2).g > 240) + } + + @Test func rotatedSourcesAreUpright() throws { + let renderer = StudioRenderer() + // Landscape pixels tagged as portrait, like a phone camera file. + let natural = CGSize(width: 640, height: 360) + var buffer: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, 640, 360, kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + // Left half red, right half blue. + let image = CIImage(color: CIColor(red: 1, green: 0, blue: 0)).cropped(to: CGRect(x: 0, y: 0, width: 320, height: 360)) + .composited(over: CIImage(color: CIColor(red: 0, green: 0, blue: 1)).cropped(to: CGRect(x: 0, y: 0, width: 640, height: 360))) + renderer.context.render(image, to: buffer!, bounds: CGRect(x: 0, y: 0, width: 640, height: 360), colorSpace: CGColorSpace(name: CGColorSpace.sRGB)!) + let info = StudioSourceInfo(naturalSize: natural, transform: CGAffineTransform(a: 0, b: 1, c: -1, d: 0, tx: 360, ty: 0)) + #expect(info.displaySize == CGSize(width: 360, height: 640)) + let scene = StudioScene(look: .untouched, screen: info, camera: nil, renderer: renderer) + let output = try #require(renderer.render(scene: scene, screen: buffer, camera: nil, size: CGSize(width: 360, height: 640))) + // Rotated a quarter turn clockwise, the left (red) half ends up on top. + #expect(Self.pixel(output, 180, 100).r > 200) + #expect(Self.pixel(output, 180, 540).b > 200) + } + + @Test func pausedRedrawUsesTheFrameJustDrawn() throws { + let renderer = StudioRenderer() + let size = CGSize(width: 64, height: 128) + let scene = StudioScene(look: .untouched, screen: StudioSourceInfo(naturalSize: size), camera: nil, renderer: renderer) + let buffer = Self.solid(size, CIColor(red: 1, green: 0, blue: 0), renderer: renderer) + // Frames composed while playing later in the video, then a seek back to 11 s. + for index in 0..<10 { scene.remember(.init(time: CMTime(seconds: 24 + Double(index) / 30, preferredTimescale: 600), screen: buffer, camera: nil, size: size)) } + scene.remember(.init(time: CMTime(seconds: 11.2, preferredTimescale: 600), screen: buffer, camera: nil, size: size)) + #expect(scene.rememberedTimes.count == StudioScene.rememberedFrames) + #expect(scene.renderStill(near: CMTime(seconds: 11.21, preferredTimescale: 600)) != nil) + #expect(scene.renderStill(near: CMTime(seconds: 5, preferredTimescale: 600)) == nil) + // The same frames serve the preset pictures, and only for the moment they show. + #expect(scene.sources(near: CMTime(seconds: 11.3, preferredTimescale: 600), tolerance: 0.25)?.screen != nil) + #expect(scene.sources(near: CMTime(seconds: 5, preferredTimescale: 600), tolerance: 0.25) == nil) + } + + @Test func frameCostStaysSmall() throws { + let renderer = StudioRenderer() + let screenSize = CGSize(width: 1179, height: 2556) + let cameraSize = CGSize(width: 720, height: 1280) + let scene = StudioScene(look: .studioDefault, screen: StudioSourceInfo(naturalSize: screenSize), camera: StudioSourceInfo(naturalSize: cameraSize), renderer: renderer) + let screen = Self.solid(screenSize, CIColor(red: 0.9, green: 0.9, blue: 0.9), renderer: renderer) + let camera = Self.solid(cameraSize, CIColor(red: 0.5, green: 0.4, blue: 0.3), renderer: renderer) + let canvas = StudioLayout.canvasSize(look: .studioDefault, primary: screenSize, camera: cameraSize, longEdge: StudioLayout.exportLongEdge) + _ = renderer.render(scene: scene, screen: screen, camera: camera, size: canvas) + var times: [Double] = [] + for _ in 0..<60 { + let start = CFAbsoluteTimeGetCurrent() + _ = renderer.render(scene: scene, screen: screen, camera: camera, size: canvas) + times.append((CFAbsoluteTimeGetCurrent() - start) * 1000) + } + times.sort() + let p50 = times[times.count / 2] + let p95 = times[Int(Double(times.count) * 0.95)] + print("STUDIO_BENCH render \(Int(canvas.width))x\(Int(canvas.height)) p50 \(String(format: "%.2f", p50)) ms p95 \(String(format: "%.2f", p95)) ms") + // An M4 measures about 1.2 ms p50 and 1.5 ms p95, so anything near a 30 fps frame is a regression. CI's virtual + // Macs share a paravirtual GPU with other jobs and swing from 18 to 136 ms, so there it's only reported. + if ProcessInfo.processInfo.environment["CI"] == nil { + #expect(p95 < 33) + } + } +} + +/// Records real files with the writer, composes them and exports, as the app does. +@Suite(.serialized) struct StudioPipelineTests { + static func writeTrack(to url: URL, size: CGSize, seconds: Double, hostStart: Double, color: CIColor, audio: Bool) throws -> StudioTrackWriter { + let writer = try StudioTrackWriter(url: url, settings: .init(kind: .camera, audio: false)) + let renderer = StudioRenderer.shared + var format: CMVideoFormatDescription? + let frames = Int(seconds * 30) + for index in 0.. Double { + try await withCheckedThrowingContinuation { continuation in + writer.finish(at: nil) { continuation.resume(with: $0) } + } + } + + @Test func recordsComposesAndExports() async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "studio-pipeline-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let screenSize = CGSize(width: 590, height: 1278) + let cameraSize = CGSize(width: 360, height: 640) + let screenWriter = try Self.writeTrack(to: directory.appending(path: "screen.mp4"), size: screenSize, seconds: 4, hostStart: 100.5, color: CIColor(red: 0.2, green: 0.2, blue: 0.8), audio: false) + let cameraWriter = try Self.writeTrack(to: directory.appending(path: "camera.mp4"), size: cameraSize, seconds: 4, hostStart: 100, color: CIColor(red: 0.8, green: 0.5, blue: 0.2), audio: false) + #expect(screenWriter.hostStart == 100.5) + let screenDuration = try await Self.finish(screenWriter) + let cameraDuration = try await Self.finish(cameraWriter) + #expect(abs(screenDuration - 4) < 0.05) + let aligned = StudioTrackInfo.align( + screen: StudioTrackInfo(file: "screen.mp4", hostStart: 100.5, duration: screenDuration), + camera: StudioTrackInfo(file: "camera.mp4", hostStart: 100, duration: cameraDuration) + ) + let project = StudioProject(title: "Pipeline", screen: aligned.screen, camera: aligned.camera) + #expect(abs(project.duration - 4.5) < 0.05) + + let screen = try await StudioTrackSource.load(url: directory.appending(path: "screen.mp4"), offset: aligned.screen!.offset) + let camera = try await StudioTrackSource.load(url: directory.appending(path: "camera.mp4"), offset: aligned.camera!.offset) + let scene = StudioScene(look: .studioDefault, screen: screen.info, camera: camera.info) + var timeline = project.timeline + timeline = timeline.splitting(atOutput: 1)!.splitting(atOutput: 2)!.deleting(clipAt: 1)! + let built = try StudioComposition.build(clips: timeline.clips, screen: screen, camera: camera, music: nil, voiceVolume: 1, scene: scene, longEdge: 1280, remembersFrames: false) + #expect(abs(built.duration - timeline.outputDuration) < 0.02) + + let out = directory.appending(path: "export.mp4") + let export = try #require(AVAssetExportSession(asset: built.composition, presetName: AVAssetExportPresetHighestQuality)) + export.videoComposition = built.videoComposition + export.audioMix = built.audioMix + let started = CFAbsoluteTimeGetCurrent() + try await export.export(to: out, as: .mp4) + let took = CFAbsoluteTimeGetCurrent() - started + print("STUDIO_BENCH export \(String(format: "%.2f", built.duration)) s video in \(String(format: "%.2f", took)) s (\(String(format: "%.1f", built.duration / took))x realtime)") + let result = AVURLAsset(url: out) + let track = try #require(try await result.loadTracks(withMediaType: .video).first) + let size = try await track.load(.naturalSize) + #expect(size == built.renderSize) + #expect(abs(try await result.load(.duration).seconds - built.duration) < 0.1) + + // The same edit through Cap's own exporter, with music mixed in. + let musicURL = URL(fileURLWithPath: #filePath).deletingLastPathComponent().appending(path: "../../../Cap/Resources/Studio/Music/lofi-beats-mirostar.m4a").standardized + let music = try await StudioMusicSource.load(url: musicURL, volume: 0.4, offset: 2) + let withMusic = try StudioComposition.build(clips: timeline.clips, screen: screen, camera: camera, music: music, voiceVolume: 1, scene: scene.frozen(), longEdge: 1920, remembersFrames: false) + let ours = try await StudioExporter().export(withMusic, to: directory.appending(path: "ours.mp4")) + print("STUDIO_BENCH exporter \(Int(ours.size.width))x\(Int(ours.size.height)) \(String(format: "%.2f", ours.duration)) s in \(String(format: "%.2f", ours.seconds)) s (\(String(format: "%.1f", ours.duration / ours.seconds))x realtime), \(ours.bytes / 1024) KB, \(Int(Double(ours.bytes) * 8 / ours.duration / 1000)) kbps") + let oursAsset = AVURLAsset(url: ours.url) + #expect(try await oursAsset.loadTracks(withMediaType: .audio).count == 1) + #expect(abs(try await oursAsset.load(.duration).seconds - withMusic.duration) < 0.1) + let waveform = await StudioWaveform.peaks(url: musicURL, count: 100) + #expect(waveform.count == 100) + #expect(waveform.max() == 1) + + // The first exported frame is the composed one: background at the edge, screen in the middle. + let generator = AVAssetImageGenerator(asset: result) + generator.requestedTimeToleranceBefore = .zero + generator.requestedTimeToleranceAfter = .zero + let (frame, _) = try await generator.image(at: CMTime(value: 1, timescale: 2)) + let context = CIContext() + var buffer: CVPixelBuffer? + CVPixelBufferCreate(nil, frame.width, frame.height, kCVPixelFormatType_32BGRA, StudioRenderer.bufferAttributes as CFDictionary, &buffer) + context.render(CIImage(cgImage: frame), to: buffer!) + let edge = StudioRenderingTests.pixel(buffer!, 3, 3) + // Cap's default gradient starts blue at the top left. + #expect(edge.b > 150 && edge.r < 150) + } +} diff --git a/apps/mobile-ios/CapUITests/CapUITests.swift b/apps/mobile-ios/CapUITests/CapUITests.swift index 4c739e6f66e..89ac8808d29 100644 --- a/apps/mobile-ios/CapUITests/CapUITests.swift +++ b/apps/mobile-ios/CapUITests/CapUITests.swift @@ -260,15 +260,23 @@ final class CapUITests: XCTestCase { waitFor(app.buttons["Close"], 10) waitFor(app.staticTexts["Instant Mode"], 10) shot("75-record-modes") - for mode in ["Studio Mode", "Screenshot Mode"] { - let button = app.buttons[mode] - waitFor(button, 10) - XCTAssertEqual(button.value as? String, "Coming soon") - button.tap() - waitFor(app.staticTexts["\(mode) is coming soon"], 5) - shot("75-record-\(mode.split(separator: " ")[0].lowercased())-soon") - waitFor(app.staticTexts["Instant Mode"], 10) - } + let screenshot = app.buttons["Screenshot Mode"] + waitFor(screenshot, 10) + XCTAssertEqual(screenshot.value as? String, "Coming soon") + screenshot.tap() + waitFor(app.staticTexts["Screenshot Mode is coming soon"], 5) + shot("75-record-screenshot-soon") + waitFor(app.staticTexts["Instant Mode"], 10) + // Studio Mode is a real mode now: choosing it keeps it chosen. + let studio = app.buttons["Studio Mode"] + waitFor(studio, 10) + studio.tap() + waitFor(app.staticTexts["Studio Mode"], 5) + XCTAssertFalse(app.staticTexts["Free plan recordings stop at 5 minutes. Cap Pro has no limit."].exists) + shot("75-record-studio") + // Leave Instant Mode chosen for the tests that follow; the choice is remembered. + app.buttons["Instant Mode"].tap() + waitFor(app.staticTexts["Instant Mode"], 5) } func testLinkAccessSwitch() { diff --git a/apps/mobile-ios/CapUITests/StudioEditorUITests.swift b/apps/mobile-ios/CapUITests/StudioEditorUITests.swift new file mode 100644 index 00000000000..9599eef979d --- /dev/null +++ b/apps/mobile-ios/CapUITests/StudioEditorUITests.swift @@ -0,0 +1,236 @@ +import XCTest + +/// The Studio editor end to end on a recorded screen and camera (`CAP_STUDIO_FIXTURE`, a folder holding `screen.mp4` +/// and `camera.mp4`): every tool, the camera dragged to another corner, a cut, and a render saved to Photos. +final class StudioEditorUITests: XCTestCase { + private static let key = "0f3a5c2e-6b4d-4a7f-9e1c-2d3b4a5c6d7e" + + private var fixtureURL: String { + ProcessInfo.processInfo.environment["CAP_FIXTURE_URL"] ?? "http://localhost:4310" + } + + override func setUpWithError() throws { + continueAfterFailure = false + try XCTSkipIf(ProcessInfo.processInfo.environment["CAP_STUDIO_FIXTURE"] == nil, "Set CAP_STUDIO_FIXTURE to a folder with screen.mp4 and camera.mp4") + } + + private func save(_ name: String) { + let screenshot = XCUIScreen.main.screenshot() + let attachment = XCTAttachment(screenshot: screenshot) + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + guard let directory = ProcessInfo.processInfo.environment["CAP_SHOT_DIR"] else { return } + try? screenshot.pngRepresentation.write(to: URL(fileURLWithPath: directory).appendingPathComponent("\(name).png")) + } + + private func openStudio(script: Bool = false) -> XCUIApplication { + let app = XCUIApplication() + app.launchEnvironment["CAP_API_BASE_URL"] = fixtureURL + app.launchEnvironment["CAP_DEBUG_SESSION"] = "\(Self.key):user_richie" + app.launchEnvironment["CAP_DEBUG_ROUTE"] = "studio:\(ProcessInfo.processInfo.environment["CAP_STUDIO_FIXTURE"]!)" + app.launchEnvironment["CAP_DEBUG_STUDIO_CAMERA_OFFSET"] = "0.4" + if script { app.launchEnvironment["CAP_DEBUG_EDITOR_SCRIPT"] = "1" } + app.launch() + XCTAssertTrue(app.buttons["Split"].waitForExistence(timeout: 30)) + return app + } + + func testEveryTool() throws { + let app = openStudio() + sleep(2) + save("s00-studio-open") + + app.buttons["editor-tool-presets"].tap() + let ocean = app.buttons["Ocean"] + XCTAssertTrue(ocean.waitForExistence(timeout: 5)) + // Saved presets outlive the app, so one left by an earlier run that stopped part way is cleared first. + for _ in 0..<3 where app.buttons["Launch"].exists { + app.buttons["Launch"].press(forDuration: 1) + if app.buttons["Delete"].waitForExistence(timeout: 3) { app.buttons["Delete"].tap() } + sleep(1) + } + sleep(2) + save("s00b-presets") + reveal(ocean, in: app) + ocean.tap() + sleep(2) + XCTAssertTrue(ocean.isSelected) + save("s00c-preset-ocean") + // A paused seek redraws the pictures at the new moment. + timelineTap(app, 0.75) + sleep(2) + save("s00c2-preset-after-seek") + timelineTap(app, 0.02) + sleep(1) + // Already a preset, so saving it again isn't offered until it's changed. + XCTAssertFalse(app.buttons["Save as Preset"].firstMatch.isEnabled) + app.buttons["editor-tool-frame"].tap() + app.buttons["4:5"].tap() + sleep(1) + app.buttons["editor-tool-presets"].tap() + XCTAssertFalse(ocean.isSelected) + // Saved as one of your own, then made the look new recordings start with. + app.buttons["Save as Preset"].firstMatch.tap() + XCTAssertTrue(app.alerts["Save Preset"].waitForExistence(timeout: 5)) + let field = app.alerts["Save Preset"].textFields.firstMatch + field.tap() + field.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: 24) + "Launch") + app.alerts["Save Preset"].buttons["Save"].tap() + let mine = app.buttons["Launch"] + XCTAssertTrue(mine.waitForExistence(timeout: 5)) + XCTAssertTrue(mine.isSelected) + mine.press(forDuration: 1) + let pin = app.buttons["Use for New Recordings"] + XCTAssertTrue(pin.waitForExistence(timeout: 5)) + pin.tap() + sleep(1) + XCTAssertEqual(app.buttons["Launch"].value as? String, "Used for new recordings") + save("s00d-preset-saved") + // Deleted again, which also hands new recordings back to Cap's Studio look. + mine.press(forDuration: 1) + let delete = app.buttons["Delete"] + XCTAssertTrue(delete.waitForExistence(timeout: 5)) + delete.tap() + XCTAssertTrue(app.buttons["Launch"].waitForNonExistence(timeout: 5)) + XCTAssertEqual(app.buttons["Studio"].value as? String, "Used for new recordings") + reveal(app.buttons["Studio"], in: app) + app.buttons["Studio"].tap() + sleep(1) + + app.buttons["editor-tool-background"].tap() + sleep(1) + save("s01-background") + // The strip opens on the chosen background; the wallpapers are just before it. + let wallpapers = app.buttons.matching(identifier: "Wallpaper").allElementsBoundByIndex + let visible = try XCTUnwrap(wallpapers.last { $0.isHittable }) + visible.tap() + sleep(1) + save("s02-wallpaper") + + app.buttons["editor-tool-frame"].tap() + sleep(1) + app.buttons["1:1"].tap() + sleep(2) + save("s03-square") + app.buttons["Original"].tap() + sleep(1) + + app.buttons["editor-tool-camera"].tap() + // Past the camera's late start, so the bubble has a picture. + timelineTap(app, 0.3) + sleep(2) + save("s04-camera") + // Drag the bubble from its corner to the top left. + let preview = app.descendants(matching: .any)["editor-preview"].firstMatch + let bubble = app.descendants(matching: .any)["editor-camera-bubble"].firstMatch + XCTAssertTrue(bubble.exists) + let before = bubble.frame + bubble.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + .press(forDuration: 0.1, thenDragTo: preview.coordinate(withNormalizedOffset: CGVector(dx: 0.15, dy: 0.12))) + sleep(2) + let after = app.descendants(matching: .any)["editor-camera-bubble"].firstMatch.frame + XCTAssertLessThan(after.midX, before.midX) + XCTAssertLessThan(after.midY, before.midY) + save("s05-camera-moved") + app.buttons["Split"].firstMatch.tap() + sleep(2) + save("s06-split-layout") + app.buttons["Bubble"].tap() + sleep(1) + + app.buttons["editor-tool-music"].tap() + sleep(1) + app.buttons["Lofi Beats"].tap() + sleep(2) + save("s07-music") + + app.buttons["editor-tool-trim"].tap() + sleep(1) + let timeline = app.descendants(matching: .any)["editor-timeline"].firstMatch + timeline.coordinate(withNormalizedOffset: CGVector(dx: 0.4, dy: 0.5)).tap() + sleep(1) + app.buttons["Split"].tap() + sleep(1) + XCTAssertTrue(app.descendants(matching: .any).matching(NSPredicate(format: "label BEGINSWITH 'Clip 2'")).firstMatch.exists) + save("s08-split") + app.buttons["Undo"].tap() + sleep(1) + XCTAssertFalse(app.descendants(matching: .any).matching(NSPredicate(format: "label BEGINSWITH 'Clip 2'")).firstMatch.exists) + } + + /// Preset cards sit in a lazy horizontal strip; one further along is scrolled into view first. + private func reveal(_ card: XCUIElement, in app: XCUIApplication) { + let width = app.windows.firstMatch.frame.width + for _ in 0..<6 where !(card.exists && card.frame.minX >= 0 && card.frame.maxX <= width) { + let strip = app.buttons["Save as Preset"].firstMatch.exists ? app.buttons["Save as Preset"].firstMatch.frame.midY : card.frame.midY + let y = strip / app.windows.firstMatch.frame.height + let window = app.windows.firstMatch + let toLeft = !card.exists || card.frame.minX > width / 2 + window.coordinate(withNormalizedOffset: CGVector(dx: toLeft ? 0.8 : 0.2, dy: y)) + .press(forDuration: 0.05, thenDragTo: window.coordinate(withNormalizedOffset: CGVector(dx: toLeft ? 0.3 : 0.7, dy: y)), withVelocity: 600, thenHoldForDuration: 0.1) + } + } + + private func timelineTap(_ app: XCUIApplication, _ fraction: CGFloat) { + app.descendants(matching: .any)["editor-timeline"].firstMatch.coordinate(withNormalizedOffset: CGVector(dx: fraction, dy: 0.5)).tap() + } + + /// Records the camera in Studio Mode with a video played in as the camera (the simulator has none), through the + /// real encoder, then lands in the editor with the recording kept as a draft. + func testStudioCameraRecordingOpensEditor() throws { + let app = XCUIApplication() + app.launchEnvironment["CAP_API_BASE_URL"] = fixtureURL + app.launchEnvironment["CAP_DEBUG_SESSION"] = "\(Self.key):user_richie" + app.launchEnvironment["CAP_DEBUG_RECORDER_FILE"] = ProcessInfo.processInfo.environment["CAP_STUDIO_FIXTURE"]! + "/camera.mp4" + app.launch() + let tab = app.tabBars.buttons["Record"] + XCTAssertTrue(tab.waitForExistence(timeout: 20)) + let hittable = expectation(for: NSPredicate(format: "isHittable == true"), evaluatedWith: tab) + wait(for: [hittable], timeout: 10) + tab.tap() + let studio = app.buttons["Studio Mode"] + if studio.waitForExistence(timeout: 10) { studio.tap() } + XCTAssertTrue(app.staticTexts["Studio Mode"].waitForExistence(timeout: 5)) + app.buttons["Camera"].firstMatch.tap() + let record = app.buttons["Start recording"] + XCTAssertTrue(record.waitForExistence(timeout: 15)) + let enabled = expectation(for: NSPredicate(format: "isEnabled == true"), evaluatedWith: record) + wait(for: [enabled], timeout: 15) + save("s20-studio-recorder") + record.tap() + let started = Date() + sleep(5) + save("s21-studio-recording") + app.buttons["Stop recording"].tap() + let recorded = Date().timeIntervalSince(started) + // The editor opens on the recording. + let opened = app.buttons["Split"].waitForExistence(timeout: 20) + if !opened { save("s21b-after-stop") } + XCTAssertTrue(opened) + XCTAssertTrue(app.navigationBars["Studio"].exists) + sleep(2) + save("s22-studio-recorded") + let length = app.staticTexts.matching(NSPredicate(format: "label BEGINSWITH 'Edited length'")).firstMatch + XCTAssertTrue(length.exists) + // As long as the time between the taps (a screenshot in between can stretch it); never the debug file's own length. + let parts = length.label.replacingOccurrences(of: "Edited length ", with: "").split(separator: ":").compactMap { Double($0) } + let seconds = parts.count == 2 ? parts[0] * 60 + parts[1] : -1 + XCTAssertLessThanOrEqual(abs(seconds - recorded), 2, "\(length.label) after \(recorded) s") + app.buttons["Done"].tap() + // Back on the recorder, the recording waits as a draft. + XCTAssertTrue(app.descendants(matching: .any)["studio-drafts"].waitForExistence(timeout: 10)) + save("s23-studio-drafts") + app.buttons["Instant Mode"].tap() + } + + /// Runs the editor's own scripted session and leaves `EditorStats.jsonl` in the app's caches. + func testScriptedMeasurements() throws { + let app = openStudio(script: true) + // The script closes the editor when it's done. + let closed = app.buttons["Split"].waitForNonExistence(timeout: 120) + if !closed { save("s30-script-stuck") } + XCTAssertTrue(closed) + sleep(1) + } +} diff --git a/apps/mobile-ios/README.md b/apps/mobile-ios/README.md index 8825abcb00d..4e055357dce 100644 --- a/apps/mobile-ios/README.md +++ b/apps/mobile-ios/README.md @@ -20,6 +20,17 @@ Camera and screen recordings upload while they're being made, so the link works - Camera mode (`Cap/Features/Record`) runs `AVCaptureSession` at 1080×1920, 30 fps, about 4 Mbps. The camera can switch mid-recording. It stops and keeps the recording if iOS takes the camera away (Cap moves to the background, a call comes in). The free plan stops at 5 minutes. - Screen mode starts the system broadcast sheet preset to Cap. The extension reads the app's session from the shared keychain group, records the screen with the microphone (or app audio when the mic is off), uploads as it goes, copies the link when it stops, and leaves a handoff so opening Cap lands on the new recording. Static screens send no frames, so the last one is repeated to keep segments regular. +## Studio Mode + +Studio Mode keeps a recording on the phone as separate tracks, so it can be laid out, styled and cut before anything is shared. Instant Mode uploads as it records; Studio Mode uploads once, the finished video. + +- Camera records into `camera.mp4` from the app. Screen records into `screen.mp4` from the broadcast extension: when Studio's screen option is showing, the app leaves a `StudioScreenRequest` in the app group, and an extension that starts within its lifetime takes it and writes to the phone instead of uploading. Screen + Camera records both at once, offered only where iOS keeps the camera running behind other apps (`isMultitaskingCameraAccessSupported`). +- `StudioTrackWriter` (CapKit) writes HEVC with a keyframe every second and a fragment every second, so a recording cut short by a crash keeps all but its last second and is recovered as a draft on the next launch. Each track notes the host clock time of its first frame as soon as it has one (`screen.json`, `camera.json`); camera times are converted from the capture session's clock. Tracks are lined up by those times, and stretches with no frames (a camera iOS paused) are recorded as gaps and shown as nothing, never as a frozen frame. +- The editor builds an `AVComposition` of references to the recorded files, so a cut rebuilds in under a millisecond. Frames are drawn by `StudioCompositor` with Core Image on the GPU: background, padding, rounded corners and shadow are drawn once per look into a cached layer, so each frame is a few texture samples. A look change while paused redraws the last decoded frame instead of asking the decoder again. The same compositor renders the export, so the preview and the shared video can't differ. +- `StudioExporter` renders H.264 at about 0.1 bits per pixel per frame (1080p class) with AAC, while Cap is in front: iOS refuses GPU work in the background. The file then goes through the usual background upload as a new Cap, or to Photos. +- The same editor opens on an existing Cap. Cuts alone still save on Cap's servers; a background, frame, or music renders on the phone and uploads as a new Cap, leaving the original as it is. +- `StudioSyncTests` records screen and camera that flash, and a voice that beeps, at the same host clock moments, with different start times, 29.97 against 30 fps, timestamp jitter, dropped frames, a camera gap and cuts, then checks every flash lands on exactly the frame its timestamp says and the voice within a few milliseconds. + ## Usage analytics The app records what people do as named events with a few plain values (`recording_started` with `mode: camera`), never titles, links, ids or anything typed. Events go to the Cap server the app is signed in to, at `/api/mobile/events`, which forwards them to its own analytics when that is configured and discards them when it is not. A server that has no such endpoint is asked once per launch. diff --git a/apps/web/__tests__/unit/mobile-api-contract.test.ts b/apps/web/__tests__/unit/mobile-api-contract.test.ts index 4cef5e49ffe..6a73802516c 100644 --- a/apps/web/__tests__/unit/mobile-api-contract.test.ts +++ b/apps/web/__tests__/unit/mobile-api-contract.test.ts @@ -31,6 +31,46 @@ describe("mobile API contract schemas", () => { ).toBe(true); }); + it("holds a whole uploaded video to the free limit and needs its length to", () => { + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: 305, + isPro: false, + }), + ).toBe(true); + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: 900, + isPro: false, + }), + ).toBe(false); + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: 900, + isPro: true, + }), + ).toBe(true); + // Leaving the length out would skip the limit, so the free plan needs it. + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: undefined, + isPro: false, + }), + ).toBe(false); + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: -1, + isPro: false, + }), + ).toBe(false); + expect( + Mobile.isMobileUploadDurationAllowed({ + durationSeconds: undefined, + isPro: true, + }), + ).toBe(true); + }); + it("gates free recordings on the segments when the reported total is lower", () => { const segments = Array.from({ length: 300 }, () => ({ duration: 2 })); const gated = Mobile.mobileRecordingGateDurationSeconds({ diff --git a/apps/web/app/api/mobile/[...route]/route.ts b/apps/web/app/api/mobile/[...route]/route.ts index 0c256a318ea..960f339e2fe 100644 --- a/apps/web/app/api/mobile/[...route]/route.ts +++ b/apps/web/app/api/mobile/[...route]/route.ts @@ -3326,6 +3326,29 @@ const createUpload = Effect.fn("Mobile.createUpload")(function* ( } const database = yield* Database; + + if (!screenshotExtension) { + const [userRow] = yield* database.use((db) => + db + .select({ + stripeSubscriptionStatus: Db.users.stripeSubscriptionStatus, + thirdPartyStripeSubscriptionId: + Db.users.thirdPartyStripeSubscriptionId, + }) + .from(Db.users) + .where(eq(Db.users.id, user.id)) + .limit(1), + ); + if ( + !Mobile.isMobileUploadDurationAllowed({ + durationSeconds: input.durationSeconds, + isPro: userIsPro(userRow), + }) + ) { + return yield* Effect.fail(new HttpApiError.Forbidden()); + } + } + const storage = yield* Storage; const repo = yield* VideosRepo; const folderId = input.folderId; diff --git a/packages/web-domain/src/Mobile.ts b/packages/web-domain/src/Mobile.ts index 2056a655053..16365f1cad6 100644 --- a/packages/web-domain/src/Mobile.ts +++ b/packages/web-domain/src/Mobile.ts @@ -32,6 +32,22 @@ export const isMobileRecordingDurationAllowed = ({ mobileFreeRecordingDurationSeconds + mobileRecordingDurationToleranceSeconds; +// A whole video uploaded at once (an edit rendered on the phone, a video from Photos) is held to the same limit as +// a recording. Its length is client-reported, as on the other upload routes, so on the free plan one that doesn't +// say how long it is isn't accepted: leaving it out would skip the limit. +export const isMobileUploadDurationAllowed = ({ + durationSeconds, + isPro, +}: { + durationSeconds: number | undefined; + isPro: boolean; +}) => + isPro || + (durationSeconds !== undefined && + Number.isFinite(durationSeconds) && + durationSeconds > 0 && + isMobileRecordingDurationAllowed({ durationSeconds, isPro })); + // The total and the segment durations are both client-reported; gating on the larger stops a client that // only understates the total. Authoritative enforcement needs the length measured after processing. export const mobileRecordingGateDurationSeconds = ({