diff --git a/.cargo/config.toml b/.cargo/config.toml index 533e9503..6d017ab4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,18 @@ # tauri.conf.json. Keep the two in sync. [env] MACOSX_DEPLOYMENT_TARGET = { value = "11.0", force = true } + +# whisper.cpp is compiled as a static library, but CMake otherwise selects the +# dynamic MSVC C/C++ runtime on Windows. That leaves the installed application +# dependent on whichever MSVCP140/VCRUNTIME140 version happens to be present on +# the machine and can fail before Rust's main() with STATUS_ENTRYPOINT_NOT_FOUND. +# CMP0091 makes CMAKE_MSVC_RUNTIME_LIBRARY authoritative for the older +# whisper.cpp CMake project; non-MSVC generators ignore both values. +CMAKE_POLICY_DEFAULT_CMP0091 = { value = "NEW", force = false } +CMAKE_MSVC_RUNTIME_LIBRARY = { value = "MultiThreaded", force = false } + +# Rust's MSVC target otherwise links VCRUNTIME/UCRT dynamically even when the +# C++ libraries above use /MT. Keep the Windows executable self-contained and +# make every cc-rs consumer observe the same crt-static target feature. +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df91812d..97b575a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,33 @@ concurrency: cancel-in-progress: true jobs: + motion-canvas: + name: Motion Canvas (locked build / audit / license) + if: github.event_name != 'workflow_dispatch' || inputs.red_task == 'none' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: plugins/motion-canvas-studio/package-lock.json + + - name: Install locked Motion Canvas dependencies + run: npm --prefix plugins/motion-canvas-studio ci --ignore-scripts + + - name: Audit dependencies and licenses + run: | + npm --prefix plugins/motion-canvas-studio audit --audit-level=moderate + npm --prefix plugins/motion-canvas-studio run licenses + + - name: Test and reproduce embedded runner + run: | + npm --prefix plugins/motion-canvas-studio test + npm --prefix plugins/motion-canvas-studio run build + git diff --exit-code -- plugins/motion-canvas-studio/bundle/runner.html plugins/motion-canvas-studio/package-lock.json + rust: name: Rust (fmt / clippy / test) if: github.event_name != 'workflow_dispatch' || inputs.red_task == 'none' @@ -78,7 +105,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml', 'Cargo.lock') }} restore-keys: ${{ runner.os }}-cargo- - name: Validate Windows product gate contract @@ -98,6 +125,8 @@ jobs: run: cargo clippy --workspace --all-targets -- -D warnings - name: cargo test + env: + OPENTAKE_MOTION_TRACE: '1' run: cargo test --workspace - name: live playback transport integration (fail closed) @@ -160,8 +189,16 @@ jobs: cache: pnpm cache-dependency-path: web/pnpm-lock.yaml - - name: Install FFmpeg - run: choco install ffmpeg --no-progress -y + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + + - name: Provision checksum-pinned packaged FFmpeg sidecars + run: python scripts/provision_ffmpeg_sidecars.py --target x86_64-pc-windows-msvc + + - name: Verify sidecar supply and probe/decode/encode boundary without PATH + shell: bash + run: ruby scripts/tests/packaged-sidecars-test.rb --name packaged_macos_windows_sidecars_resolve_and_execute - name: Cache cargo uses: actions/cache@v4 @@ -181,10 +218,27 @@ jobs: - name: Rust workspace clippy run: cargo clippy --workspace --all-targets -- -D warnings + - name: Windows Chromium motion capture regression + shell: bash + env: + OPENTAKE_MOTION_TRACE: '1' + run: | + cargo test -p opentake-motion --features chromium --lib \ + renderer::tests::chromium_skeleton_reports_unavailable_not_panic \ + -- --exact --nocapture --test-threads=1 + cargo test -p opentake-motion --features chromium --test chromium \ + virtual_time_network_csp_timeout_cleanup_and_frame_identity \ + -- --exact --nocapture --test-threads=1 + cargo test -p opentake-tauri --test motion_command \ + sandbox_progress_cancel_validated_mp4_result \ + -- --exact --nocapture --test-threads=1 + - name: Web editor behavior suite run: pnpm -C web test - name: Rust workspace tests + env: + OPENTAKE_MOTION_TRACE: '1' run: cargo test --workspace -- --test-threads=1 - name: Minimal-feature Tauri clippy @@ -200,6 +254,40 @@ jobs: Remove-Item 'target/release/bundle/nsis' -Recurse -Force -ErrorAction SilentlyContinue & .\web\node_modules\.bin\tauri.cmd build --ci --bundles msi,nsis + - name: Install NSIS package and execute installed product without PATH + shell: pwsh + run: | + $installer = @(Get-ChildItem 'target/release/bundle/nsis/*.exe' -File) + if ($installer.Count -ne 1) { throw 'expected exactly one NSIS installer' } + $process = Start-Process -FilePath $installer[0].FullName -ArgumentList '/S' -Wait -PassThru + if ($process.ExitCode -ne 0) { throw "silent NSIS install failed: $($process.ExitCode)" } + $candidates = @( + (Join-Path $env:LOCALAPPDATA 'OpenTake/opentake.exe'), + (Join-Path $env:LOCALAPPDATA 'Programs/OpenTake/opentake.exe'), + (Join-Path $env:ProgramFiles 'OpenTake/opentake.exe') + ) + $application = $candidates | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $application) { + $uninstall = Get-ItemProperty 'HKCU:\Software\Microsoft\Windows\CurrentVersion\Uninstall\*' ` + -ErrorAction SilentlyContinue | Where-Object { $_.DisplayName -eq 'OpenTake' } | Select-Object -First 1 + if ($uninstall.InstallLocation) { + $application = Get-ChildItem $uninstall.InstallLocation -Filter 'opentake.exe' -File ` + -ErrorAction SilentlyContinue | Select-Object -ExpandProperty FullName -First 1 + } + } + if (-not $application) { throw "installed OpenTake executable not found: $($candidates -join ', ')" } + $installDirectory = Split-Path -Parent $application + ruby scripts/tests/packaged-sidecars-test.rb ` + --name packaged_macos_windows_sidecars_resolve_and_execute ` + --package $installDirectory + $app = Start-Process -FilePath $application -PassThru + Start-Sleep -Seconds 5 + if ($app.HasExited) { + throw "installed OpenTake exited during launch smoke test: $($app.ExitCode)" + } + Stop-Process -Id $app.Id -Force + Wait-Process -Id $app.Id -ErrorAction SilentlyContinue + - name: Bind installers to the exact source SHA shell: pwsh env: @@ -249,8 +337,8 @@ jobs: - name: Install Rust toolchain run: rustup component add rustfmt - - name: Install FFmpeg - run: choco install ffmpeg --no-progress -y + - name: Provision checksum-pinned packaged FFmpeg sidecars + run: python scripts/provision_ffmpeg_sidecars.py --target x86_64-pc-windows-msvc - name: Cache cargo uses: actions/cache@v4 @@ -258,16 +346,106 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - target - key: ${{ runner.os }}-security-cargo-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-security-cargo-${{ hashFiles('**/Cargo.toml', 'Cargo.lock') }} restore-keys: ${{ runner.os }}-security-cargo- - name: Portable FFmpeg cancellation lifecycle shell: pwsh + env: + OPENTAKE_FFMPEG: ${{ github.workspace }}\src-tauri\binaries\ffmpeg-x86_64-pc-windows-msvc.exe run: | + & $env:OPENTAKE_FFMPEG -version + if ($LASTEXITCODE -ne 0) { throw 'checksum-pinned packaged FFmpeg is not runnable' } cargo test -p opentake-media --lib windows_cancelling_running_pcm_child_reaps_both_pipe_readers cargo test -p opentake-media --lib windows_cancelling_mux_wait_reaps_child + - name: Verify portable Tauri test image imports + shell: pwsh + run: | + cargo test -p opentake-tauri --lib --no-run + $testImage = Get-ChildItem 'target/debug/deps/opentake_tauri_lib-*.exe' -File | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + if (-not $testImage) { throw 'compiled Tauri test image not found' } + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' + $dumpbin = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -find 'VC/Tools/MSVC/**/bin/Hostx64/x64/dumpbin.exe' | + Select-Object -First 1 + if (-not $dumpbin) { throw 'dumpbin.exe not found' } + & $dumpbin /DEPENDENTS $testImage.FullName | Set-Content windows-tauri-test-imports.txt + & $dumpbin /IMPORTS $testImage.FullName | Add-Content windows-tauri-test-imports.txt + $imports = Get-Content windows-tauri-test-imports.txt -Raw + if ($imports -match '(?im)^\s*(MSVCP140|VCRUNTIME140(?:_1)?|api-ms-win-crt-[^\s]+|onnxruntime|DirectML)\.dll\s*$') { + throw 'Tauri test image retains a non-portable native runtime dependency' + } + + $mt = Get-ChildItem (Join-Path ${env:ProgramFiles(x86)} 'Windows Kits/10/bin') ` + -Filter mt.exe -File -Recurse | + Where-Object { $_.FullName -match '\\x64\\mt\.exe$' } | + Sort-Object FullName -Descending | + Select-Object -First 1 + if (-not $mt) { throw 'mt.exe not found' } + $manifestLog = & $mt.FullName "-inputresource:$($testImage.FullName);#1" ` + '-out:windows-tauri-test.manifest' 2>&1 + $manifestLog | Set-Content windows-tauri-test-manifest.txt + if ($LASTEXITCODE -ne 0) { throw 'Tauri test image has no readable RT_MANIFEST resource #1' } + Get-Content windows-tauri-test.manifest | Add-Content windows-tauri-test-manifest.txt + if ((Get-Content windows-tauri-test.manifest -Raw) -notmatch 'Microsoft\.Windows\.Common-Controls') { + throw 'Tauri test image manifest does not activate Common Controls v6' + } + + Add-Type @' + using System; + using System.Runtime.InteropServices; + public static class OpenTakeNativeExportProbe { + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr LoadLibraryExW(string path, IntPtr file, uint flags); + [DllImport("kernel32.dll", CharSet = CharSet.Ansi, SetLastError = true, ExactSpelling = true)] + public static extern IntPtr GetProcAddress(IntPtr module, string name); + } + '@ + $modules = @{} + $currentDll = $null + $inImportSection = $false + $missingExports = @() + foreach ($line in (Get-Content windows-tauri-test-imports.txt)) { + if ($line -match '^\s*Section contains the following imports:\s*$') { + $inImportSection = $true + $currentDll = $null + continue + } + if ($line -match '^\s*Summary\s*$') { + $inImportSection = $false + $currentDll = $null + continue + } + if (-not $inImportSection) { continue } + if ($line -match '^\s{4}([A-Za-z0-9_.-]+\.dll)\s*$') { + $currentDll = $Matches[1] + if (-not $modules.ContainsKey($currentDll)) { + $modules[$currentDll] = [OpenTakeNativeExportProbe]::LoadLibraryExW( + $currentDll, [IntPtr]::Zero, 0x00000800 + ) + } + } elseif ($currentDll -and $line -match '^\s+[0-9A-F]+\s+(\S+)\s*$') { + $name = $Matches[1] + $module = $modules[$currentDll] + $activationContextExport = + $currentDll -ieq 'comctl32.dll' -and $name -eq 'TaskDialogIndirect' + if (-not $activationContextExport -and ($module -eq [IntPtr]::Zero -or + [OpenTakeNativeExportProbe]::GetProcAddress($module, $name) -eq [IntPtr]::Zero)) { + $missingExports += "$currentDll!$name" + } + } + } + $modules.GetEnumerator() | Sort-Object Name | ForEach-Object { + "LOAD $($_.Name)=$($_.Value)" + } | Set-Content windows-tauri-export-probe.txt + $missingExports | ForEach-Object { "MISSING $_" } | Add-Content windows-tauri-export-probe.txt + if ($missingExports.Count -ne 0) { + throw "Tauri test image imports unavailable system exports: $($missingExports -join ', ')" + } + - name: Reserved output identity and reparse safety shell: pwsh run: | @@ -275,6 +453,19 @@ jobs: cargo test -p opentake-tauri --lib windows_directory_handoff_blocks_junction_replacement_before_child_create cargo test -p opentake-tauri --lib windows_retained_output_handle_blocks_final_name_replacement + - name: Upload Tauri test image imports + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-tauri-test-imports-${{ github.sha }} + path: | + windows-tauri-test-imports.txt + windows-tauri-test-manifest.txt + windows-tauri-test.manifest + windows-tauri-export-probe.txt + if-no-files-found: error + retention-days: 7 + web: name: Web (install / build) if: github.event_name != 'workflow_dispatch' || inputs.red_task == 'none' @@ -311,14 +502,16 @@ jobs: - name: Install Rust toolchain run: rustup component add rustfmt clippy + - name: Provision checksum-pinned packaged FFmpeg sidecars + run: python scripts/provision_ffmpeg_sidecars.py --target x86_64-pc-windows-msvc + - name: Cache cargo uses: actions/cache@v4 with: path: | ~/.cargo/registry ~/.cargo/git - target - key: ${{ runner.os }}-library-security-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-library-security-${{ hashFiles('**/Cargo.toml', 'Cargo.lock') }} restore-keys: ${{ runner.os }}-library-security- - name: Test retained-handle and junction defenses diff --git a/.gitignore b/.gitignore index 36988ff9..211fe639 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ dist/ # Tauri src-tauri/target/ src-tauri/gen/ +src-tauri/binaries/ffmpeg-* +src-tauri/binaries/ffprobe-* # Env / secrets (never commit API keys) .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3e8cb7..3bba4dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,40 @@ 本文件记录 OpenTake 的重要改动。格式参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/)。 +## [1.0.0-beta.1] — 2026-08-01 + +OpenTake 的第一个可安装 Beta。核心闭环为:创建/打开工程 → 导入与管理素材 → +多轨剪辑、字幕、关键帧、特效/蒙版/调色/转场 → 本地 AI 与生成式 AI 审阅工作流 → +预览、保存重开、H.264/H.265/ProRes 与字幕/交换格式导出。 + +### 新增(Added) + +- 可恢复的工程持久化、全局素材库、缩略图/波形/代理媒体、缺失素材重链接。 +- Rust/WGPU 预览与导出共享合成路径,支持文本、调色、绿幕、蒙版、LUT、HSL、 + Lift/Gamma/Gain、通用特效、交叉溶解、嵌套时间线、补帧与防抖。 +- Agent/MCP 编辑、内置聊天、官方 Codex CLI / ChatGPT 登录、BYOK 生成作业、Motion Canvas + 动效与原生 Chromium fallback。Codex 登录态完全由官方 CLI 管理,OpenTake 不读取或保存令牌。 +- 本地口播清理、响度统一、降噪、声部分离、RVM 抠像、智能擦除、参考色彩匹配和 + 可视化运动追踪;字幕翻译、图文成片、数字人与音色克隆提供审阅/同意/成本边界。 +- 完整键盘、焦点、菜单、拖拽、撤销/重做与辅助功能回归门禁。 + +### 安全与可靠性(Security / Reliability) + +- 生产 CSP、最小 asset scope、凭据 keychain 边界、URL/重定向/大小限制、下载校验与 + 项目修订原子提交。 +- macOS/Linux/Windows 安全文件系统契约、打包 FFmpeg sidecar 供应链校验、取消与失败 + 清理、生成/导出恢复与防陈旧结果提交。 + +### Beta 已知边界 + +- 本地 macOS Beta 包未使用 Developer ID 签名或 Apple 公证;首次打开需要用户明确允许。 +- 数字人、音色克隆和通用云生成需要用户自己的 provider key,并可能产生第三方费用; + Agent 可选择 provider key,也可复用官方 Codex CLI 的 ChatGPT 登录。无可用登录或 key 时 + 功能会显式不可用或拒绝,不会静默调用。 +- Windows 安装包由精确 SHA CI 构建和验证;原生 Windows WebView 的最终人工交互烟测仍是 + 平台发布门槛,不影响本次 Apple Silicon macOS 本地 Beta。 +- 任意 Motion Canvas TSX、透明动效、神经语义级任意人声分离等属于后续 Beta 范围。 + ## [未发布] — 2026-06-23 第三轮(自动 PR 审核:全局素材库 + 文本工具 + 字幕/视频导出 + list_models) 本轮为**自动 PR 审核流程**:逐 PR 专家审核 + 对抗验证 + 对照开发文档,审核通过且 CI 双绿的纯新增项合并,其余 @作者 rebase/修改。 diff --git a/CLAUDE.md b/CLAUDE.md index 0820c5de..916bb5a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,13 +70,13 @@ - **#51/#52 合成预览**(PR #59):时间线标签按播放头贴 GPU 合成帧(视频+图)。 - **#61/#62**:多素材拖入、保存/自动保存/退出 flush、预览整数帧、音频探测、播放卡顿缓解。 - **#36 MCP 工具派发层 + Skills**(PR #66/#67):单一能力派发(25 工具接线:18 EditCommand + rename/delete + workflow/Skills)+ 默认"音频先入"内置 Skill(`crates/opentake-agent/src/plugin/builtin/audio-first/`)。 -- **#65 文字光栅化**(PR #68):`CosmicTextRasterizer`(cosmic-text+swash)把文字 clip 框渲染为预乘 RGBA,经既有 affine 1:1 合成置顶(对应上游 CATextLayer);字体/字号/颜色/对齐/背景/投影/边框全覆盖;真机视觉自检中英混排正常。**剩 Lottie 烘焙**。 +- **#65 文字/Lottie 光栅化**:`CosmicTextRasterizer` 处理文字,Velato/Vello 处理 Lottie;preview/playback/export 共用预乘 RGBA 纹理合约,Agent `inspect_media` 及 `inspect_timeline` 也使用同一 Lottie 渲染路径。 - **#36 MCP server 网络面**(PR #69,**issue 已关闭**):rmcp Streamable-HTTP `127.0.0.1:19789/mcp` + 回环 Origin/Host 守卫 + OAuth well-known;src-tauri `mcp.rs` 在 setup spawn(会话共享的 AppCore 克隆 + 内置/用户 workflow registry)。HTTP 集成测试完成 `initialize` 握手 + 远程 Origin 403。`claude mcp add --transport http opentake http://127.0.0.1:19789/mcp` 可连。 ## 5. 🟦 可认领/未完成(供同事,注意文件区避免冲突) -- **🔴 #53 [#47-C] 时间线播放引擎**(连续解码 + cpal 音频 + A/V 同步 + MJPEG 回环传输)。子项 #63(cpal)/#64(MJPEG 传输)/#65(Lottie 烘焙)。最大未完成项,需专门会话 + 真机视觉验证。 +- **#53 [#47-C] 时间线播放引擎**代码竖切已完成:有界解码、cpal 音频、A/V 时钟、seek/pause/resume/cancel 和 Lottie 均已接入;最终发布仍需按完成规划重放打包 GUI 验收。 - **#48 片段编辑收尾**:Delete/切割/片段右键菜单/Inspector 三段式/Toolbar 接线。 -- **剩余隐藏能力**:`inspect_media/get_transcript/search_media/inspect_timeline/import_media/add_captions` 已接真实路径;生成/超分/Motion 六个线名在生产后端完成前不进入发现面。`inspect_media` 尚缺 Lottie;`generate_*`/upscale 仍需异步 GenClient + BYOK,Motion 仍需确定性渲染/导入事务。 +- **动态能力面**:媒体/转写/检索/时间线检查均为真实路径,`inspect_media` 支持图片/视频/音频/Lottie;生成/超分仅在可用授权时发布,Motion add/edit 仅在 Chromium/FFmpeg 生产桥就绪时发布。 - **#49 项目内文件夹导入 + 嵌套文件夹浏览(剪映式)**:文件夹图标/双击进入/面包屑/拖出;DTO 加 folderId+folders;import_folder 镜像目录树。用户很想要。 - **#37 全局可复用素材库 + 收藏**(跨项目/分类/音效库/全库可见):**后端已并入 main** —— 存储层 `crates/opentake-media/src/library.rs`(#37-A/#54,PR #104,copy-on-favorite + SHA-256 内容寻址去重 + JSON manifest 原子写)+ Tauri 命令层 `src-tauri/src/library.rs`(#37-B/#55,PR #106,7 命令 list/favorite/unfavorite/categorize/rename/delete/import_to_project)。**前端 #37-C/#56 已并入 main**(PR #115:独立 `LibraryView` 全屏视图 + `libraryStore`/`libraryApi`,分类树/网格/搜索/排序/跨视图聚合/音效库;Home/TitleBar 入口;前端↔后端 7 命令契约已核实)。**#37 epic 收口**(后端 #104/#106 + 前端 #115)。剩:库→时间线拖拽(现用「导入当前项目」按钮)、媒体面板「星标→library_favorite」接线、收藏从 localStorage 迁后端。follow-up:`library.rs:322` remove() 静默吞 remove_file 错误,建议补 `tracing::warn!`;`library_delete` 与 `library_unfavorite` 现为纯别名,建议语义区分。 - **#39 提取音频星标 · #40 设置多分页+主页 1:1 · #34 motion dispatch · #27–30 进阶 B/C/D/E · #22–25 #12 follow-up · #35 bundle id 改名**。 diff --git a/Cargo.lock b/Cargo.lock index 6088042f..6e72204a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -89,6 +98,18 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "arbitrary" version = "1.4.2" @@ -214,6 +235,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link 0.2.1", +] + [[package]] name = "base64" version = "0.13.1" @@ -238,6 +274,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -276,15 +321,30 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec 0.6.3", +] + [[package]] name = "bit-set" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit-vec" version = "0.8.0" @@ -644,6 +704,18 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "color" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7f99105610438d4b3ee7ae8e453c2990e325c806a14d71e8ea937d584c5289" + +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" + [[package]] name = "color_quant" version = "1.1.0" @@ -1039,6 +1111,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + [[package]] name = "der" version = "0.8.0" @@ -1058,6 +1140,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -1198,6 +1291,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + [[package]] name = "document-features" version = "0.2.12" @@ -1213,7 +1312,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521e380c0c8afb8d9a1e83a1822ee03556fc3e3e7dbc1fd30be14e37f9cb3f89" dependencies = [ - "bit-set", + "bit-set 0.8.0", "cssparser", "foldhash 0.2.0", "html5ever", @@ -1222,6 +1321,12 @@ dependencies = [ "tendril", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dpi" version = "0.1.2" @@ -1273,6 +1378,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15401da73a9ed8c80e3b2d4dc05fe10e7b72d7243b9f614e516a44fa99986e88" + [[package]] name = "either" version = "1.16.0" @@ -1332,6 +1443,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -1426,6 +1546,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "font-types" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa6a5e5a77b5f3f7f9e32879f484aa5b3632ddfbe568a16266c904a6f32cdaf" +dependencies = [ + "bytemuck", +] + [[package]] name = "fontdb" version = "0.16.2" @@ -1549,6 +1678,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -1753,6 +1893,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "gio" version = "0.18.4" @@ -1900,6 +2046,18 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "gpu-allocator" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c151a2a5ef800297b4e79efa4f4bec035c5f51d5ae587287c9b952bdf734cacd" +dependencies = [ + "log", + "presser", + "thiserror 1.0.69", + "windows 0.58.0", +] + [[package]] name = "gpu-descriptor" version = "0.3.2" @@ -1972,6 +2130,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "guillotiere" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62d5865c036cb1393e23c50693df631d3f5d7bcca4c04fe4cc0fd592e74a782" +dependencies = [ + "euclid", + "svg_fmt", +] + [[package]] name = "half" version = "2.7.1" @@ -1980,6 +2148,7 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "num-traits", "zerocopy", ] @@ -1989,6 +2158,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2381,6 +2559,24 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -2526,6 +2722,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "keyframe" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60708bf7981518d09095d6f5673ce5cf6a64f1e0d9708b554f670e6d9d2bd9a9" +dependencies = [ + "mint", + "num-traits", +] + [[package]] name = "keyring" version = "3.6.3" @@ -2558,6 +2764,33 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" +[[package]] +name = "kstring" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" +dependencies = [ + "serde", + "static_assertions", +] + +[[package]] +name = "kurbo" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62026ae44756f8a599ba21140f350303d4f08dcdcc71b5ad9c9bb8128c13c62" +dependencies = [ + "arrayvec", + "euclid", + "smallvec 1.15.2", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libappindicator" version = "0.9.0" @@ -2632,12 +2865,75 @@ dependencies = [ "libc", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "liquid" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9338405fdbc0bce9b01695b2a2ef6b20eca5363f385d47bce48ddf8323cc25" +dependencies = [ + "doc-comment", + "liquid-core", + "liquid-derive", + "liquid-lib", + "serde", +] + +[[package]] +name = "liquid-core" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb8fed70857010ed9016ed2ce5a7f34e7cc51d5d7255c9c9dc2e3243e490b42" +dependencies = [ + "anymap2", + "itertools 0.13.0", + "kstring", + "liquid-derive", + "num-traits", + "pest", + "pest_derive", + "regex", + "serde", + "time", +] + +[[package]] +name = "liquid-derive" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b51f1d220e3fa869e24cfd75915efe3164bd09bb11b3165db3f37f57bf673e3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "liquid-lib" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1794b5605e9f8864a8a4f41aa97976b42512cc81093f8c885d29fb94c6c556" +dependencies = [ + "itertools 0.13.0", + "liquid-core", + "once_cell", + "percent-encoding", + "regex", + "time", + "unicode-segmentation", +] + [[package]] name = "litemap" version = "0.8.2" @@ -2705,6 +3001,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markup5ever" version = "0.38.0" @@ -2809,6 +3111,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mint" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53debba6bda7a793e5f99b8dacf19e626084f525f7829104ba9898f367d85ff" + [[package]] name = "mio" version = "1.2.1" @@ -2880,7 +3188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "364f94bc34f61332abebe8cad6f6cd82a5b65cff22c828d05d0968911462ca4f" dependencies = [ "arrayvec", - "bit-set", + "bit-set 0.8.0", "bitflags 2.13.0", "cfg_aliases 0.1.1", "codespan-reporting", @@ -3037,6 +3345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3266,6 +3575,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "oboe" version = "0.6.1" @@ -3362,7 +3680,7 @@ dependencies = [ [[package]] name = "opentake-agent" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "anyhow", "async-trait", @@ -3394,7 +3712,7 @@ dependencies = [ [[package]] name = "opentake-core" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "opentake-domain", "opentake-ops", @@ -3408,7 +3726,7 @@ dependencies = [ [[package]] name = "opentake-domain" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "serde", "serde_json", @@ -3416,7 +3734,7 @@ dependencies = [ [[package]] name = "opentake-gen" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "anyhow", "async-trait", @@ -3434,7 +3752,7 @@ dependencies = [ [[package]] name = "opentake-media" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "anyhow", "byteorder", @@ -3449,7 +3767,9 @@ dependencies = [ "ndarray", "opentake-domain", "ort", + "ort-tract", "reqwest 0.12.28", + "rustfft", "ryu-js", "same-file", "serde", @@ -3468,10 +3788,12 @@ dependencies = [ [[package]] name = "opentake-motion" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ + "base64 0.22.1", "hex", "image", + "libc", "opentake-domain", "opentake-render", "serde", @@ -3479,11 +3801,12 @@ dependencies = [ "sha2", "tempfile", "thiserror 1.0.69", + "tungstenite", ] [[package]] name = "opentake-ops" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "opentake-domain", "serde_json", @@ -3491,7 +3814,7 @@ dependencies = [ [[package]] name = "opentake-project" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "cap-fs-ext", "cap-std", @@ -3510,20 +3833,24 @@ dependencies = [ [[package]] name = "opentake-render" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "bytemuck", "cosmic-text", + "half", "image", "opentake-domain", + "opentake-media", + "opentake-ops", "pollster", + "serde_json", "thiserror 2.0.18", "wgpu", ] [[package]] name = "opentake-tauri" -version = "1.0.0" +version = "1.0.0-beta.1" dependencies = [ "axum", "base64 0.22.1", @@ -3531,30 +3858,37 @@ dependencies = [ "cap-fs-ext", "cap-std", "cpal", + "crossbeam-channel", "futures", "futures-util", "image", "libc", "objc2-app-kit", + "objc2-foundation", "opentake-agent", "opentake-core", "opentake-domain", "opentake-gen", "opentake-media", + "opentake-motion", "opentake-ops", "opentake-project", "opentake-render", "reqwest 0.12.28", "same-file", + "sentry", "serde", "serde_json", "sha2", "tauri", "tauri-build", "tauri-plugin-dialog", + "tauri-plugin-fs", + "tauri-plugin-persisted-scope", "tempfile", "tokio", "uuid", + "velato", "windows-sys 0.61.2", ] @@ -3589,6 +3923,17 @@ dependencies = [ "ureq", ] +[[package]] +name = "ort-tract" +version = "0.1.0+0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41450290a215a579f8a723bb255a872666f98609b37fac8f57c9affcadfd78b" +dependencies = [ + "ort-sys", + "parking_lot", + "tract-onnx", +] + [[package]] name = "pango" version = "0.18.3" @@ -3659,14 +4004,80 @@ dependencies = [ ] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "peniko" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "5c1f594c54ccdc9bd177a726885f066bf28d20e17169e31a8a1456217b1316b4" +dependencies = [ + "color 0.2.4", + "kurbo", + "peniko 0.4.1", + "smallvec 1.15.2", +] [[package]] -name = "phf" -version = "0.13.1" +name = "peniko" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b44f9ddd2f480176b34278eb653ec1c8062f3b143a4e16eeff5ffac3334e288" +dependencies = [ + "color 0.3.3", + "kurbo", + "linebender_resource_handle", + "smallvec 1.15.2", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pest_meta" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" +dependencies = [ + "pest", +] + +[[package]] +name = "phf" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ @@ -3819,6 +4230,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "prettyplease" version = "0.2.37" @@ -3829,6 +4246,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3897,6 +4323,29 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pxfm" version = "0.1.29" @@ -3994,13 +4443,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -4015,6 +4475,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -4025,6 +4495,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -4040,6 +4519,22 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.7", +] + +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + [[package]] name = "rangemap" version = "1.7.1" @@ -4096,7 +4591,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69aacb76b5c29acfb7f90155d39759a29496aebb49395830e928a9703d2eec2f" dependencies = [ "bytemuck", - "font-types", + "font-types 0.7.3", +] + +[[package]] +name = "read-fonts" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f9e8a4f503e5c8750e4cd3b32a4e090035c46374b305a15c70bad833dca05f" +dependencies = [ + "bytemuck", + "font-types 0.8.4", ] [[package]] @@ -4333,6 +4838,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4354,6 +4865,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4456,6 +4981,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scan_fmt" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248" +dependencies = [ + "regex", +] + [[package]] name = "schannel" version = "0.1.29" @@ -4607,6 +5141,85 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sentry" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e4790d8c2f43a6645ee2cb12aac2db79221fddd035b94c8166b88ea094408" +dependencies = [ + "cfg_aliases 0.2.1", + "httpdate", + "sentry-backtrace", + "sentry-core", + "sentry-panic", + "sentry-tracing", + "ureq", +] + +[[package]] +name = "sentry-backtrace" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "326e106874a7ea90636f1ca42e2f7b912929d29307982cab37f81208c16cf043" +dependencies = [ + "backtrace", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-core" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9efafefbb78d7e02cb06c10aec08b77e7500428389eabbf6d5a668325265e8" +dependencies = [ + "rand 0.9.4", + "sentry-types", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "sentry-panic" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fc536a3e1fc68d626ae8dd18b628cafd474d9d36fea74e4bbb4d038877f003" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-tracing" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96be2253fe14b3fa10c1ba047af2d3e58ce85c8cb297bbd19d6fa81b604e7877" +dependencies = [ + "bitflags 2.13.0", + "sentry-backtrace", + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f967748632cd4c5405dbed45426b27b407c0e53ac59b7df1c163e7f5aa57a77c" +dependencies = [ + "debugid", + "hex", + "rand 0.9.4", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "url", + "uuid", +] + [[package]] name = "serde" version = "1.0.228" @@ -4832,6 +5445,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.9" @@ -4851,7 +5474,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e1c44ad1f6c5bdd4eefed8326711b7dbda9ea45dfd36068c427d332aa382cbe" dependencies = [ "bytemuck", - "read-fonts", + "read-fonts 0.22.7", +] + +[[package]] +name = "skrifa" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc1aa86c26dbb1b63875a7180aa0819709b33348eb5b1491e4321fae388179d" +dependencies = [ + "bytemuck", + "read-fonts 0.25.3", ] [[package]] @@ -4996,6 +5629,23 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "string-interner" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07f9fdfdd31a0ff38b59deb401be81b73913d76c9cc5b1aed4e1330a223420b9" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "string_cache" version = "0.9.0" @@ -5032,13 +5682,19 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "svg_fmt" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0193cc4331cfd2f3d2011ef287590868599a2f33c3e69bc22c1a3d3acf9e02fb" + [[package]] name = "swash" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbd59f3f359ddd2c95af4758c18270eddd9c730dde98598023cdabff472c2ca2" dependencies = [ - "skrifa", + "skrifa 0.22.3", "yazi", "zeno", ] @@ -5061,6 +5717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", + "quote", "unicode-ident", ] @@ -5357,6 +6014,22 @@ dependencies = [ "url", ] +[[package]] +name = "tauri-plugin-persisted-scope" +version = "2.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b560a5962bf975d38fb4ec98a0e64e52929992ec708acb812add9c1ab8d186d" +dependencies = [ + "aho-corasick", + "bincode", + "log", + "serde", + "serde_json", + "tauri", + "tauri-plugin-fs", + "thiserror 2.0.18", +] + [[package]] name = "tauri-runtime" version = "2.11.3" @@ -5641,6 +6314,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -5900,6 +6574,163 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "tracing-core", +] + +[[package]] +name = "tract-core" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b5347639690871b124593a8c8903f1f369531498b8abaebd18eb5c58163971" +dependencies = [ + "anyhow", + "anymap3", + "bit-set 0.5.3", + "derive-new", + "downcast-rs", + "dyn-clone", + "lazy_static", + "log", + "maplit", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "paste", + "rustfft", + "smallvec 1.15.2", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a3f476a1804e05708e9bc5e2d29dcab82bad531e357d3d14d7da80fbba0b6d" +dependencies = [ + "anyhow", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "itertools 0.12.1", + "lazy_static", + "maplit", + "ndarray", + "nom", + "num-integer", + "num-traits", + "parking_lot", + "scan_fmt", + "smallvec 1.15.2", + "string-interner", +] + +[[package]] +name = "tract-hir" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dca047ba1151fe3446fb0194d4b6ddb9ae8f361337c47a267870c53605fbafb" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8e0703eb53ef1bbf77050ff261675818dd5f0d6c27044c6e48ede9b845f9e0" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "lazy_static", + "liquid", + "liquid-core", + "liquid-derive", + "log", + "num-traits", + "paste", + "rayon", + "scan_fmt", + "smallvec 1.15.2", + "time", + "tract-data", + "unicode-normalization", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cb88a4367ec2c695610223cf886f01fc1deb5c9a82c7a74b1a5d32dc0b1466" +dependencies = [ + "byteorder", + "flate2", + "log", + "nom", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5830aa672b2aa4dc98a97a36e5988eaf77b3ecee65e2601619588d2ca557008" +dependencies = [ + "bytes", + "derive-new", + "log", + "memmap2", + "num-integer", + "prost", + "smallvec 1.15.2", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121d3d224c806ba3d941f4bb50943ad33b59d1da5ae704d0e4e76d2808221f96" +dependencies = [ + "getrandom 0.2.17", + "log", + "rand 0.8.7", + "rand_distr", + "rustfft", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", ] [[package]] @@ -5970,6 +6801,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -6192,12 +7029,77 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "velato" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e40789f32ccca73bf2cef43426ecb616bfddcd623029d4d0b37bf00ab276580" +dependencies = [ + "keyframe", + "once_cell", + "serde", + "serde_json", + "serde_repr", + "thiserror 2.0.18", + "vello", +] + +[[package]] +name = "vello" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d5b0bafa35e0c2e4132104576d6bcec4bf7cd0044f1760e92ecae0d4d9bc0e7" +dependencies = [ + "bytemuck", + "futures-intrusive", + "log", + "peniko 0.3.2", + "png 0.17.16", + "skrifa 0.26.6", + "static_assertions", + "thiserror 2.0.18", + "vello_encoding", + "vello_shaders", + "wgpu", +] + +[[package]] +name = "vello_encoding" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbbdec68dea2b39ece9f82ab15ec4cf2c4f8600ce6926df0638290702d95b3f7" +dependencies = [ + "bytemuck", + "guillotiere", + "peniko 0.3.2", + "skrifa 0.26.6", + "smallvec 1.15.2", +] + +[[package]] +name = "vello_shaders" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0179d74cf9131dfd7882323751d2544f3aefdfda9d16c39bbe2729799410d2" +dependencies = [ + "bytemuck", + "naga", + "thiserror 2.0.18", + "vello_encoding", +] + [[package]] name = "version-compare" version = "0.2.1" @@ -6492,6 +7394,7 @@ dependencies = [ "document-features", "js-sys", "log", + "naga", "parking_lot", "profiling", "raw-window-handle", @@ -6512,7 +7415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d63c3c478de8e7e01786479919c8769f62a22eec16788d8c2ac77ce2c132778a" dependencies = [ "arrayvec", - "bit-vec", + "bit-vec 0.8.0", "bitflags 2.13.0", "cfg_aliases 0.1.1", "document-features", @@ -6539,6 +7442,7 @@ dependencies = [ "android_system_properties", "arrayvec", "ash", + "bit-set 0.8.0", "bitflags 2.13.0", "block", "bytemuck", @@ -6547,6 +7451,7 @@ dependencies = [ "glow", "glutin_wgl_sys", "gpu-alloc", + "gpu-allocator", "gpu-descriptor", "js-sys", "khronos-egl", @@ -6560,6 +7465,7 @@ dependencies = [ "once_cell", "parking_lot", "profiling", + "range-alloc", "raw-window-handle", "renderdoc-sys", "rustc-hash 1.1.0", @@ -6569,6 +7475,7 @@ dependencies = [ "web-sys", "wgpu-types", "windows 0.58.0", + "windows-core 0.58.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 40890120..718f56f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ ] [workspace.package] -version = "1.0.0" +version = "1.0.0-beta.1" edition = "2021" license = "GPL-3.0-or-later" repository = "https://github.com/appergb/OpenTake" @@ -24,6 +24,8 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_ignored = "0.1" uuid = { version = "1", features = ["v4"] } +velato = { version = "=0.5.0", features = ["wgpu"] } +sentry = { version = "=0.49.0", default-features = false, features = ["backtrace", "panic", "ureq"] } opentake-domain = { path = "crates/opentake-domain" } opentake-ops = { path = "crates/opentake-ops" } opentake-project = { path = "crates/opentake-project" } diff --git a/NOTICE b/NOTICE index e25822ea..5f110371 100644 --- a/NOTICE +++ b/NOTICE @@ -28,3 +28,16 @@ Nature of the fork / summary of changes replacing AVFoundation's declarative composition. - Self-hosted / BYOK generative-AI backend. The upstream generative-AI processing is closed-source and is NOT part of this fork; OpenTake provides its own. + +------------------------------------------------------------------------------- +Optional on-device model +------------------------------------------------------------------------------- + +The AI portrait-matting feature can download the official Robust Video Matting +(RVM) MobileNetV3 FP32 ONNX model, version 1.0.0, from +https://github.com/PeterL1n/RobustVideoMatting. RVM was developed at ByteDance +Inc. by Shanchuan Lin, Linjie Yang, Imran Saleemi, and Soumyadip Sengupta and is +distributed under the GNU General Public License version 3. The model is not +embedded in the OpenTake application; it is installed on demand after the user +selects the model-install action and is verified against a pinned byte size and +SHA-256 digest before use. diff --git a/README.ja.md b/README.ja.md index 1e795af2..0addfe85 100644 --- a/README.ja.md +++ b/README.ja.md @@ -233,7 +233,9 @@ cd web && pnpm install && pnpm build cd .. && cargo tauri dev ``` -> ⚠️ **現在の状態**: 初期設計段階。アーキテクチャ、ロードマップ、モジュール移植マップは完了。コード実装中。 +> **現在の状態**: `1.0.0-beta.1` 候補版。ローカル編集、プレビュー、保存、 +> 書き出し、Agent、Motion Canvas、レビュー可能な AI ワークフローを実装済みです。 +> 検証範囲と制限は [Beta リリースノート](docs/releases/1.0.0-beta.1.md) を参照してください。 --- @@ -242,6 +244,7 @@ cd .. && cargo tauri dev | バージョン | 日付 | マイルストーン | |:--|:--|:--| | `0.1.0-dev` | 2026-06 | Phase 0+1: Cargo workspace + Domain models + Edit ops | +| `1.0.0-beta.1` | 2026-08-01 | 初回インストール可能 Beta:ローカル編集、Agent、Motion、レビュー可能な AI ワークフロー | | *(planned)* `1.0.0` | TBD | Phase 10: フルリリース | 📖 [完全なロードマップ](docs/architecture/ROADMAP.md) diff --git a/README.md b/README.md index 0cedf295..94b1d777 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ OpenTake is not a replacement for CapCut / DaVinci Resolve / Final Cut Pro — i |:--|:--|:--| | Agent doesn't know how to edit | Agent reads skill docs on its own | Software pushes Context Signal — "this track is A-roll, cut with talking-head rhythm" | | Cross-platform needs 3 codebases | macOS: Swift/AVFoundation, Windows: C++/DirectShow | Single Rust codebase, FFmpeg + wgpu, identical experience on all 3 platforms | -| I want to use my own AI keys | Locked into vendor cloud services | BYOK — direct to fal.ai / Replicate / OpenAI, zero backend, zero ops cost | +| I want to use AI directly | Locked into vendor cloud services | Official Codex / ChatGPT sign-in for Agent, plus BYOK for fal.ai / Replicate / OpenAI | | Agent can chat but can't act | CLI agent reads text output | MCP Server with 31 tools — Agent directly runs add_clips / split_clip / set_keyframes | | Rewriting prompts for every video type | "You are editing a product review..." every time | Workflow Plugin System: review/tutorial/gaming/wedding, each pre-packaged with methodology | | Steep learning curve for new tools | Complex UI, long onboarding | Agent operates for you — just say "edit this interview into a 3-minute highlight" | @@ -110,7 +110,9 @@ Full MCP server at `127.0.0.1:19789`. Agents control the timeline directly: | Library | 7 | `create_folder`, `move_to_folder`, `rename_media` | | Resources | 2 | `models/video`, `models/image` | -Built-in Agent chat panel shares tool definitions and system prompt with MCP. +Built-in Agent chat panel shares tool definitions and system prompt with MCP. It can use direct +OpenAI/Anthropic BYOK or the user-installed official Codex CLI's ChatGPT sign-in; OpenTake never +reads or stores the Codex credential. ### 🎬 Cross-Platform Media Engine @@ -295,7 +297,10 @@ cd .. cargo tauri dev ``` -> ⚠️ **Current Status**: Early design phase. Architecture, roadmap, and module port maps are complete; code implementation in progress. +> **Current Status**: `1.0.0-beta.1` candidate. The local editing, preview, +> persistence, export, Agent, Motion Canvas, and reviewed AI workflow verticals +> are implemented. See the [Beta release notes](docs/releases/1.0.0-beta.1.md) +> for validation scope and platform/provider limits. The sibling directory `palmier-pro-upstream/` contains upstream Swift sources for reference during porting. @@ -306,6 +311,7 @@ The sibling directory `palmier-pro-upstream/` contains upstream Swift sources fo | Version | Date | Milestone | |:--|:--|:--| | `0.1.0-dev` | 2026-06 | Phase 0+1: Cargo workspace + Domain models + Edit ops + Tauri scaffold | +| `1.0.0-beta.1` | 2026-08-01 | First installable Beta: end-to-end local editor, Agent, Motion and reviewed AI workflows | | *(planned)* `0.2.0` | TBD | Phase 2: Persistence + Media import + Thumbnails + Waveform | | *(planned)* `0.3.0` | TBD | Phase 3: Timeline UI + Preview + MCP Server | | *(planned)* `0.4.0` | TBD | Phase 4: GPU Compositor (wgpu) + Text rasterization | diff --git a/README.zh-CN.md b/README.zh-CN.md index c15ed08b..98ff1343 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -258,7 +258,9 @@ cd web && pnpm install && pnpm build cd .. && cargo tauri dev ``` -> ⚠️ **当前状态**: 早期设计阶段。架构设计、路线图、模块移植地图已完成,代码正在落地中。 +> **当前状态**:`1.0.0-beta.1` 候选版。本地剪辑、预览、持久化、导出、Agent、 +> Motion Canvas 与可审阅 AI 工作流竖切均已实现。验证范围及平台/provider 限制见 +> [Beta 发布说明](docs/releases/1.0.0-beta.1.md)。 --- @@ -267,6 +269,7 @@ cd .. && cargo tauri dev | 版本 | 日期 | 里程碑 | |:--|:--|:--| | `0.1.0-dev` | 2026-06 | Phase 0+1: Cargo workspace + Domain models + Edit ops + Tauri scaffold | +| `1.0.0-beta.1` | 2026-08-01 | 首个可安装 Beta:本地编辑闭环、Agent、Motion 与可审阅 AI 工作流 | | *(planned)* `0.2.0` | TBD | Phase 2: Persistence + Media import + Thumbnails + Waveform | | *(planned)* `0.3.0` | TBD | Phase 3: Timeline UI + Preview + MCP Server | | *(planned)* `0.4.0` | TBD | Phase 4: GPU Compositor (wgpu) + Text rasterization | diff --git a/crates/opentake-agent/src/chat/loop.rs b/crates/opentake-agent/src/chat/loop.rs index 8bb3665a..59f824e2 100644 --- a/crates/opentake-agent/src/chat/loop.rs +++ b/crates/opentake-agent/src/chat/loop.rs @@ -29,7 +29,6 @@ use crate::plugin::registry::PluginRegistry; use crate::prompt::assemble::assemble_system_prompt; use crate::signal::engine::build_signal; use crate::tools::descriptions::{description, input_schema}; -use crate::tools::names::ToolName; use crate::tools::panic_boundary::with_redacted_dispatch_panic; use crate::tools::result::ToolResult; @@ -251,7 +250,7 @@ impl ChatLoop { } /// The tool catalog in the OpenAI function-calling shape. Built fresh per - /// turn (cheap; currently 38 live tools) so the model always sees the + /// turn (cheap; currently 39 base live tools) so the model always sees the /// current fail-closed catalog. /// /// When the dispatcher lacks a media bridge, hide the bridge-dependent @@ -260,13 +259,6 @@ impl ChatLoop { self.dispatcher .advertised_tools() .into_iter() - .filter(|tool| { - self.dispatcher.has_media_bridge() - || !matches!( - tool, - ToolName::InspectMedia | ToolName::InspectTimeline | ToolName::ImportMedia - ) - }) .map(|tool| ToolSchema { name: tool.as_str().to_string(), description: description(tool).to_string(), @@ -285,7 +277,10 @@ impl ChatLoop { if let Ok(json) = serde_json::to_value(&signal) { s.push_str("\n\n# Current timeline context signal\n"); s.push_str(&serde_json::to_string_pretty(&json).unwrap_or_default()); - s.push_str("\n\nUse this signal to pick the right tool without re-reading the timeline first. For example, if the user asks to tighten silences on a talking-head timeline, call `tighten_silences` then `ripple_delete_ranges` with the returned ranges."); + s.push_str("\n\nUse this signal to pick the right tool without re-reading the timeline first. For example, if the user asks to tighten silences on a talking-head timeline, call `tighten_silences` then `ripple_delete_ranges` with the accepted returned ranges."); + if self.dispatcher.has_media_bridge() { + s.push_str(" If the user asks to remove filler words, call `remove_filler_words`, let them review the word-aligned cuts, then apply only the accepted ranges with `ripple_delete_ranges`."); + } } s } @@ -607,6 +602,9 @@ mod tests { let loop_ = build_loop(talking_head_timeline(), Arc::new(MemoryKeyStore::new())); let tools = loop_.tool_catalog(); assert!(tools.iter().any(|t| t.name == "tighten_silences")); + assert!(!tools.iter().any(|t| t.name == "remove_filler_words")); + assert!(!tools.iter().any(|t| t.name == "get_transcript")); + assert!(!tools.iter().any(|t| t.name == "search_media")); assert!(!tools.iter().any(|t| t.name == "inspect_media")); assert!(!tools.iter().any(|t| t.name == "inspect_timeline")); assert!(!tools.iter().any(|t| t.name == "import_media")); diff --git a/crates/opentake-agent/src/mcp/advanced.rs b/crates/opentake-agent/src/mcp/advanced.rs new file mode 100644 index 00000000..a7c24e99 --- /dev/null +++ b/crates/opentake-agent/src/mcp/advanced.rs @@ -0,0 +1,90 @@ +//! Host boundary for capability-gated advanced editing workflows. +//! +//! The agent owns stable, strict tool contracts. The desktop host owns model +//! availability, provider authorization, rendering, imports, and atomic edits. +//! A tool is discoverable only when the injected host bridge explicitly lists +//! it as supported; this prevents schema-only placeholders from reaching users. + +use serde_json::Value; + +use crate::tools::args::{ + CloneVoiceArgs, GenerateAvatarArgs, GenerateMatteArgs, MatchColorArgs, RemoveObjectArgs, + ScriptToVideoArgs, SeparateStemsArgs, TrackMotionArgs, TranslateCaptionsArgs, +}; +use crate::tools::names::ToolName; + +#[derive(Debug, Clone, PartialEq)] +pub enum AdvancedWorkflowRequest { + TrackMotion(TrackMotionArgs), + GenerateMatte(GenerateMatteArgs), + RemoveObject(RemoveObjectArgs), + MatchColor(MatchColorArgs), + SeparateStems(SeparateStemsArgs), + TranslateCaptions(TranslateCaptionsArgs), + ScriptToVideo(ScriptToVideoArgs), + GenerateAvatar(GenerateAvatarArgs), + CloneVoice(CloneVoiceArgs), +} + +impl AdvancedWorkflowRequest { + pub fn tool(&self) -> ToolName { + match self { + Self::TrackMotion(_) => ToolName::TrackMotion, + Self::GenerateMatte(_) => ToolName::GenerateMatte, + Self::RemoveObject(_) => ToolName::RemoveObject, + Self::MatchColor(_) => ToolName::MatchColor, + Self::SeparateStems(_) => ToolName::SeparateStems, + Self::TranslateCaptions(_) => ToolName::TranslateCaptions, + Self::ScriptToVideo(_) => ToolName::ScriptToVideo, + Self::GenerateAvatar(_) => ToolName::GenerateAvatar, + Self::CloneVoice(_) => ToolName::CloneVoice, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AdvancedWorkflowCommit { + /// Structured result returned to the agent after a successful operation. + pub result: Value, + /// Present only when the host committed an undoable project mutation. + pub action_name: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdvancedWorkflowErrorKind { + InvalidArguments, + ResourceNotFound, + CapabilityUnavailable, + ConsentRequired, + CostAuthorizationRequired, + AnalysisLowConfidence, + Cancelled, + ExecutionFailed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdvancedWorkflowError { + pub kind: AdvancedWorkflowErrorKind, + pub message: String, +} + +impl AdvancedWorkflowError { + pub fn new(kind: AdvancedWorkflowErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } +} + +pub trait AdvancedWorkflowBridge: Send + Sync { + /// Exact advanced tools backed by a production implementation right now. + /// The dispatcher ignores names outside [`ToolName::ADVANCED_AI`]. + fn supported_tools(&self) -> Vec; + + fn execute( + &self, + request: AdvancedWorkflowRequest, + cancel: &opentake_media::MediaCancelToken, + ) -> Result; +} diff --git a/crates/opentake-agent/src/mcp/convert.rs b/crates/opentake-agent/src/mcp/convert.rs index a8a63e91..472f0966 100644 --- a/crates/opentake-agent/src/mcp/convert.rs +++ b/crates/opentake-agent/src/mcp/convert.rs @@ -144,6 +144,10 @@ fn safe_public_detail(kind: PublicErrorKind, private_detail: &str) -> Option Some(format!( + "{} could not identify the selected subject reliably.", + tool.as_str() + )), } } @@ -342,6 +346,21 @@ mod tests { assert!(!value.to_string().contains("/Users/alice")); } + #[test] + fn low_confidence_error_has_a_fixed_safe_retry_contract() { + let result = ToolResult::public_error( + PublicErrorKind::AnalysisLowConfidence(ToolName::TrackMotion), + "confidence=0.02 path=/private/source.mp4", + ); + let value = safe_tool_result_for_llm(&result); + assert_eq!(value["code"], "MCP_ANALYSIS_LOW_CONFIDENCE"); + assert_eq!( + value["details"], + "track_motion could not identify the selected subject reliably." + ); + assert!(!value.to_string().contains("/private/source.mp4")); + } + #[test] fn explicitly_public_marker_cannot_bypass_detail_guard() { let private = "/Users/alice/private.mp4"; diff --git a/crates/opentake-agent/src/mcp/core_handle.rs b/crates/opentake-agent/src/mcp/core_handle.rs index b12766fa..eb81e12b 100644 --- a/crates/opentake-agent/src/mcp/core_handle.rs +++ b/crates/opentake-agent/src/mcp/core_handle.rs @@ -152,6 +152,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-agent/src/mcp/dispatch.rs b/crates/opentake-agent/src/mcp/dispatch.rs index 8b094886..e4ebf563 100644 --- a/crates/opentake-agent/src/mcp/dispatch.rs +++ b/crates/opentake-agent/src/mcp/dispatch.rs @@ -17,7 +17,7 @@ //! names for compatibility but stay out of discovery until their backends are //! production-ready. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, RwLock}; use opentake_domain::{AnimPair, Crop, Interpolation, Keyframe, KeyframeTrack}; @@ -35,6 +35,10 @@ use opentake_ops::{ }; use serde_json::Value; +use crate::mcp::advanced::{ + AdvancedWorkflowBridge, AdvancedWorkflowError, AdvancedWorkflowErrorKind, + AdvancedWorkflowRequest, +}; use crate::mcp::core_handle::CoreHandle; use crate::mcp::gen_catalog; use crate::mcp::generation::{GenerationBridge, GenerationRequest}; @@ -43,6 +47,10 @@ use crate::mcp::media_bridge::{ InspectMediaResult, InspectResult, MediaBridge, SearchCandidate, TranscriptSource, IMPORT_BYTES_BASE64_MAX, }; +use crate::mcp::motion::{ + AddMotionRequest, EditMotionRequest, MotionBridge, MotionBridgeError, MotionBridgeErrorKind, + MotionSourceRequest, +}; use crate::plugin::registry::PluginRegistry; use crate::signal::engine; use crate::signal::rules::OpContext; @@ -64,16 +72,6 @@ const INSPECT_MEDIA_MAX_FRAMES: usize = 12; const INSPECT_MEDIA_MAX_SEGMENTS: usize = 400; const INSPECT_MEDIA_MAX_WORDS: usize = 10_000; -fn is_generation_tool(tool: ToolName) -> bool { - matches!( - tool, - ToolName::GenerateVideo - | ToolName::GenerateImage - | ToolName::GenerateAudio - | ToolName::UpscaleMedia - ) -} - /// The in-process tool dispatcher. Holds the [`CoreHandle`] boundary, the plugin /// registry (read-locked for the active plugin), and a per-dispatcher agent-undo /// stack so `undo` only reverts edits this session made. @@ -88,6 +86,12 @@ pub struct Dispatcher { /// Paid generation/upscale side-door. The desktop host injects this only /// when it can persist jobs and run configured providers. generation_bridge: Option>, + /// Deterministic render + atomic import/place host capability. Motion tools + /// are discoverable only while this bridge reports production readiness. + motion_bridge: Option>, + /// Capability-gated advanced workflows. Each tool is discovered only when + /// this bridge explicitly reports a production implementation for it. + advanced_bridge: Option>, /// Action names of agent edits applied through this dispatcher, newest last. /// Guards `undo`: we only revert when this session has pushed an edit. agent_undo: Mutex>, @@ -118,12 +122,45 @@ impl Dispatcher { registry: Arc>, bridge: Option>, generation_bridge: Option>, + ) -> Self { + Self::with_capability_bridges(handle, registry, bridge, generation_bridge, None) + } + + /// New dispatcher with every optional host capability injected + /// independently. The narrower constructors remain source-compatible for + /// non-desktop hosts and tests. + pub fn with_capability_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + ) -> Self { + Self::with_all_capability_bridges( + handle, + registry, + bridge, + generation_bridge, + motion_bridge, + None, + ) + } + + pub fn with_all_capability_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + advanced_bridge: Option>, ) -> Self { Dispatcher { handle, registry, bridge, generation_bridge, + motion_bridge, + advanced_bridge, agent_undo: Mutex::new(Vec::new()), } } @@ -140,11 +177,30 @@ impl Dispatcher { .is_some_and(|bridge| bridge.can_generate()) } + pub fn can_render_motion(&self) -> bool { + self.motion_bridge + .as_ref() + .is_some_and(|bridge| bridge.can_render_motion()) + } + pub fn advertised_tools(&self) -> Vec { let mut tools = ToolName::ALL.to_vec(); + if !self.has_media_bridge() { + tools.retain(|tool| !tool.requires_media_bridge()); + } if self.can_generate() { tools.extend(ToolName::GENERATION); } + if self.can_render_motion() { + tools.extend(ToolName::MOTION); + } + if let Some(bridge) = &self.advanced_bridge { + for tool in bridge.supported_tools() { + if ToolName::ADVANCED_AI.contains(&tool) && !tools.contains(&tool) { + tools.push(tool); + } + } + } tools } @@ -184,9 +240,7 @@ impl Dispatcher { error.message, ); } - if !(ToolName::ALL.contains(&tool) - || is_generation_tool(tool) && self.generation_bridge.is_some()) - { + if !self.advertised_tools().contains(&tool) { return ToolResult::public_error( PublicErrorKind::UnknownTool, format!("Tool is not advertised: {}", tool.as_str()), @@ -300,9 +354,10 @@ impl Dispatcher { // --- Analysis-driven edit surface --- ToolName::DetectBeats => self.detect_beats(args, before), - ToolName::AutoCutToBeats => self.auto_cut_to_beats(args, before), + ToolName::AutoCutToBeats => self.auto_cut_to_beats(args, before, op), ToolName::SmartReframe => self.smart_reframe(args), ToolName::TightenSilences => self.tighten_silences(args, before), + ToolName::RemoveFillerWords => self.remove_filler_words(args, before, manifest), // --- Render + import + transcript + search (wired to the injected MediaBridge) --- ToolName::InspectTimeline => self.inspect_timeline(args, before), @@ -319,12 +374,143 @@ impl Dispatcher { | ToolName::GenerateImage | ToolName::GenerateAudio | ToolName::UpscaleMedia => self.submit_generation(tool, args, cancel), - ToolName::AddMotionGraphic | ToolName::EditMotionGraphic => Ok(ToolResult::error( - format!("{}: capability is not advertised", tool.as_str()), - )), + ToolName::AddMotionGraphic => self.add_motion_graphic(args, cancel), + ToolName::EditMotionGraphic => self.edit_motion_graphic(args, cancel), + ToolName::TrackMotion + | ToolName::GenerateMatte + | ToolName::RemoveObject + | ToolName::MatchColor + | ToolName::SeparateStems + | ToolName::TranslateCaptions + | ToolName::ScriptToVideo + | ToolName::GenerateAvatar + | ToolName::CloneVoice => self.run_advanced_workflow(tool, args, cancel), } } + fn run_advanced_workflow( + &self, + tool: ToolName, + args: &Value, + cancel: &opentake_media::MediaCancelToken, + ) -> Result { + let request = match tool { + ToolName::TrackMotion => { + AdvancedWorkflowRequest::TrackMotion(decode_tool_args(args, "")?) + } + ToolName::GenerateMatte => { + AdvancedWorkflowRequest::GenerateMatte(decode_tool_args(args, "")?) + } + ToolName::RemoveObject => { + AdvancedWorkflowRequest::RemoveObject(decode_tool_args(args, "")?) + } + ToolName::MatchColor => { + AdvancedWorkflowRequest::MatchColor(decode_tool_args(args, "")?) + } + ToolName::SeparateStems => { + AdvancedWorkflowRequest::SeparateStems(decode_tool_args(args, "")?) + } + ToolName::TranslateCaptions => { + AdvancedWorkflowRequest::TranslateCaptions(decode_tool_args(args, "")?) + } + ToolName::ScriptToVideo => { + AdvancedWorkflowRequest::ScriptToVideo(decode_tool_args(args, "")?) + } + ToolName::GenerateAvatar => { + AdvancedWorkflowRequest::GenerateAvatar(decode_tool_args(args, "")?) + } + ToolName::CloneVoice => { + AdvancedWorkflowRequest::CloneVoice(decode_tool_args(args, "")?) + } + _ => return Err(ToolError::new("not an advanced workflow tool")), + }; + let bridge = self + .advanced_bridge + .as_ref() + .ok_or_else(|| ToolError::new("advanced workflow host capability is not available"))?; + match bridge.execute(request, cancel) { + Ok(commit) => { + if let Some(action_name) = commit.action_name { + self.agent_undo + .lock() + .expect("agent-undo mutex") + .push(action_name); + } + Ok(ToolResult::ok(commit.result.to_string())) + } + Err(error) => Ok(advanced_workflow_error(tool, error)), + } + } + + fn add_motion_graphic( + &self, + args: &Value, + cancel: &opentake_media::MediaCancelToken, + ) -> Result { + let decoded: AddMotionGraphicArgs = decode_tool_args(args, "")?; + let source: MotionSourceArg = decode_tool_args(&decoded.source, "source")?; + let source = match (source.code, source.template_id) { + (Some(code), None) => MotionSourceRequest::Code(code), + (None, Some(template_id)) => MotionSourceRequest::Template { + template_id, + params: source.params.unwrap_or_default(), + }, + _ => return Err(ToolError::new("source: exactly one source is required")), + }; + let bridge = self.motion_bridge.as_ref().ok_or_else(|| { + ToolError::new("add_motion_graphic: motion renderer is not available") + })?; + let commit = match bridge.add( + AddMotionRequest { + source, + start_frame: decoded.start_frame, + duration_frames: decoded.duration_frames, + transparent: decoded.transparent.unwrap_or(false), + track_index: decoded.track_index, + }, + cancel, + ) { + Ok(commit) => commit, + Err(error) => return Ok(motion_bridge_error(ToolName::AddMotionGraphic, error)), + }; + self.agent_undo + .lock() + .expect("agent-undo mutex") + .push(commit.action_name.clone()); + serde_json::to_string(&commit) + .map(ToolResult::ok) + .map_err(|error| ToolError::new(format!("motion result encoding failed: {error}"))) + } + + fn edit_motion_graphic( + &self, + args: &Value, + cancel: &opentake_media::MediaCancelToken, + ) -> Result { + let decoded: EditMotionGraphicArgs = decode_tool_args(args, "")?; + let bridge = self.motion_bridge.as_ref().ok_or_else(|| { + ToolError::new("edit_motion_graphic: motion renderer is not available") + })?; + let commit = match bridge.edit( + EditMotionRequest { + clip_id: decoded.clip_id, + code: decoded.code, + params: decoded.params, + }, + cancel, + ) { + Ok(commit) => commit, + Err(error) => return Ok(motion_bridge_error(ToolName::EditMotionGraphic, error)), + }; + self.agent_undo + .lock() + .expect("agent-undo mutex") + .push(commit.action_name.clone()); + serde_json::to_string(&commit) + .map(ToolResult::ok) + .map_err(|error| ToolError::new(format!("motion result encoding failed: {error}"))) + } + // MARK: - Generative read bodies fn submit_generation( @@ -1321,8 +1507,19 @@ impl Dispatcher { Ok(ToolResult::ok(round_floats_3dp(payload).to_string())) } - fn auto_cut_to_beats(&self, args: &Value, before: &Timeline) -> Result { + fn auto_cut_to_beats( + &self, + args: &Value, + before: &Timeline, + op: &mut OpContext, + ) -> Result { let a: AutoCutToBeatsArgs = decode_tool_args(args, "")?; + let write = a.write.unwrap_or(false); + if write && a.align_cuts == Some(false) { + return Err(ToolError::new( + "auto_cut_to_beats: write=true conflicts with alignCuts=false", + )); + } let beats = self.detect_beat_hints( before, BeatAnalysisRequest { @@ -1354,10 +1551,9 @@ impl Dispatcher { cut_frames.sort_unstable(); cut_frames.dedup(); - let placements = a - .clip_ids - .unwrap_or_default() - .into_iter() + let requested_clip_ids = a.clip_ids.unwrap_or_default(); + let placements = requested_clip_ids + .iter() .zip(cut_frames.iter().copied()) .map(|(clip_id, to_frame)| { serde_json::json!({ @@ -1367,16 +1563,35 @@ impl Dispatcher { }) .collect::>(); + let (applied, summary, placements) = if write { + let (moves, applied_placements) = + plan_beat_alignment_moves(before, &requested_clip_ids, &cut_frames)?; + op.clip_ids = moves + .iter() + .map(|movement| movement.clip_id.clone()) + .collect(); + op.track_index = moves.first().map(|movement| movement.to_track); + let result = self.apply(EditCommand::MoveClips { moves })?; + (result.changed, Some(result.summary), applied_placements) + } else { + (false, None, placements) + }; + let payload = serde_json::json!({ - "applied": false, - "alignCuts": a.align_cuts.unwrap_or(false), + "applied": applied, + "alignCuts": a.align_cuts.unwrap_or(write), "beats": beats.iter().map(|beat| serde_json::json!({ "frame": beat.frame, "strength": beat.strength, })).collect::>(), "cutFrames": cut_frames, "placements": placements, - "note": "Preview only. Apply returned frames through split_clip/move_clips/ripple_delete_ranges as needed.", + "summary": summary, + "note": if write { + "Applied selected clip placements and linked A/V partners through one atomic move_clips command." + } else { + "Preview only. Set write=true to apply placements atomically, or use returned frames with existing edit tools." + }, }); Ok(ToolResult::ok(round_floats_3dp(payload).to_string())) } @@ -1481,6 +1696,205 @@ impl Dispatcher { Ok(ToolResult::ok(round_floats_3dp(payload).to_string())) } + fn remove_filler_words( + &self, + args: &Value, + before: &Timeline, + manifest: &MediaManifest, + ) -> Result { + let a: RemoveFillerWordsArgs = decode_tool_args(args, "")?; + if a.clip_ids.is_some() && a.track_index.is_some() { + return Err(ToolError::new( + "remove_filler_words: pass clipIds or trackIndex, not both", + )); + } + if let Some(ids) = a.clip_ids.as_ref() { + if ids.is_empty() { + return Err(ToolError::new("remove_filler_words: clipIds is empty")); + } + for id in ids { + if find_clip(before, id).is_none() { + return Err(ToolError::new(format!( + "remove_filler_words: clip not found: {id}" + ))); + } + } + } + if let Some(track_index) = a.track_index { + if before.tracks.get(track_index).is_none() { + return Err(ToolError::new(format!( + "remove_filler_words: track not found: {track_index}" + ))); + } + } + + let lexicon = a.filler_words.unwrap_or_else(|| { + ["um", "uh", "er", "erm", "ah", "like", "you know"] + .into_iter() + .map(str::to_string) + .collect() + }); + let mut phrases = lexicon + .into_iter() + .filter_map(|phrase| { + let tokens = phrase + .split_whitespace() + .map(normalize_spoken_token) + .filter(|token| !token.is_empty()) + .collect::>(); + (!tokens.is_empty()).then_some(tokens) + }) + .collect::>(); + phrases.sort(); + phrases.dedup(); + phrases.sort_by_key(|tokens| std::cmp::Reverse(tokens.len())); + if phrases.is_empty() { + return Err(ToolError::new( + "remove_filler_words: fillerWords has no usable phrases", + )); + } + + let transcript = self.get_transcript(&serde_json::json!({}), before, manifest)?; + if transcript.is_error { + return Ok(transcript); + } + let transcript_json: Value = serde_json::from_str(&transcript.text_joined()) + .map_err(|_| ToolError::new("remove_filler_words: transcript response is invalid"))?; + let clips = transcript_json["clips"] + .as_array() + .ok_or_else(|| ToolError::new("remove_filler_words: transcript clips are missing"))?; + let requested_ids = a + .clip_ids + .as_ref() + .map(|ids| ids.iter().map(String::as_str).collect::>()) + .or_else(|| { + a.track_index.map(|track_index| { + before.tracks[track_index] + .clips + .iter() + .map(|clip| clip.id.as_str()) + .collect::>() + }) + }); + let selected_ids = requested_ids.map(|requested| { + let mut expanded = requested + .iter() + .map(|id| (*id).to_string()) + .collect::>(); + let link_groups = requested + .iter() + .filter_map(|id| find_clip(before, id)) + .filter_map(|clip| clip.link_group_id.as_deref()) + .collect::>(); + for clip in before.tracks.iter().flat_map(|track| &track.clips) { + if clip + .link_group_id + .as_deref() + .is_some_and(|group| link_groups.contains(group)) + { + expanded.insert(clip.id.clone()); + } + } + expanded + }); + let padding = a.padding_frames.unwrap_or(1).max(0) as i64; + let mut cuts = Vec::new(); + let mut ranges_by_track: BTreeMap> = BTreeMap::new(); + + for clip in clips { + let Some(clip_id) = clip["clipId"].as_str() else { + continue; + }; + let Some(track_index) = clip["trackIndex"].as_u64() else { + continue; + }; + if selected_ids + .as_ref() + .is_some_and(|ids| !ids.contains(clip_id)) + { + continue; + } + let clip_start = clip["startFrame"].as_i64().unwrap_or(0); + let clip_end = clip["endFrame"].as_i64().unwrap_or(clip_start); + let Some(rows) = clip["words"].as_array() else { + continue; + }; + let normalized = rows + .iter() + .map(|row| normalize_spoken_token(row[0].as_str().unwrap_or_default())) + .collect::>(); + let mut word_index = 0; + while word_index < rows.len() { + let Some(phrase) = phrases.iter().find(|phrase| { + word_index + phrase.len() <= normalized.len() + && normalized[word_index..word_index + phrase.len()] == phrase[..] + }) else { + word_index += 1; + continue; + }; + let last_index = word_index + phrase.len() - 1; + let start = (rows[word_index][1].as_i64().unwrap_or(clip_start) + padding) + .clamp(clip_start, clip_end); + let end = (rows[last_index][2].as_i64().unwrap_or(start) - padding) + .clamp(clip_start, clip_end); + if end > start { + let text = rows[word_index..=last_index] + .iter() + .filter_map(|row| row[0].as_str()) + .collect::>() + .join(" "); + let cut_id = format!("filler-{clip_id}-{word_index}"); + cuts.push(serde_json::json!({ + "id": cut_id, + "clipId": clip_id, + "trackIndex": track_index, + "text": text, + "range": [start, end], + "accepted": true, + })); + ranges_by_track + .entry(track_index) + .or_default() + .push([start, end]); + } + word_index += phrase.len(); + } + } + + for ranges in ranges_by_track.values_mut() { + ranges.sort_unstable(); + ranges.dedup(); + } + cuts.sort_by_key(|cut| { + ( + cut["trackIndex"].as_u64().unwrap_or(0), + cut["range"][0].as_i64().unwrap_or(0), + ) + }); + let commands = ranges_by_track + .into_iter() + .map(|(track_index, ranges)| { + serde_json::json!({ + "tool": "ripple_delete_ranges", + "args": { + "trackIndex": track_index, + "units": "frames", + "ranges": ranges, + } + }) + }) + .collect::>(); + Ok(ToolResult::ok( + serde_json::json!({ + "applied": false, + "cuts": cuts, + "commands": commands, + "note": "Review cuts and remove rejected ranges before calling each returned ripple_delete_ranges command. Each command applies as one undoable edit.", + }) + .to_string(), + )) + } + fn detect_beat_hints( &self, timeline: &Timeline, @@ -2050,6 +2464,7 @@ fn validate_tool_args(tool: ToolName, args: &Value) -> Result<(), ToolError> { ToolName::AutoCutToBeats => decode!(AutoCutToBeatsArgs), ToolName::SmartReframe => decode!(SmartReframeArgs), ToolName::TightenSilences => decode!(TightenSilencesArgs), + ToolName::RemoveFillerWords => decode!(RemoveFillerWordsArgs), ToolName::GenerateVideo => decode!(GenerateVideoArgs), ToolName::GenerateImage => decode!(GenerateImageArgs), ToolName::GenerateAudio => decode!(GenerateAudioArgs), @@ -2146,6 +2561,21 @@ fn validate_tool_args(tool: ToolName, args: &Value) -> Result<(), ToolError> { validate_motion_params(params, "params")?; } } + ToolName::TrackMotion => { + decode!(TrackMotionArgs); + validate_required_object::(args, "region", "region")?; + } + ToolName::GenerateMatte => decode!(GenerateMatteArgs), + ToolName::RemoveObject => decode!(RemoveObjectArgs), + ToolName::MatchColor => decode!(MatchColorArgs), + ToolName::SeparateStems => decode!(SeparateStemsArgs), + ToolName::TranslateCaptions => decode!(TranslateCaptionsArgs), + ToolName::ScriptToVideo => { + decode!(ScriptToVideoArgs); + validate_array::(args, "segments")?; + } + ToolName::GenerateAvatar => decode!(GenerateAvatarArgs), + ToolName::CloneVoice => decode!(CloneVoiceArgs), } Ok(()) } @@ -2164,6 +2594,45 @@ fn validate_motion_params( Ok(()) } +fn motion_bridge_error(tool: ToolName, error: MotionBridgeError) -> ToolResult { + match error.kind { + MotionBridgeErrorKind::InvalidArguments => { + ToolResult::public_error(PublicErrorKind::InvalidArguments(tool), error.message) + } + MotionBridgeErrorKind::ResourceNotFound => { + ToolResult::public_error(PublicErrorKind::ResourceNotFound(tool), error.message) + } + MotionBridgeErrorKind::CapabilityUnavailable => { + ToolResult::public_error(PublicErrorKind::CapabilityUnavailable(tool), error.message) + } + MotionBridgeErrorKind::Cancelled => ToolResult::error("motion render cancelled"), + MotionBridgeErrorKind::RenderFailed => ToolResult::error("motion render failed"), + } +} + +fn advanced_workflow_error(tool: ToolName, error: AdvancedWorkflowError) -> ToolResult { + match error.kind { + AdvancedWorkflowErrorKind::InvalidArguments => { + ToolResult::public_error(PublicErrorKind::InvalidArguments(tool), error.message) + } + AdvancedWorkflowErrorKind::ResourceNotFound => { + ToolResult::public_error(PublicErrorKind::ResourceNotFound(tool), error.message) + } + AdvancedWorkflowErrorKind::CapabilityUnavailable => { + ToolResult::public_error(PublicErrorKind::CapabilityUnavailable(tool), error.message) + } + AdvancedWorkflowErrorKind::AnalysisLowConfidence => { + ToolResult::public_error(PublicErrorKind::AnalysisLowConfidence(tool), error.message) + } + AdvancedWorkflowErrorKind::ConsentRequired + | AdvancedWorkflowErrorKind::CostAuthorizationRequired + | AdvancedWorkflowErrorKind::ExecutionFailed => { + ToolResult::error("advanced workflow failed") + } + AdvancedWorkflowErrorKind::Cancelled => ToolResult::error("advanced workflow cancelled"), + } +} + fn validate_array(args: &Value, field: &str) -> Result<(), ToolError> { let Some(values) = args.get(field).and_then(Value::as_array) else { return Ok(()); // the owning top-level decode reports missing/wrong type @@ -2434,6 +2903,107 @@ fn clip_location(timeline: &Timeline, clip_id: &str) -> (Option, Option Result<(Vec, Vec), ToolError> { + if clip_ids.is_empty() { + return Err(ToolError::new( + "auto_cut_to_beats: write=true requires a non-empty clipIds array", + )); + } + + let mut roots = Vec::new(); + let mut seen_roots = BTreeSet::new(); + for clip_id in clip_ids { + let clip = find_clip(timeline, clip_id).ok_or_else(|| { + ToolError::new(format!("auto_cut_to_beats: clip not found: {clip_id}")) + })?; + if !clip.media_type.is_visual() { + return Err(ToolError::new(format!( + "auto_cut_to_beats: clip is not visual: {clip_id}" + ))); + } + let root_key = clip + .link_group_id + .as_ref() + .map(|group| format!("link:{group}")) + .unwrap_or_else(|| format!("clip:{clip_id}")); + if seen_roots.insert(root_key) { + roots.push(( + clip.id.clone(), + clip.start_frame, + clip.link_group_id.clone(), + )); + } + } + if beat_frames.len() < roots.len() { + return Err(ToolError::new(format!( + "auto_cut_to_beats: need at least {} beat frame(s) for write, got {}", + roots.len(), + beat_frames.len() + ))); + } + + let mut moves = Vec::new(); + let mut placements = Vec::new(); + let mut moved_ids = BTreeSet::new(); + for ((root_id, root_start, link_group), beat_frame) in + roots.into_iter().zip(beat_frames.iter().copied()) + { + let delta = beat_frame + .checked_sub(root_start) + .ok_or_else(|| ToolError::new("auto_cut_to_beats: placement frame delta overflow"))?; + let mut linked_clip_ids = Vec::new(); + for (track_index, clip) in + timeline + .tracks + .iter() + .enumerate() + .flat_map(|(track_index, track)| { + track.clips.iter().map(move |clip| (track_index, clip)) + }) + { + let belongs = match link_group.as_deref() { + Some(group) => clip.link_group_id.as_deref() == Some(group), + None => clip.id == root_id, + }; + if !belongs || !moved_ids.insert(clip.id.clone()) { + continue; + } + let to_frame = clip.start_frame.checked_add(delta).ok_or_else(|| { + ToolError::new(format!( + "auto_cut_to_beats: linked placement frame overflow: {}", + clip.id + )) + })?; + if to_frame < 0 { + return Err(ToolError::new(format!( + "auto_cut_to_beats: linked placement would start before frame zero: {}", + clip.id + ))); + } + linked_clip_ids.push(clip.id.clone()); + moves.push(ClipMove { + clip_id: clip.id.clone(), + to_track: track_index, + to_frame, + }); + } + placements.push(serde_json::json!({ + "clipId": root_id, + "fromFrame": root_start, + "toFrame": beat_frame, + "linkedClipIds": linked_clip_ids, + })); + } + Ok((moves, placements)) +} + #[derive(Clone, Debug)] struct BeatHint { frame: i32, @@ -2669,6 +3239,14 @@ fn normalized_speed(clip: &opentake_domain::Clip) -> f64 { } } +fn normalize_spoken_token(value: &str) -> String { + value + .chars() + .flat_map(char::to_lowercase) + .filter(|character| character.is_alphanumeric() || *character == '\'') + .collect() +} + fn source_seconds_to_timeline_frame_clamped( clip: &opentake_domain::Clip, source_seconds: f64, @@ -2941,6 +3519,7 @@ fn color_grade_from_args(a: &SetColorGradeArgs) -> ColorGrade { }, contrast: a.contrast.unwrap_or(base.contrast), saturation: a.saturation.unwrap_or(base.saturation), + hsl_secondary: None, } } @@ -3007,6 +3586,7 @@ fn mask_from_arg(m: &MaskArg, path: &str) -> Result { shape, feather: m.feather.unwrap_or(0.0), invert: m.invert.unwrap_or(false), + ..Mask::default() }) } @@ -3616,8 +4196,7 @@ mod tests { ); } - #[test] - fn hidden_tool_is_rejected_as_unadvertised() { + fn assert_hidden_tool_is_rejected_as_unadvertised() { let d = dispatcher_with(Arc::new(TestHandle::new())); let r = d.dispatch("generate_video", serde_json::json!({"prompt": "x"})); assert!(r.is_error); @@ -3625,6 +4204,18 @@ mod tests { assert!(r.text_joined().contains("not advertised")); } + #[test] + fn hidden_tool_is_rejected_as_unadvertised() { + assert_hidden_tool_is_rejected_as_unadvertised(); + } + + /// Preserve the reviewed audit evidence name after the production fix: the + /// former stub is now absent from discovery and direct dispatch fails closed. + #[test] + fn stub_tool_reports_not_implemented() { + assert_hidden_tool_is_rejected_as_unadvertised(); + } + #[test] fn get_media_returns_json_object() { let d = dispatcher_with(Arc::new(TestHandle::new())); @@ -3701,6 +4292,12 @@ mod tests { pcm: opentake_media::PcmBuffer, } + struct WritableAnalysisHandle { + state: Mutex, + pcm: opentake_media::PcmBuffer, + commands: Mutex>, + } + impl CoreHandle for AnalysisHandle { fn timeline(&self) -> Timeline { self.timeline.clone() @@ -3731,6 +4328,32 @@ mod tests { } } + impl CoreHandle for WritableAnalysisHandle { + fn timeline(&self) -> Timeline { + self.state.lock().unwrap().timeline.clone() + } + fn media(&self) -> MediaManifest { + self.state.lock().unwrap().manifest.clone() + } + fn apply(&self, cmd: EditCommand) -> anyhow::Result { + self.commands.lock().unwrap().push(cmd.clone()); + let ids = SeqIdGen::new("beat-"); + ops_apply(&mut self.state.lock().unwrap(), cmd, &ids) + .map_err(|e| anyhow::anyhow!("{e}")) + } + fn project_dir(&self) -> Option { + None + } + fn extract_analysis_pcm( + &self, + _media_ref: &str, + _spec: opentake_media::PcmSpec, + _range: Option<(f64, f64)>, + ) -> anyhow::Result { + Ok(self.pcm.clone()) + } + } + fn pcm(samples: Vec, sample_rate: u32) -> opentake_media::PcmBuffer { opentake_media::PcmBuffer { spec: opentake_media::PcmSpec { @@ -3781,6 +4404,8 @@ mod tests { source_height: None, source_fps: None, has_audio: Some(false), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -3797,6 +4422,32 @@ mod tests { e } + fn linked_beat_handle() -> Arc { + let mut timeline = Timeline::new(); + timeline.fps = 10; + let mut video_track = Track::new("video-track", ClipType::Video); + let mut video = Clip::new("video-a", "video-source", 20, 5); + video.link_group_id = Some("linked-av".into()); + video_track.clips.push(video); + let mut audio_track = Track::new("audio-track", ClipType::Audio); + let mut audio = Clip::new("audio-a", "video-source", 20, 5); + audio.media_type = ClipType::Audio; + audio.source_clip_type = ClipType::Audio; + audio.link_group_id = Some("linked-av".into()); + audio_track.clips.push(audio); + timeline.tracks = vec![video_track, audio_track]; + + let mut manifest = MediaManifest::new(); + manifest.entries.push(audio_entry("music", "Music")); + let mut samples = vec![0.0; 1_000]; + samples[500..530].fill(1.0); + Arc::new(WritableAnalysisHandle { + state: Mutex::new(EditorState::new(timeline, manifest)), + pcm: pcm(samples, 1_000), + commands: Mutex::new(Vec::new()), + }) + } + /// A video asset whose source carries an audio track (`hasAudio: true`) — /// the case `add_clips`/`insert_clips` should auto-create a linked audio /// partner for. @@ -4367,6 +5018,88 @@ mod tests { ); } + #[test] + fn auto_cut_to_beats_write_false_is_read_only() { + let handle = linked_beat_handle(); + let before = handle.timeline(); + let dispatcher = dispatcher_with(handle.clone()); + + let result = dispatcher.dispatch( + "auto_cut_to_beats", + serde_json::json!({ + "clipIds": ["video-a"], + "beatMediaRef": "music", + "write": false + }), + ); + + assert!(!result.is_error, "{}", result.text_joined()); + assert_eq!(first_json(&result)["applied"], false); + assert!(handle.commands.lock().unwrap().is_empty()); + assert_eq!(handle.timeline(), before); + } + + #[test] + fn auto_cut_to_beats_write_true_is_one_atomic_command_and_preserves_links() { + let handle = linked_beat_handle(); + let dispatcher = dispatcher_with(handle.clone()); + + let before = handle.timeline(); + let contradictory = dispatcher.dispatch( + "auto_cut_to_beats", + serde_json::json!({ + "clipIds": ["video-a"], + "beatMediaRef": "music", + "alignCuts": false, + "write": true + }), + ); + assert!(contradictory.is_error); + assert!(contradictory.text_joined().contains("conflicts")); + assert!(handle.commands.lock().unwrap().is_empty()); + assert_eq!(handle.timeline(), before); + + let rejected = dispatcher.dispatch( + "auto_cut_to_beats", + serde_json::json!({ + "clipIds": ["missing-clip"], + "beatMediaRef": "music", + "write": true + }), + ); + assert!(rejected.is_error); + assert!(rejected.text_joined().contains("clip not found")); + assert!(handle.commands.lock().unwrap().is_empty()); + assert_eq!(handle.timeline(), before); + + let result = dispatcher.dispatch( + "auto_cut_to_beats", + serde_json::json!({ + "clipIds": ["video-a"], + "beatMediaRef": "music", + "write": true + }), + ); + + assert!(!result.is_error, "{}", result.text_joined()); + assert_eq!(first_json(&result)["applied"], true); + let commands = handle.commands.lock().unwrap(); + assert_eq!(commands.len(), 1); + let EditCommand::MoveClips { moves } = &commands[0] else { + panic!("auto cut must use one MoveClips command: {:?}", commands[0]); + }; + assert_eq!(moves.len(), 2); + drop(commands); + + let after = handle.timeline(); + let video = find_clip(&after, "video-a").expect("video remains"); + let audio = find_clip(&after, "audio-a").expect("linked audio remains"); + assert!((4..=5).contains(&video.start_frame)); + assert_eq!(audio.start_frame, video.start_frame); + assert_eq!(video.link_group_id.as_deref(), Some("linked-av")); + assert_eq!(audio.link_group_id, video.link_group_id); + } + #[test] fn smart_reframe_reports_needs_vision_backend() { let d = dispatcher_with(empty_manifest_handle(vec![])); @@ -4543,18 +5276,99 @@ mod tests { } #[test] - fn remove_filler_words_stays_disabled_until_transcript_is_wired() { - let d = dispatcher_with(empty_manifest_handle(vec![])); - let r = d.dispatch("remove_filler_words", serde_json::json!({})); - assert!(r.is_error); - assert!( - r.text_joined() - .contains("Unknown tool: remove_filler_words"), - "{}", - r.text_joined() + fn remove_filler_words_returns_reviewable_word_aligned_ranges() { + let (d, _bridge) = transcript_dispatcher(transcript(vec![ + word("Well", 0.0, 0.2), + word("um", 0.2, 0.4), + word("you", 0.5, 0.7), + word("know", 0.7, 0.9), + word("go", 1.0, 1.2), + ])); + assert!(d.advertised_tools().contains(&ToolName::RemoveFillerWords)); + let r = d.dispatch( + "remove_filler_words", + serde_json::json!({ + "clipIds": ["clip-a"], + "fillerWords": ["um", "you know"], + "paddingFrames": 0 + }), + ); + assert!(!r.is_error, "{}", r.text_joined()); + let json = first_json(&r); + assert_eq!(json["applied"], false); + assert_eq!(json["cuts"].as_array().unwrap().len(), 2); + assert_eq!(json["cuts"][0]["text"], "um"); + assert_eq!(json["cuts"][0]["range"], serde_json::json!([6, 12])); + assert_eq!(json["cuts"][1]["text"], "you know"); + assert_eq!(json["cuts"][1]["range"], serde_json::json!([15, 27])); + assert_eq!( + json["commands"][0]["args"]["ranges"], + serde_json::json!([[6, 12], [15, 27]]) ); } + #[test] + fn reviewed_filler_cut_applies_once_and_undo_restores_the_timeline() { + let (d, _bridge) = linked_talking_head_dispatcher(transcript(vec![ + word("Well", 0.0, 0.2), + word("um", 0.2, 0.4), + word("you", 0.5, 0.7), + word("know", 0.7, 0.9), + word("go", 1.0, 1.2), + ])); + let before = d.handle.timeline(); + let preview = d.dispatch( + "remove_filler_words", + serde_json::json!({ + "clipIds": ["clip-v"], + "fillerWords": ["um", "you know"], + "paddingFrames": 0 + }), + ); + let json = first_json(&preview); + let apply = d.dispatch( + "ripple_delete_ranges", + serde_json::json!({ + "trackIndex": 1, + "units": "frames", + "ranges": [json["cuts"][0]["range"].clone()] + }), + ); + assert!(!apply.is_error, "{}", apply.text_joined()); + let after = d.handle.timeline(); + assert_ne!(after, before); + assert_eq!(after.tracks.len(), 2); + let video_ranges = after.tracks[0] + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>(); + let audio_ranges = after.tracks[1] + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>(); + assert_eq!(video_ranges, audio_ranges, "linked A/V ranges drifted"); + assert_eq!(video_ranges.last().map(|range| range.1), Some(894)); + + let post_cut = d.dispatch("get_transcript", serde_json::json!({})); + assert!(!post_cut.is_error, "{}", post_cut.text_joined()); + let post_cut_json = first_json(&post_cut); + let spoken = post_cut_json["clips"] + .as_array() + .unwrap() + .iter() + .flat_map(|clip| clip["words"].as_array().unwrap()) + .filter_map(|word| word[0].as_str()) + .collect::>(); + assert!(!spoken.contains(&"um"), "{spoken:?}"); + assert!(spoken.windows(2).any(|words| words == ["you", "know"])); + + let undo = d.dispatch("undo", serde_json::json!({})); + assert!(!undo.is_error, "{}", undo.text_joined()); + assert_eq!(d.handle.timeline(), before); + } + #[test] fn rename_media_updates_manifest_name() { let h = seeded_handle(); @@ -5140,14 +5954,12 @@ mod tests { } #[test] - fn inspect_timeline_without_bridge_reports_unavailable() { - // The seeded TestHandle timeline is empty, so first assert the empty guard, - // then a non-empty timeline with no bridge reports "not available". + fn inspect_timeline_without_bridge_is_not_advertised() { let d = dispatcher_with(seeded_handle()); let r = d.dispatch("inspect_timeline", serde_json::json!({ "startFrame": 0 })); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5272,7 +6084,7 @@ mod tests { } #[test] - fn import_media_without_bridge_reports_unavailable() { + fn import_media_without_bridge_is_not_advertised() { let d = dispatcher_with(seeded_handle()); let r = d.dispatch( "import_media", @@ -5280,7 +6092,7 @@ mod tests { ); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5617,7 +6429,7 @@ mod tests { } #[test] - fn search_media_without_bridge_reports_unavailable() { + fn search_media_without_bridge_is_not_advertised() { let mut m = MediaManifest::new(); m.entries.push(entry("v", "Clip")); let handle = Arc::new(StateHandle::new(Timeline::new(), m)); @@ -5625,7 +6437,7 @@ mod tests { let r = d.dispatch("search_media", serde_json::json!({ "query": "x" })); assert!(r.is_error); assert!( - r.text_joined().contains("not available in this build"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -5674,6 +6486,39 @@ mod tests { (d, bridge) } + /// A fixed 30-second talking-head fixture with linked video/audio clips. + /// Only the audio partner is transcribed, matching production caption target + /// selection, while a reviewed ripple cut must keep both tracks frame-exact. + fn linked_talking_head_dispatcher(t: TranscriptionResult) -> (Dispatcher, Arc) { + let mut tl = Timeline::new(); + tl.fps = 30; + + let mut video_track = Track::new("track-v", ClipType::Video); + let mut video = Clip::new("clip-v", "vid", 0, 30 * 30); + video.link_group_id = Some("talking-head-av".into()); + video_track.clips.push(video); + + let mut audio_track = Track::new("track-a", ClipType::Audio); + let mut audio = Clip::new("clip-a", "aud", 0, 30 * 30); + audio.media_type = ClipType::Audio; + audio.link_group_id = Some("talking-head-av".into()); + audio_track.clips.push(audio); + + tl.tracks.push(video_track); + tl.tracks.push(audio_track); + let mut manifest = MediaManifest::new(); + manifest.entries.push(entry("vid", "Camera")); + manifest.entries.push(audio_entry("aud", "Voice")); + let handle = Arc::new(StateHandle::new(tl, manifest)); + let bridge = Arc::new(FakeBridge::default().with_transcript("aud", t)); + let dispatcher = Dispatcher::with_bridge( + handle, + Arc::new(RwLock::new(PluginRegistry::new())), + Some(bridge.clone() as Arc), + ); + (dispatcher, bridge) + } + #[test] fn get_transcript_maps_words_to_project_frames() { let (d, _b) = transcript_dispatcher(transcript(vec![ @@ -5700,8 +6545,7 @@ mod tests { } #[test] - fn get_transcript_without_bridge_reports_unavailable() { - // Same audio timeline but no bridge wired → honest "not available". + fn get_transcript_without_bridge_is_not_advertised() { let mut tl = Timeline::new(); tl.fps = 30; let mut track = opentake_domain::Track::new("track-a", ClipType::Audio); @@ -5715,7 +6559,7 @@ mod tests { let r = d.dispatch("get_transcript", serde_json::json!({})); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); @@ -6061,7 +6905,7 @@ mod tests { } #[test] - fn add_captions_without_bridge_reports_unavailable() { + fn add_captions_without_bridge_is_not_advertised() { let mut tl = Timeline::new(); tl.fps = 30; tl.width = 1920; @@ -6077,7 +6921,7 @@ mod tests { let r = d.dispatch("add_captions", serde_json::json!({})); assert!(r.is_error); assert!( - r.text_joined().contains("not available"), + r.text_joined().contains("not advertised"), "{}", r.text_joined() ); diff --git a/crates/opentake-agent/src/mcp/mod.rs b/crates/opentake-agent/src/mcp/mod.rs index 98c01322..2d497180 100644 --- a/crates/opentake-agent/src/mcp/mod.rs +++ b/crates/opentake-agent/src/mcp/mod.rs @@ -9,10 +9,12 @@ //! shortens outbound ids. The rmcp server / HTTP handler is a thin shim over this //! and lands in a later phase. +pub mod advanced; pub mod convert; pub mod core_handle; pub mod dispatch; pub mod gen_catalog; pub mod generation; pub mod media_bridge; +pub mod motion; pub mod server; diff --git a/crates/opentake-agent/src/mcp/motion.rs b/crates/opentake-agent/src/mcp/motion.rs new file mode 100644 index 00000000..befbac89 --- /dev/null +++ b/crates/opentake-agent/src/mcp/motion.rs @@ -0,0 +1,101 @@ +//! Host boundary for deterministic motion-graphic rendering and placement. +//! +//! The Agent crate owns schemas and discovery, while the desktop host owns the +//! browser/renderer, project filesystem authority, media import, and atomic +//! timeline transaction. Keeping those capabilities behind this trait lets the +//! tool contract run against deterministic fakes without advertising a stub in +//! hosts that do not provide the production bridge. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Debug, Clone, PartialEq)] +pub enum MotionSourceRequest { + Code(String), + Template { + template_id: String, + params: Map, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AddMotionRequest { + pub source: MotionSourceRequest, + pub start_frame: i32, + pub duration_frames: i32, + pub transparent: bool, + pub track_index: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct EditMotionRequest { + pub clip_id: String, + pub code: Option, + pub params: Option>, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MotionCommit { + pub clip_id: String, + pub asset_id: String, + pub content_hash: String, + pub action_name: String, + pub output: MotionOutputMetadata, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MotionOutputMetadata { + pub renderer: String, + pub renderer_version: String, + pub output_file: String, + pub fps: f64, + pub width: u32, + pub height: u32, + pub duration_frames: i32, + pub duration_seconds: f64, + pub content_hash: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MotionBridgeErrorKind { + InvalidArguments, + ResourceNotFound, + CapabilityUnavailable, + Cancelled, + RenderFailed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MotionBridgeError { + pub kind: MotionBridgeErrorKind, + pub message: String, +} + +impl MotionBridgeError { + pub fn new(kind: MotionBridgeErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } +} + +pub trait MotionBridge: Send + Sync { + /// True only when the host has a production renderer and project commit + /// path. Discovery omits both motion tools when this returns false. + fn can_render_motion(&self) -> bool; + + fn add( + &self, + request: AddMotionRequest, + cancel: &opentake_media::MediaCancelToken, + ) -> Result; + + fn edit( + &self, + request: EditMotionRequest, + cancel: &opentake_media::MediaCancelToken, + ) -> Result; +} diff --git a/crates/opentake-agent/src/mcp/server.rs b/crates/opentake-agent/src/mcp/server.rs index 3afc2b06..31fa2105 100644 --- a/crates/opentake-agent/src/mcp/server.rs +++ b/crates/opentake-agent/src/mcp/server.rs @@ -24,11 +24,13 @@ use rmcp::service::RequestContext; use rmcp::{ErrorData as McpError, RoleServer, ServerHandler}; use serde_json::{Map, Value}; +use crate::mcp::advanced::AdvancedWorkflowBridge; use crate::mcp::convert::to_call_tool_result; use crate::mcp::core_handle::CoreHandle; use crate::mcp::dispatch::Dispatcher; use crate::mcp::generation::GenerationBridge; use crate::mcp::media_bridge::{MediaBridge, MCP_REQUEST_BODY_MAX}; +use crate::mcp::motion::MotionBridge; use crate::plugin::registry::PluginRegistry; use crate::prompt::assemble::assemble_system_prompt; use crate::tools::descriptions::{description, input_schema}; @@ -80,17 +82,47 @@ impl McpServer { registry: Arc>, bridge: Option>, generation_bridge: Option>, + ) -> Self { + Self::with_capability_bridges(handle, registry, bridge, generation_bridge, None) + } + + pub fn with_capability_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + ) -> Self { + Self::with_all_capability_bridges( + handle, + registry, + bridge, + generation_bridge, + motion_bridge, + None, + ) + } + + pub fn with_all_capability_bridges( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + advanced_bridge: Option>, ) -> Self { let instructions = registry .read() .map(|r| assemble_system_prompt(&r, "default")) .unwrap_or_default(); McpServer { - dispatcher: Arc::new(Dispatcher::with_bridges( + dispatcher: Arc::new(Dispatcher::with_all_capability_bridges( handle, registry, bridge, generation_bridge, + motion_bridge, + advanced_bridge, )), instructions, } @@ -480,6 +512,44 @@ pub fn build_router_with_bridges_for_port( bridge: Option>, generation_bridge: Option>, expected_port: u16, +) -> axum::Router { + build_router_with_capability_bridges_for_port( + handle, + registry, + bridge, + generation_bridge, + None, + expected_port, + ) +} + +pub fn build_router_with_capability_bridges_for_port( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + expected_port: u16, +) -> axum::Router { + build_router_with_all_capability_bridges_for_port( + handle, + registry, + bridge, + generation_bridge, + motion_bridge, + None, + expected_port, + ) +} + +pub fn build_router_with_all_capability_bridges_for_port( + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + advanced_bridge: Option>, + expected_port: u16, ) -> axum::Router { use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use rmcp::transport::streamable_http_server::{ @@ -490,11 +560,13 @@ pub fn build_router_with_bridges_for_port( let service = StreamableHttpService::new( move || { - Ok(McpServer::with_bridges( + Ok(McpServer::with_all_capability_bridges( handle.clone(), registry.clone(), bridge.clone(), generation_bridge.clone(), + motion_bridge.clone(), + advanced_bridge.clone(), )) }, Arc::new(LocalSessionManager::default()), @@ -546,6 +618,38 @@ pub async fn serve_with_bridges( registry: Arc>, bridge: Option>, generation_bridge: Option>, +) -> std::io::Result<()> { + serve_with_capability_bridges(addr, handle, registry, bridge, generation_bridge, None).await +} + +pub async fn serve_with_capability_bridges( + addr: SocketAddr, + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, +) -> std::io::Result<()> { + serve_with_all_capability_bridges( + addr, + handle, + registry, + bridge, + generation_bridge, + motion_bridge, + None, + ) + .await +} + +pub async fn serve_with_all_capability_bridges( + addr: SocketAddr, + handle: Arc, + registry: Arc>, + bridge: Option>, + generation_bridge: Option>, + motion_bridge: Option>, + advanced_bridge: Option>, ) -> std::io::Result<()> { if !addr.ip().is_loopback() { return Err(std::io::Error::new( @@ -555,11 +659,13 @@ pub async fn serve_with_bridges( } let listener = tokio::net::TcpListener::bind(addr).await?; let bound_addr = listener.local_addr()?; - let router = build_router_with_bridges_for_port( + let router = build_router_with_all_capability_bridges_for_port( handle, registry, bridge, generation_bridge, + motion_bridge, + advanced_bridge, bound_addr.port(), ); tracing::info!("MCP server listening on http://{bound_addr}/mcp"); @@ -612,12 +718,17 @@ mod tests { #[test] fn lists_every_advertised_tool() { let server = server(); - assert_eq!(server.tools().len(), ToolName::ALL.len()); + let expected = ToolName::ALL + .iter() + .filter(|tool| !tool.requires_media_bridge()) + .count(); + assert_eq!(server.tools().len(), expected); // Names round-trip to the wire names. let names: Vec = server.tools().iter().map(|t| t.name.to_string()).collect(); assert!(names.contains(&"add_clips".to_string())); assert!(names.contains(&"detect_beats".to_string())); assert!(names.contains(&"activate_workflow".to_string())); + assert!(!names.contains(&"remove_filler_words".to_string())); } #[test] diff --git a/crates/opentake-agent/src/tools/args.rs b/crates/opentake-agent/src/tools/args.rs index cc67a9f2..25634c68 100644 --- a/crates/opentake-agent/src/tools/args.rs +++ b/crates/opentake-agent/src/tools/args.rs @@ -6,6 +6,7 @@ //! entry keys). use serde::Deserialize; +use serde_json::Value; use crate::tools::errors::ToolArgs; @@ -631,6 +632,7 @@ pub struct AutoCutToBeatsArgs { pub min_clip_frames: Option, pub max_clip_frames: Option, pub align_cuts: Option, + pub write: Option, } impl ToolArgs for AutoCutToBeatsArgs { const ALLOWED_KEYS: &'static [&'static str] = &[ @@ -642,6 +644,7 @@ impl ToolArgs for AutoCutToBeatsArgs { "minClipFrames", "maxClipFrames", "alignCuts", + "write", ]; } @@ -678,6 +681,226 @@ impl ToolArgs for TightenSilencesArgs { ]; } +// --- remove_filler_words --- +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RemoveFillerWordsArgs { + pub clip_ids: Option>, + pub track_index: Option, + pub filler_words: Option>, + pub padding_frames: Option, +} +impl ToolArgs for RemoveFillerWordsArgs { + const ALLOWED_KEYS: &'static [&'static str] = + &["clipIds", "trackIndex", "fillerWords", "paddingFrames"]; +} + +// --- capability-gated advanced AI workflows --- +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +pub struct MotionRegionArg { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} +impl ToolArgs for MotionRegionArg { + const ALLOWED_KEYS: &'static [&'static str] = &["x", "y", "width", "height"]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TrackMotionArgs { + pub clip_id: String, + pub region: Value, + pub start_frame: Option, + pub end_frame: Option, + pub apply: Option, +} +impl ToolArgs for TrackMotionArgs { + const ALLOWED_KEYS: &'static [&'static str] = + &["clipId", "region", "startFrame", "endFrame", "apply"]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct GenerateMatteArgs { + pub clip_id: String, + pub model: Option, + pub start_frame: Option, + pub end_frame: Option, + pub apply: Option, +} +impl ToolArgs for GenerateMatteArgs { + const ALLOWED_KEYS: &'static [&'static str] = + &["clipId", "model", "startFrame", "endFrame", "apply"]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RemoveObjectArgs { + pub clip_id: String, + pub mask_id: String, + pub start_frame: Option, + pub end_frame: Option, + pub provider: Option, + pub model: Option, + pub cost_authorized: Option, + pub apply: Option, +} +impl ToolArgs for RemoveObjectArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "clipId", + "maskId", + "startFrame", + "endFrame", + "provider", + "model", + "costAuthorized", + "apply", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MatchColorArgs { + pub clip_id: String, + pub reference_media_ref: String, + pub reference_frame: Option, + pub target_frame: Option, + pub apply: Option, +} +impl ToolArgs for MatchColorArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "clipId", + "referenceMediaRef", + "referenceFrame", + "targetFrame", + "apply", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SeparateStemsArgs { + pub media_ref: String, + pub provider: Option, + pub model: Option, + pub import_to_tracks: Option, + pub start_frame: Option, +} +impl ToolArgs for SeparateStemsArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "mediaRef", + "provider", + "model", + "importToTracks", + "startFrame", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TranslateCaptionsArgs { + pub caption_clip_ids: Vec, + pub source_locale: Option, + pub target_locale: String, + pub provider: Option, + pub model: Option, + pub cost_authorized: Option, + pub apply: Option, +} +impl ToolArgs for TranslateCaptionsArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "captionClipIds", + "sourceLocale", + "targetLocale", + "provider", + "model", + "costAuthorized", + "apply", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ScriptToVideoArgs { + pub segments: Vec, + pub apply: Option, +} +impl ToolArgs for ScriptToVideoArgs { + const ALLOWED_KEYS: &'static [&'static str] = &["segments", "apply"]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct ScriptSegmentArg { + pub script: String, + pub media_ref: String, + pub narration_media_ref: Option, + pub duration_frames: i32, + pub transition: Option, +} +impl ToolArgs for ScriptSegmentArg { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "script", + "mediaRef", + "narrationMediaRef", + "durationFrames", + "transition", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct GenerateAvatarArgs { + pub portrait_media_ref: String, + pub audio_media_ref: String, + pub consent_id: String, + pub provider: Option, + pub model: Option, + pub cost_authorized: Option, + pub start_frame: Option, +} +impl ToolArgs for GenerateAvatarArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "portraitMediaRef", + "audioMediaRef", + "consentId", + "provider", + "model", + "costAuthorized", + "startFrame", + ]; +} + +#[derive(Debug, Clone, Default, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CloneVoiceArgs { + pub action: String, + pub reference_audio_media_ref: Option, + pub consent_id: String, + pub voice_id: Option, + pub voice_name: Option, + pub prompt: Option, + pub provider: Option, + pub model: Option, + pub cost_authorized: Option, +} +impl ToolArgs for CloneVoiceArgs { + const ALLOWED_KEYS: &'static [&'static str] = &[ + "action", + "referenceAudioMediaRef", + "consentId", + "voiceId", + "voiceName", + "prompt", + "provider", + "model", + "costAuthorized", + ]; +} + // --- generate_video --- #[derive(Debug, Clone, Default, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -1185,13 +1408,13 @@ mod tests { fn apply_effect_decodes_with_params() { let v = serde_json::json!({ "clipIds": ["a"], - "effects": [{"name": "gaussianBlur", "params": {"radius": 4.0}}] + "effects": [{"name": "grayscale", "params": {"amount": 0.4}}] }); let a: ApplyEffectArgs = decode_tool_args(&v, "").unwrap(); assert_eq!(a.effects.len(), 1); let e: EffectArg = decode_tool_args(&a.effects[0], "effects[0]").unwrap(); - assert_eq!(e.name, "gaussianBlur"); - assert_eq!(e.params.unwrap().get("radius"), Some(&4.0)); + assert_eq!(e.name, "grayscale"); + assert_eq!(e.params.unwrap().get("amount"), Some(&0.4)); } #[test] diff --git a/crates/opentake-agent/src/tools/descriptions.rs b/crates/opentake-agent/src/tools/descriptions.rs index 74042e7f..df810189 100644 --- a/crates/opentake-agent/src/tools/descriptions.rs +++ b/crates/opentake-agent/src/tools/descriptions.rs @@ -55,12 +55,14 @@ pub fn description(tool: ToolName) -> &'static str { ToolName::DetectBeats => "Detects musical beat positions for a clip or media asset using lightweight PCM energy/onset analysis. Returns project-frame beat hints and strengths; it does not mutate the timeline.", - ToolName::AutoCutToBeats => "Plans beat-synced cuts for one or more clips against an audio or music source. Returns beat frames, suggested cut frames, and optional clip placement hints; it does not mutate the timeline. Apply the plan with existing edit tools.", + ToolName::AutoCutToBeats => "Plans beat-synced alignment for visual clips against an audio or music source. The default write=false returns beat frames, suggested cut frames, and clip placement hints without mutation. Set write=true to align the selected visual clips and their linked A/V partners through one atomic MoveClips command.", ToolName::SmartReframe => "Plans subject-aware reframing for target aspect ratios such as 9:16 or 1:1. The typed surface is present, but MCP frame sampling / vision analysis is not wired yet; calls return a deterministic needs-vision-backend error and do not mutate the timeline.", ToolName::TightenSilences => "Plans silence tightening by finding low-energy PCM spans and converting them into ripple_delete_ranges candidate commands. Returns a preview only; it does not mutate the timeline.", + ToolName::RemoveFillerWords => "Transcribes the current spoken timeline and returns reviewable filler-word cuts aligned to word timestamps. Supports an exact configurable lexicon including multi-word phrases. It does not mutate the timeline: remove rejected cuts, then call each returned ripple_delete_ranges command to apply the accepted ranges as one undoable edit per track.", + ToolName::GenerateVideo => "Starts an async AI video generation. Returns a placeholder asset ID immediately; generation runs in the background and the asset becomes usable in add_clips once ready. Costs real money and is not undoable.", ToolName::GenerateImage => "Starts an async AI image generation. Returns a placeholder asset ID immediately; generation runs in the background. Costs real money and is not undoable.", @@ -101,12 +103,22 @@ pub fn description(tool: ToolName) -> &'static str { ToolName::SetMask => "Sets the vector mask(s) on one or more clips in one undoable action — the masks generate a per-pixel alpha that hides everything outside them (intersection of all masks). Each mask is one of: a linear/gradient split (a line through a point with a normal), a circle/ellipse (center + per-axis radius), or a polygon/pen shape (a list of points). feather softens the edge in normalized canvas units; invert flips inside/outside. Coordinates are 0–1 normalized canvas space. Pass an empty masks array to clear all masks. Applies to every clip in clipIds.", - ToolName::ApplyEffect => "Sets the effect chain on one or more clips in one undoable action — an ordered list of named pixel effects, each a shader pass with named numeric parameters. Each effect is { name, params } where name selects the effect (e.g. 'gaussianBlur') and params are its scalar inputs (e.g. { radius: 4 }); pass enabled:false to keep a disabled effect in the chain. The list replaces the clip's current effects; pass an empty array to clear them. Applies to every clip in clipIds.", + ToolName::ApplyEffect => "Sets the effect chain on one or more clips in one undoable action. The closed effect registry is grayscale, sepia, and invert; each accepts an optional amount from 0 to 1 (default 1). Effects execute in list order in the shared preview/export GPU compositor. Pass enabled:false to retain a disabled effect. The list replaces the current chain; pass an empty array to clear it. Unknown names, parameters, non-finite values, and out-of-range values are rejected instead of rendering unchanged. Applies to every clip in clipIds.", + + // --- OpenTake deterministic motion graphics (Issue #34 fallback vertical) --- + ToolName::AddMotionGraphic => "Renders a deterministic motion graphic to MP4, imports it, and places it on the timeline as one durable undoable workflow. Returns the new clipId. The packaged Beta uses the pinned Motion Canvas 3.17.2 runner for 'title-card'; it also supports the local 'lower-third.glass' template and self-contained HTML/CSS/JS fallback (animated through OpenTake.onSeek). Raw TypeScript/TSX and transparent output are reported as unsupported instead of being accepted as placeholders.\n\nstartFrame/durationFrames are project frames (from get_timeline). trackIndex is optional — omit to auto-create a new visual track; set it to target an existing non-audio track.", - // --- OpenTake Motion Canvas graphics (docs/MOTION-GRAPHICS-PLUGIN.md, Issue #34) --- - ToolName::AddMotionGraphic => "Adds a Motion Canvas-generated animation/video segment (animated title, explainer card, data callout, timeline insert, or transition card) to the timeline as a single undoable workflow, and returns its clipId. In v1, OpenTake asks the Motion Canvas plugin to render a materialized .mp4, imports that output as a normal media asset, then places it on the timeline. Preview and export therefore reuse the ordinary video pipeline.\n\nThe 'source' object is exactly one of:\n • { code: \"\" } — a self-contained Motion Canvas scene/project snippet. Prefer deterministic frame-driven animation, not wall-clock timers.\n • { templateId, params } — instantiate a registered Motion Canvas template by id with typed params (string/number/bool/color; colors are hex '#RRGGBB'/'#RRGGBBAA'). The template declares which params it accepts.\n\nstartFrame/durationFrames are project frames (from get_timeline). trackIndex is optional — omit to auto-create a new video track at the top for the generated segment; set it to target an existing non-audio track. transparent is accepted for forward compatibility, but v1 mp4 materialization is opaque; transparent overlays are a later PNG-sequence/native-motion path.", + ToolName::EditMotionGraphic => "Re-renders an existing OpenTake motion graphic as one durable undoable workflow while preserving its timeline clipId and placement. Pass the clipId and either replacement self-contained HTML/CSS/JS for a code-authored graphic or parameter overrides for a template-authored graphic. Ordinary video clips and unsupported source types are rejected with typed errors.", - ToolName::EditMotionGraphic => "Edits an existing Motion Canvas-generated clip and re-renders it as a single undoable workflow. Pass the clipId (from add_motion_graphic or get_timeline) and at least one of:\n • code — replace the Motion Canvas TS/TSX source of a code-authored graphic.\n • params — override template params (merged over the current bindings) of a template-authored graphic.\n\nThe clip must carry Motion Canvas metadata from add_motion_graphic; ordinary video clips are rejected. Re-rendering should update or replace the generated media asset and keep the timeline placement stable so later agent steps can keep using the same clip context.", + ToolName::TrackMotion => "Analyzes a bounded source region and returns editable position keyframes that follow the subject. Defaults to preview-only; set apply=true only after reviewing confidence and samples. Applying is one undoable edit. The tool is advertised only when a production tracking backend is available.", + ToolName::GenerateMatte => "Generates a frame-aligned reusable alpha matte for one clip without modifying the source asset. Defaults to preview-only and reports model/version/progress metadata. Applying the matte is one undoable edit. The tool is advertised only when an installed compatible model is available.", + ToolName::RemoveObject => "Produces a non-destructive derivative for the selected mask and frame range. Defaults to preview-only; provider costs require costAuthorized=true. Apply imports and swaps the reviewed derivative as one undoable workflow. Cancellation or failure leaves media and timeline unchanged.", + ToolName::MatchColor => "Analyzes a target clip and reference frame, then returns an editable ColorGrade plus deterministic comparison metrics. Defaults to preview-only; apply=true accepts the grade in one undoable edit. Source media and the previous grade remain recoverable.", + ToolName::SeparateStems => "Separates an audio-bearing asset into aligned vocals and accompaniment derivatives with source/model provenance. Optionally imports both stems to synchronized tracks in one undoable workflow. Cancellation or failure adds no media or tracks.", + ToolName::TranslateCaptions => "Translates selected caption clips while preserving every clip id and frame range. Defaults to a reviewable per-caption diff; apply=true accepts only the returned changes as one undoable edit. Provider costs require costAuthorized=true.", + ToolName::ScriptToVideo => "Builds and validates a persisted, reviewable multi-segment assembly plan from exact media and narration references. Defaults to planning only; apply=true places the reviewed segments and transitions through existing edit commands as one undoable workflow.", + ToolName::GenerateAvatar => "Generates a lip-synchronized avatar video from a portrait and narration through a configured provider. Requires explicit recorded consent and costAuthorized=true. Success imports the result; cancellation or failure imports nothing.", + ToolName::CloneVoice => "Enrolls, uses, or revokes a provider voice model. Every action requires a recorded consent id; paid enrollment/generation requires costAuthorized=true. Raw credentials and reference-audio bytes are never persisted in project metadata, and revoked voices cannot generate.", } } @@ -374,7 +386,8 @@ pub fn input_schema(tool: ToolName) -> Value { "endFrame": {"type": "integer", "description": "Optional project-frame window end (exclusive)."}, "minClipFrames": {"type": "integer", "description": "Optional lower bound for generated cut lengths."}, "maxClipFrames": {"type": "integer", "description": "Optional upper bound for generated cut lengths."}, - "alignCuts": {"type": "boolean", "description": "Optional. true means move/split cuts to the detected beat grid."} + "alignCuts": {"type": "boolean", "description": "Optional. true means align proposed cuts to the detected beat grid."}, + "write": {"type": "boolean", "description": "Optional, default false. true applies all selected clip placements and linked A/V partners in one atomic command."} }), &[], ), @@ -400,6 +413,16 @@ pub fn input_schema(tool: ToolName) -> Value { &[], ), + ToolName::RemoveFillerWords => object( + json!({ + "clipIds": {"type": "array", "items": {"type": "string"}, "description": "Optional spoken clip ids to transcribe and analyze."}, + "trackIndex": {"type": "integer", "description": "Optional spoken track index to analyze. Mutually exclusive with clipIds."}, + "fillerWords": {"type": "array", "items": {"type": "string"}, "description": "Optional exact filler lexicon. Multi-word phrases such as 'you know' are supported."}, + "paddingFrames": {"type": "integer", "minimum": 0, "description": "Optional context frames to preserve before and after each matched filler phrase."} + }), + &[], + ), + ToolName::GenerateVideo => object( json!({ "costAuthorized": {"type": "boolean", "description": "Required true. Set only after the user explicitly approves the paid generation request in this conversation."}, @@ -661,8 +684,8 @@ pub fn input_schema(tool: ToolName) -> Value { "items": { "type": "object", "properties": { - "name": {"type": "string", "description": "Effect identifier, e.g. 'gaussianBlur'."}, - "params": {"type": "object", "description": "Named numeric parameters for the effect, e.g. { \"radius\": 4 }.", "additionalProperties": {"type": "number"}}, + "name": {"type": "string", "enum": ["grayscale", "sepia", "invert"], "description": "Identifier from the closed rendered effect registry."}, + "params": {"type": "object", "description": "Optional effect strength; defaults to 1.", "properties": {"amount": {"type": "number", "minimum": 0, "maximum": 1}}, "additionalProperties": false}, "enabled": {"type": "boolean", "description": "Whether the effect is active (default true)."} }, "required": ["name"] @@ -677,16 +700,16 @@ pub fn input_schema(tool: ToolName) -> Value { json!({ "source": { "type": "object", - "description": "Exactly one of code or templateId must be set. code is Motion Canvas TS/TSX scene/project source; templateId instantiates a registered Motion Canvas template with params.", + "description": "Exactly one of code or templateId must be set. code is self-contained HTML/CSS/JS using OpenTake.onSeek; templateId selects a registered local template.", "properties": { - "code": {"type": "string", "description": "Motion Canvas TypeScript/TSX scene or project source. Prefer deterministic frame-driven animation, not wall-clock timers."}, - "templateId": {"type": "string", "description": "Registered Motion Canvas template id (e.g. 'lower-third.glass'). Mutually exclusive with code."}, + "code": {"type": "string", "description": "Self-contained HTML/CSS/JS document. Animate deterministically with OpenTake.onSeek; raw TS/TSX is not supported by this Beta renderer."}, + "templateId": {"type": "string", "enum": ["title-card", "lower-third.glass"], "description": "Registered local motion template. Mutually exclusive with code."}, "params": {"type": "object", "description": "Template params: name -> value. Values are string, number, bool, or a hex color string '#RRGGBB'/'#RRGGBBAA'. Only valid with templateId.", "additionalProperties": {"oneOf": [{"type": "string"}, {"type": "number"}, {"type": "boolean"}]}} } }, "startFrame": {"type": "integer", "description": "Timeline frame position to place the graphic (project frames)."}, "durationFrames": {"type": "integer", "description": "Clip length on the timeline, in project frames (>= 1)."}, - "transparent": {"type": "boolean", "description": "Forward-compatible alpha intent. v1 Motion Canvas mp4 materialization is opaque; transparent overlays are a later PNG-sequence/native-motion path."}, + "transparent": {"type": "boolean", "description": "Forward-compatible alpha intent. The current MP4 path rejects true with a typed unsupported-capability error."}, "trackIndex": {"type": "integer", "description": "Optional. Existing non-audio track index (0-based) to place the graphic on. Omit to auto-create a new video track at the top."} }), &["source", "startFrame", "durationFrames"], @@ -694,12 +717,99 @@ pub fn input_schema(tool: ToolName) -> Value { ToolName::EditMotionGraphic => object( json!({ - "clipId": {"type": "string", "description": "The Motion Canvas-generated clip id to edit (from add_motion_graphic or get_timeline)."}, - "code": {"type": "string", "description": "Replacement Motion Canvas TypeScript/TSX source for a code-authored graphic. Only valid when the clip was authored with code."}, - "params": {"type": "object", "description": "Template param overrides (merged over current bindings) for a template-authored Motion Canvas graphic. Values are string, number, bool, or a hex color string. Only valid when the clip was authored from a template.", "additionalProperties": {"oneOf": [{"type": "string"}, {"type": "number"}, {"type": "boolean"}]}} + "clipId": {"type": "string", "description": "The OpenTake motion clip id to edit (from add_motion_graphic or get_timeline)."}, + "code": {"type": "string", "description": "Replacement self-contained HTML/CSS/JS for a code-authored graphic."}, + "params": {"type": "object", "description": "Template parameter overrides merged over current bindings. Only valid for a template-authored graphic.", "additionalProperties": {"oneOf": [{"type": "string"}, {"type": "number"}, {"type": "boolean"}]}} }), &["clipId"], ), + + ToolName::TrackMotion => object( + json!({ + "clipId": {"type": "string"}, + "region": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}, "required": ["x", "y", "width", "height"]}, + "startFrame": {"type": "integer"}, "endFrame": {"type": "integer"}, + "apply": {"type": "boolean", "default": false} + }), + &["clipId", "region"], + ), + ToolName::GenerateMatte => object( + json!({ + "clipId": {"type": "string"}, "model": {"type": "string"}, + "startFrame": {"type": "integer"}, "endFrame": {"type": "integer"}, + "apply": {"type": "boolean", "default": false} + }), + &["clipId"], + ), + ToolName::RemoveObject => object( + json!({ + "clipId": {"type": "string"}, "maskId": {"type": "string"}, + "startFrame": {"type": "integer"}, "endFrame": {"type": "integer"}, + "provider": {"type": "string"}, "model": {"type": "string"}, + "costAuthorized": {"type": "boolean"}, "apply": {"type": "boolean", "default": false} + }), + &["clipId", "maskId"], + ), + ToolName::MatchColor => object( + json!({ + "clipId": {"type": "string"}, "referenceMediaRef": {"type": "string"}, + "referenceFrame": {"type": "integer"}, "targetFrame": {"type": "integer"}, + "apply": {"type": "boolean", "default": false} + }), + &["clipId", "referenceMediaRef"], + ), + ToolName::SeparateStems => object( + json!({ + "mediaRef": {"type": "string"}, "provider": {"type": "string"}, + "model": {"type": "string"}, "importToTracks": {"type": "boolean", "default": false}, + "startFrame": {"type": "integer"} + }), + &["mediaRef"], + ), + ToolName::TranslateCaptions => object( + json!({ + "captionClipIds": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "sourceLocale": {"type": "string"}, "targetLocale": {"type": "string"}, + "provider": {"type": "string"}, "model": {"type": "string"}, + "costAuthorized": {"type": "boolean"}, "apply": {"type": "boolean", "default": false} + }), + &["captionClipIds", "targetLocale"], + ), + ToolName::ScriptToVideo => object( + json!({ + "segments": {"type": "array", "minItems": 1, "items": {"type": "object", "properties": { + "script": {"type": "string"}, "mediaRef": {"type": "string"}, + "narrationMediaRef": {"type": "string"}, "durationFrames": {"type": "integer"}, + "transition": {"type": "string"} + }, "required": ["script", "mediaRef", "durationFrames"]}}, + "apply": {"type": "boolean", "default": false} + }), + &["segments"], + ), + ToolName::GenerateAvatar => object( + json!({ + "portraitMediaRef": {"type": "string"}, "audioMediaRef": {"type": "string"}, + "consentId": {"type": "string"}, "provider": {"type": "string"}, + "model": {"type": "string"}, "costAuthorized": {"type": "boolean"}, + "startFrame": {"type": "integer"} + }), + &[ + "portraitMediaRef", + "audioMediaRef", + "consentId", + "costAuthorized", + ], + ), + ToolName::CloneVoice => object( + json!({ + "action": {"type": "string", "enum": ["enroll", "generate", "revoke"]}, + "referenceAudioMediaRef": {"type": "string"}, "consentId": {"type": "string"}, + "voiceId": {"type": "string"}, "voiceName": {"type": "string"}, + "prompt": {"type": "string"}, "provider": {"type": "string"}, + "model": {"type": "string"}, "costAuthorized": {"type": "boolean"} + }), + &["action", "consentId"], + ), }; close_declared_objects(&mut schema); schema diff --git a/crates/opentake-agent/src/tools/names.rs b/crates/opentake-agent/src/tools/names.rs index 52c48d39..085d03d9 100644 --- a/crates/opentake-agent/src/tools/names.rs +++ b/crates/opentake-agent/src/tools/names.rs @@ -34,6 +34,7 @@ pub enum ToolName { AutoCutToBeats, SmartReframe, TightenSilences, + RemoveFillerWords, // --- Media generation / import (5) --- GenerateVideo, GenerateImage, @@ -60,9 +61,35 @@ pub enum ToolName { // --- OpenTake Motion Canvas graphics (docs/MOTION-GRAPHICS-PLUGIN.md, Issue #34) --- AddMotionGraphic, EditMotionGraphic, + // --- Advanced AI workflows (capability-gated by the desktop host) --- + TrackMotion, + GenerateMatte, + RemoveObject, + MatchColor, + SeparateStems, + TranslateCaptions, + ScriptToVideo, + GenerateAvatar, + CloneVoice, } impl ToolName { + /// Whether discovery of this tool requires a live host media bridge. + /// Keeping this predicate next to the catalog prevents MCP and in-app Chat + /// from drifting into different fail-closed capability sets. + pub const fn requires_media_bridge(self) -> bool { + matches!( + self, + ToolName::InspectMedia + | ToolName::GetTranscript + | ToolName::InspectTimeline + | ToolName::SearchMedia + | ToolName::AddCaptions + | ToolName::RemoveFillerWords + | ToolName::ImportMedia + ) + } + /// The wire name (matches upstream / spec exactly). pub fn as_str(self) -> &'static str { match self { @@ -89,6 +116,7 @@ impl ToolName { ToolName::AutoCutToBeats => "auto_cut_to_beats", ToolName::SmartReframe => "smart_reframe", ToolName::TightenSilences => "tighten_silences", + ToolName::RemoveFillerWords => "remove_filler_words", ToolName::GenerateVideo => "generate_video", ToolName::GenerateImage => "generate_image", ToolName::GenerateAudio => "generate_audio", @@ -110,13 +138,22 @@ impl ToolName { ToolName::ApplyEffect => "apply_effect", ToolName::AddMotionGraphic => "add_motion_graphic", ToolName::EditMotionGraphic => "edit_motion_graphic", + ToolName::TrackMotion => "track_motion", + ToolName::GenerateMatte => "generate_matte", + ToolName::RemoveObject => "remove_object", + ToolName::MatchColor => "match_color", + ToolName::SeparateStems => "separate_stems", + ToolName::TranslateCaptions => "translate_captions", + ToolName::ScriptToVideo => "script_to_video", + ToolName::GenerateAvatar => "generate_avatar", + ToolName::CloneVoice => "clone_voice", } } - /// Tools advertised to MCP and in-app Chat in registration order. Provider- - /// backed generation and Motion Canvas tools remain known wire names, but - /// stay out of discovery until their production backends are connected. - pub const ALL: [ToolName; 38] = [ + /// Base tools advertised to MCP and in-app Chat in registration order. + /// Provider-backed generation and Motion tools are appended only when the + /// current host reports their respective live capabilities. + pub const ALL: [ToolName; 39] = [ ToolName::GetTimeline, ToolName::GetMedia, ToolName::InspectMedia, @@ -140,6 +177,7 @@ impl ToolName { ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ToolName::ImportMedia, ToolName::ListFolders, ToolName::CreateFolder, @@ -166,11 +204,31 @@ impl ToolName { ToolName::UpscaleMedia, ]; + /// Motion tools appended only by a host with a production render/import/ + /// placement bridge. They remain known for strict compatibility parsing in + /// all other hosts. + pub const MOTION: [ToolName; 2] = [ToolName::AddMotionGraphic, ToolName::EditMotionGraphic]; + + /// Advanced workflows are schema-known but never unconditionally + /// advertised. The desktop host appends only the exact capabilities backed + /// by installed local models or a configured provider. + pub const ADVANCED_AI: [ToolName; 9] = [ + ToolName::TrackMotion, + ToolName::GenerateMatte, + ToolName::RemoveObject, + ToolName::MatchColor, + ToolName::SeparateStems, + ToolName::TranslateCaptions, + ToolName::ScriptToVideo, + ToolName::GenerateAvatar, + ToolName::CloneVoice, + ]; + /// Every recognized schema/wire name, including capabilities deliberately /// hidden from discovery until a real backend exists. Keeping this set lets /// strict argument validation and compatibility tests cover future tools /// without advertising placeholder behavior to models. - pub const KNOWN: [ToolName; 44] = [ + pub const KNOWN: [ToolName; 54] = [ ToolName::GetTimeline, ToolName::GetMedia, ToolName::InspectMedia, @@ -194,6 +252,7 @@ impl ToolName { ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ToolName::GenerateVideo, ToolName::GenerateImage, ToolName::GenerateAudio, @@ -215,6 +274,15 @@ impl ToolName { ToolName::ApplyEffect, ToolName::AddMotionGraphic, ToolName::EditMotionGraphic, + ToolName::TrackMotion, + ToolName::GenerateMatte, + ToolName::RemoveObject, + ToolName::MatchColor, + ToolName::SeparateStems, + ToolName::TranslateCaptions, + ToolName::ScriptToVideo, + ToolName::GenerateAvatar, + ToolName::CloneVoice, ]; /// The 31 upstream-equivalent tools (Issue #9's "31 tools"). @@ -274,25 +342,37 @@ mod tests { } #[test] - fn advertised_set_is_38_and_known_set_is_44() { - assert_eq!(ToolName::ALL.len(), 38); - assert_eq!(ToolName::KNOWN.len(), 44); + fn advertised_set_is_39_and_known_set_is_54() { + assert_eq!(ToolName::ALL.len(), 39); + assert_eq!(ToolName::KNOWN.len(), 54); assert!(ToolName::ALL .iter() .all(|tool| ToolName::KNOWN.contains(tool))); } + #[test] + fn advanced_ai_tools_are_known_but_capability_gated() { + for tool in ToolName::ADVANCED_AI { + assert_eq!(ToolName::from_str(tool.as_str()), Ok(tool)); + assert!(ToolName::KNOWN.contains(&tool)); + assert!(!ToolName::ALL.contains(&tool)); + assert!(!ToolName::UPSTREAM.contains(&tool)); + } + } + #[test] fn analysis_tools_have_expected_wire_names() { assert_eq!(ToolName::DetectBeats.as_str(), "detect_beats"); assert_eq!(ToolName::AutoCutToBeats.as_str(), "auto_cut_to_beats"); assert_eq!(ToolName::SmartReframe.as_str(), "smart_reframe"); assert_eq!(ToolName::TightenSilences.as_str(), "tighten_silences"); + assert_eq!(ToolName::RemoveFillerWords.as_str(), "remove_filler_words"); for t in [ ToolName::DetectBeats, ToolName::AutoCutToBeats, ToolName::SmartReframe, ToolName::TightenSilences, + ToolName::RemoveFillerWords, ] { assert_eq!(ToolName::from_str(t.as_str()), Ok(t)); assert!(!ToolName::UPSTREAM.contains(&t)); @@ -307,8 +387,8 @@ mod tests { for t in [ToolName::AddMotionGraphic, ToolName::EditMotionGraphic] { assert_eq!(ToolName::from_str(t.as_str()), Ok(t)); } - // They remain known for schema compatibility, but are not advertised - // until the production Motion Canvas renderer is wired. + // They stay out of the unconditional base catalog: a capable desktop + // host appends MOTION, while non-rendering hosts remain fail-closed. assert_eq!( ToolName::KNOWN .iter() diff --git a/crates/opentake-agent/src/tools/result.rs b/crates/opentake-agent/src/tools/result.rs index c5d647ee..c9c5f20c 100644 --- a/crates/opentake-agent/src/tools/result.rs +++ b/crates/opentake-agent/src/tools/result.rs @@ -53,6 +53,7 @@ pub(crate) enum PublicErrorKind { InvalidArguments(ToolName), ResourceNotFound(ToolName), CapabilityUnavailable(ToolName), + AnalysisLowConfidence(ToolName), } impl PublicErrorKind { @@ -62,6 +63,7 @@ impl PublicErrorKind { Self::InvalidArguments(_) => "MCP_INVALID_ARGUMENTS", Self::ResourceNotFound(_) => "MCP_RESOURCE_NOT_FOUND", Self::CapabilityUnavailable(_) => "MCP_CAPABILITY_UNAVAILABLE", + Self::AnalysisLowConfidence(_) => "MCP_ANALYSIS_LOW_CONFIDENCE", } } @@ -73,6 +75,9 @@ impl PublicErrorKind { Self::CapabilityUnavailable(_) => { "This capability is unavailable for the referenced media." } + Self::AnalysisLowConfidence(_) => { + "The analysis could not identify the requested subject reliably." + } } } @@ -86,6 +91,9 @@ impl PublicErrorKind { Self::CapabilityUnavailable(_) => { "Use a supported source type or restore the source media, then retry." } + Self::AnalysisLowConfidence(_) => { + "Choose a tighter, higher-contrast subject region and retry." + } } } } diff --git a/crates/opentake-agent/tests/advanced_ai_workflows.rs b/crates/opentake-agent/tests/advanced_ai_workflows.rs new file mode 100644 index 00000000..cc6eb91d --- /dev/null +++ b/crates/opentake-agent/tests/advanced_ai_workflows.rs @@ -0,0 +1,157 @@ +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use opentake_agent::mcp::advanced::{ + AdvancedWorkflowBridge, AdvancedWorkflowCommit, AdvancedWorkflowError, AdvancedWorkflowRequest, +}; +use opentake_agent::mcp::core_handle::CoreHandle; +use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::plugin::registry::PluginRegistry; +use opentake_agent::tools::names::ToolName; +use opentake_domain::{MediaManifest, Timeline}; +use opentake_ops::{EditCommand, EditResult}; +use serde_json::{json, Value}; + +struct ReadOnlyHandle; + +impl CoreHandle for ReadOnlyHandle { + fn timeline(&self) -> Timeline { + Timeline::new() + } + + fn media(&self) -> MediaManifest { + MediaManifest::new() + } + + fn apply(&self, _cmd: EditCommand) -> anyhow::Result { + anyhow::bail!("advanced workflow fixture is read-only") + } + + fn project_dir(&self) -> Option { + None + } +} + +struct DeterministicAdvancedBridge; + +impl AdvancedWorkflowBridge for DeterministicAdvancedBridge { + fn supported_tools(&self) -> Vec { + ToolName::ADVANCED_AI.to_vec() + } + + fn execute( + &self, + request: AdvancedWorkflowRequest, + _cancel: &opentake_media::MediaCancelToken, + ) -> Result { + Ok(AdvancedWorkflowCommit { + result: json!({"tool": request.tool().as_str(), "status": "completed"}), + action_name: None, + }) + } +} + +fn dispatcher(advanced: bool) -> Dispatcher { + Dispatcher::with_all_capability_bridges( + Arc::new(ReadOnlyHandle), + Arc::new(RwLock::new(PluginRegistry::new())), + None, + None, + None, + advanced.then(|| Arc::new(DeterministicAdvancedBridge) as Arc), + ) +} + +fn cases() -> [(ToolName, Value); 9] { + [ + ( + ToolName::TrackMotion, + json!({"clipId":"clip","region":{"x":0.1,"y":0.1,"width":0.2,"height":0.2}}), + ), + (ToolName::GenerateMatte, json!({"clipId":"clip"})), + ( + ToolName::RemoveObject, + json!({"clipId":"clip","maskId":"mask"}), + ), + ( + ToolName::MatchColor, + json!({"clipId":"clip","referenceMediaRef":"asset"}), + ), + (ToolName::SeparateStems, json!({"mediaRef":"asset"})), + ( + ToolName::TranslateCaptions, + json!({"captionClipIds":["caption"],"targetLocale":"zh-CN"}), + ), + ( + ToolName::ScriptToVideo, + json!({"segments":[{"script":"intro","mediaRef":"asset","durationFrames":30}]}), + ), + ( + ToolName::GenerateAvatar, + json!({"portraitMediaRef":"portrait","audioMediaRef":"audio","consentId":"consent","costAuthorized":true}), + ), + ( + ToolName::CloneVoice, + json!({"action":"revoke","consentId":"consent","voiceId":"voice"}), + ), + ] +} + +#[test] +fn advanced_ai_workflows_are_hidden_without_a_live_host() { + let dispatcher = dispatcher(false); + for (tool, args) in cases() { + assert!(!dispatcher.advertised_tools().contains(&tool)); + let result = dispatcher.dispatch(tool.as_str(), args); + assert!(result.is_error); + assert!(result.text_joined().contains("not advertised")); + } +} + +#[test] +fn advanced_ai_workflows_route_through_exact_tool_contracts() { + let dispatcher = dispatcher(true); + for (tool, args) in cases() { + assert!(ToolName::KNOWN.contains(&tool)); + assert!(dispatcher.advertised_tools().contains(&tool)); + let result = dispatcher.dispatch(tool.as_str(), args); + assert!( + !result.is_error, + "{}: {}", + tool.as_str(), + result.text_joined() + ); + assert!(result.text_joined().contains(tool.as_str())); + assert!(result.text_joined().contains("completed")); + } + + for tool in [ToolName::TightenSilences, ToolName::RemoveFillerWords] { + assert!(ToolName::ALL.contains(&tool)); + } +} + +#[test] +fn advanced_nested_contracts_reject_unknown_fields_before_host_execution() { + let dispatcher = dispatcher(true); + let result = dispatcher.dispatch( + "track_motion", + json!({"clipId":"clip","region":{"x":0.0,"y":0.0,"width":1.0,"height":1.0,"secret":true}}), + ); + assert!(result.is_error); + assert!( + result.text_joined().contains("region:") && result.text_joined().contains("'secret'"), + "{}", + result.text_joined() + ); + + let result = dispatcher.dispatch( + "script_to_video", + json!({"segments":[{"script":"x","mediaRef":"asset","durationFrames":30,"secret":true}]}), + ); + assert!(result.is_error); + assert!( + result.text_joined().contains("segments[0]:") && result.text_joined().contains("'secret'"), + "{}", + result.text_joined() + ); +} diff --git a/crates/opentake-agent/tests/advertised_tool_acceptance.rs b/crates/opentake-agent/tests/advertised_tool_acceptance.rs index 19526565..703efc25 100644 --- a/crates/opentake-agent/tests/advertised_tool_acceptance.rs +++ b/crates/opentake-agent/tests/advertised_tool_acceptance.rs @@ -3,6 +3,10 @@ use std::sync::{Arc, RwLock}; use opentake_agent::mcp::core_handle::CoreHandle; use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::mcp::motion::{ + AddMotionRequest, EditMotionRequest, MotionBridge, MotionBridgeError, MotionCommit, + MotionOutputMetadata, +}; use opentake_agent::plugin::registry::PluginRegistry; use opentake_agent::tools::names::ToolName; use opentake_domain::{MediaManifest, Timeline}; @@ -10,6 +14,56 @@ use opentake_ops::{EditCommand, EditResult}; struct ReadOnlyHandle; +struct DeterministicMotionBridge; + +fn output_metadata(content_hash: &str) -> MotionOutputMetadata { + MotionOutputMetadata { + renderer: "fixture".into(), + renderer_version: "1".into(), + output_file: "output.mp4".into(), + fps: 30.0, + width: 64, + height: 36, + duration_frames: 30, + duration_seconds: 1.0, + content_hash: content_hash.into(), + } +} + +impl MotionBridge for DeterministicMotionBridge { + fn can_render_motion(&self) -> bool { + true + } + + fn add( + &self, + _request: AddMotionRequest, + _cancel: &opentake_media::MediaCancelToken, + ) -> Result { + Ok(MotionCommit { + clip_id: "motion-clip".into(), + asset_id: "motion-asset".into(), + content_hash: "add-hash".into(), + action_name: "Add Motion Graphic".into(), + output: output_metadata("add-hash"), + }) + } + + fn edit( + &self, + request: EditMotionRequest, + _cancel: &opentake_media::MediaCancelToken, + ) -> Result { + Ok(MotionCommit { + clip_id: request.clip_id, + asset_id: "edited-motion-asset".into(), + content_hash: "edit-hash".into(), + action_name: "Edit Motion Graphic".into(), + output: output_metadata("edit-hash"), + }) + } +} + impl CoreHandle for ReadOnlyHandle { fn timeline(&self) -> Timeline { Timeline::new() @@ -30,9 +84,12 @@ impl CoreHandle for ReadOnlyHandle { #[test] fn every_advertised_tool_is_live_or_absent() { - let dispatcher = Dispatcher::new( + let dispatcher = Dispatcher::with_capability_bridges( Arc::new(ReadOnlyHandle), Arc::new(RwLock::new(PluginRegistry::new())), + None, + None, + Some(Arc::new(DeterministicMotionBridge)), ); let cases = [ ( @@ -68,9 +125,10 @@ fn every_advertised_tool_is_live_or_absent() { serde_json::json!({"clipId": "clip", "code": "export default {}"}), ), ]; + let advertised = dispatcher.advertised_tools(); for (tool, args) in cases { - if !ToolName::ALL.contains(&tool) { + if !advertised.contains(&tool) { let result = dispatcher.dispatch(tool.as_str(), args); assert!( result.text_joined().contains("not advertised"), @@ -87,5 +145,23 @@ fn every_advertised_tool_is_live_or_absent() { tool.as_str(), result.text_joined() ); + assert!( + !result.text_joined().contains("not advertised"), + "{} was advertised but dispatch rejected it: {}", + tool.as_str(), + result.text_joined() + ); + } +} + +#[test] +fn motion_tools_are_absent_without_a_live_host_bridge() { + let dispatcher = Dispatcher::new( + Arc::new(ReadOnlyHandle), + Arc::new(RwLock::new(PluginRegistry::new())), + ); + + for tool in ToolName::MOTION { + assert!(!dispatcher.advertised_tools().contains(&tool)); } } diff --git a/crates/opentake-agent/tests/completion_43312d5e9f613913.rs b/crates/opentake-agent/tests/completion_43312d5e9f613913.rs new file mode 100644 index 00000000..32653bbd --- /dev/null +++ b/crates/opentake-agent/tests/completion_43312d5e9f613913.rs @@ -0,0 +1,76 @@ +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use opentake_agent::mcp::core_handle::CoreHandle; +use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::plugin::registry::PluginRegistry; +use opentake_domain::{Clip, ClipType, MediaManifest, Timeline, Track}; +use opentake_ops::{apply as ops_apply, EditCommand, EditResult, EditorState, SeqIdGen}; + +struct RecordingCore { + state: Mutex, + apply_calls: AtomicUsize, +} + +impl RecordingCore { + fn new() -> Self { + let mut timeline = Timeline::new(); + let mut track = Track::new("video-track", ClipType::Video); + track.clips.push(Clip::new("clip-a", "asset-a", 0, 30)); + timeline.tracks.push(track); + Self { + state: Mutex::new(EditorState::new(timeline, MediaManifest::new())), + apply_calls: AtomicUsize::new(0), + } + } +} + +impl CoreHandle for RecordingCore { + fn timeline(&self) -> Timeline { + self.state.lock().expect("state lock").timeline.clone() + } + + fn media(&self) -> MediaManifest { + self.state.lock().expect("state lock").manifest.clone() + } + + fn apply(&self, command: EditCommand) -> anyhow::Result { + self.apply_calls.fetch_add(1, Ordering::SeqCst); + ops_apply( + &mut self.state.lock().expect("state lock"), + command, + &SeqIdGen::new("contract-"), + ) + .map_err(|error| anyhow::anyhow!(error.to_string())) + } + + fn project_dir(&self) -> Option { + None + } +} + +#[test] +fn completion_43312d5e9f613913_tauri_exposes_typed_core_edit_commands_with_stab() { + let core = Arc::new(RecordingCore::new()); + let dispatcher = Dispatcher::new(core.clone(), Arc::new(RwLock::new(PluginRegistry::new()))); + let before = core.timeline(); + + let malformed = dispatcher.dispatch( + "remove_clips", + serde_json::json!({"clipIds": "not-an-array", "hostPath": "/private/secret"}), + ); + assert!(malformed.is_error); + assert_eq!(core.apply_calls.load(Ordering::SeqCst), 0); + assert_eq!(core.timeline(), before); + assert!(!malformed.text_joined().contains("/private/secret")); + + let valid = dispatcher.dispatch("remove_clips", serde_json::json!({"clipIds": ["clip-a"]})); + assert!(!valid.is_error, "{}", valid.text_joined()); + assert_eq!(core.apply_calls.load(Ordering::SeqCst), 1); + assert!(core + .timeline() + .tracks + .iter() + .all(|track| track.clips.is_empty())); +} diff --git a/crates/opentake-agent/tests/editing_automation_acceptance.rs b/crates/opentake-agent/tests/editing_automation_acceptance.rs new file mode 100644 index 00000000..e3afc7aa --- /dev/null +++ b/crates/opentake-agent/tests/editing_automation_acceptance.rs @@ -0,0 +1,236 @@ +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use opentake_agent::mcp::core_handle::CoreHandle; +use opentake_agent::mcp::dispatch::Dispatcher; +use opentake_agent::plugin::registry::PluginRegistry; +use opentake_domain::{ + Clip, ClipType, Crop, MediaManifest, MediaManifestEntry, MediaSource, Timeline, Track, +}; +use opentake_media::analysis::{ + detect_autocrop, detect_beats, AutocropConfig, BeatDetectionConfig, FrameBuffer, PixelFormat, +}; +use opentake_media::{PcmBuffer, PcmFormat, PcmSpec}; +use opentake_ops::intent::{plan_beat_sync_placement, plan_smart_reframe, IntentClipEntry}; +use opentake_ops::{apply as ops_apply, EditCommand, EditResult, EditorState, SeqIdGen}; + +struct AutomationHandle { + state: Mutex, + pcm: PcmBuffer, + apply_calls: AtomicUsize, +} + +impl AutomationHandle { + fn new() -> Self { + let mut timeline = Timeline::new(); + timeline.fps = 10; + let mut track = Track::new("video-track", ClipType::Video); + track.clips.push(Clip::new("clip-a", "asset-1", 0, 10)); + timeline.tracks.push(track); + + let mut manifest = MediaManifest::new(); + manifest.entries.push(MediaManifestEntry { + id: "asset-1".into(), + name: "Source.mov".into(), + kind: ClipType::Video, + source: MediaSource::External { + absolute_path: "/fixture/Source.mov".into(), + }, + duration: 1.0, + generation_input: None, + source_width: Some(1920), + source_height: Some(1080), + source_fps: Some(10.0), + has_audio: Some(true), + color: None, + proxy: None, + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + }); + + let mut samples = vec![0.5; 300]; + samples.extend(std::iter::repeat_n(0.0, 400)); + samples.extend(std::iter::repeat_n(0.5, 300)); + Self { + state: Mutex::new(EditorState::new(timeline, manifest)), + pcm: PcmBuffer { + spec: PcmSpec { + sample_rate: 1_000, + channels: 1, + format: PcmFormat::F32, + }, + samples_f32: samples, + }, + apply_calls: AtomicUsize::new(0), + } + } +} + +impl CoreHandle for AutomationHandle { + fn timeline(&self) -> Timeline { + self.state.lock().expect("state lock").timeline.clone() + } + + fn media(&self) -> MediaManifest { + self.state.lock().expect("state lock").manifest.clone() + } + + fn apply(&self, command: EditCommand) -> anyhow::Result { + self.apply_calls.fetch_add(1, Ordering::AcqRel); + let ids = SeqIdGen::new("automation-"); + ops_apply(&mut self.state.lock().expect("state lock"), command, &ids) + .map_err(|error| anyhow::anyhow!(error.to_string())) + } + + fn project_dir(&self) -> Option { + None + } + + fn extract_analysis_pcm( + &self, + _media_ref: &str, + _spec: PcmSpec, + _range: Option<(f64, f64)>, + ) -> anyhow::Result { + Ok(self.pcm.clone()) + } +} + +fn first_json(result: &opentake_agent::tools::result::ToolResult) -> serde_json::Value { + let text = match &result.content[0] { + opentake_agent::tools::result::Block::Text { text } => text, + other => panic!("expected JSON text block, got {other:?}"), + }; + serde_json::from_str(text).expect("valid tool JSON") +} + +#[test] +fn automation_children_are_atomic_reviewable_and_command_routed() { + // The media children are deterministic and reject malformed input without + // producing a partial proposal. + let beat_config = BeatDetectionConfig { + sample_rate: 1_000, + fps: 10.0, + window_size_samples: 100, + hop_size_samples: 100, + min_onset_strength: 0.05, + min_gap_frames: 1, + }; + let mut pulse = vec![0.0; 1_000]; + pulse[500..530].fill(1.0); + assert_eq!( + detect_beats(&pulse, beat_config), + detect_beats(&pulse, beat_config) + ); + assert!(detect_beats( + &pulse, + BeatDetectionConfig { + sample_rate: 0, + ..beat_config + } + ) + .is_empty()); + + let pixels = [0_u8, 0, 0, 255, 255, 255]; + let valid_frame = FrameBuffer { + width: 2, + height: 1, + data: &pixels, + pixel_format: PixelFormat::Rgb, + }; + assert_eq!( + detect_autocrop(&valid_frame, AutocropConfig::default()), + detect_autocrop(&valid_frame, AutocropConfig::default()) + ); + let truncated = FrameBuffer { + data: &pixels[..3], + ..valid_frame + }; + assert_eq!(detect_autocrop(&truncated, AutocropConfig::default()), None); + + let handle = Arc::new(AutomationHandle::new()); + let dispatcher = Dispatcher::new(handle.clone(), Arc::new(RwLock::new(PluginRegistry::new()))); + let before = handle.timeline(); + + // Analysis surfaces return reviewable proposals or typed diagnostics. They + // never reach the edit boundary themselves. + let beats = dispatcher.dispatch("detect_beats", serde_json::json!({"mediaRef": "asset-1"})); + assert!(!beats.is_error, "{}", beats.text_joined()); + assert_eq!(first_json(&beats)["applied"], false); + + let silences = dispatcher.dispatch( + "tighten_silences", + serde_json::json!({ + "clipIds": ["clip-a"], + "thresholdDb": -40.0, + "minSilenceFrames": 2, + "paddingFrames": 0 + }), + ); + assert!(!silences.is_error, "{}", silences.text_joined()); + let silence_json = first_json(&silences); + assert_eq!(silence_json["applied"], false); + assert_eq!(silence_json["commands"][0]["tool"], "ripple_delete_ranges"); + + let unavailable = dispatcher.dispatch( + "smart_reframe", + serde_json::json!({"clipIds": ["clip-a"], "aspectRatio": "9:16"}), + ); + assert!(unavailable.is_error); + assert!(unavailable.text_joined().contains("needs vision")); + assert_eq!(handle.apply_calls.load(Ordering::Acquire), 0); + assert_eq!(handle.timeline(), before); + + // A valid write is normalized to exactly one existing EditCommand. The + // command boundary owns the mutation and its single undo restores the exact + // prior timeline. + let crop = Crop { + left: 0.1, + top: 0.0, + right: 0.1, + bottom: 0.0, + }; + let plan = plan_smart_reframe(&["clip-a".into()], crop, None).expect("reframe plan"); + assert_eq!(plan.label, "smart_reframe"); + assert_eq!(plan.commands.len(), 1); + let applied = handle + .apply(plan.commands[0].clone()) + .expect("atomic command"); + assert!(applied.changed); + assert_eq!(handle.apply_calls.load(Ordering::Acquire), 1); + assert_eq!(handle.timeline().tracks[0].clips[0].crop, crop); + handle.apply(EditCommand::Undo).expect("single undo"); + assert_eq!(handle.timeline(), before); + + // Rejected plans are typed, remain command-free, and preserve state. + let rejected = plan_smart_reframe(&[], crop, None).expect_err("empty clip ids must fail"); + assert!(rejected.to_string().contains("empty clipIds")); + assert_eq!(handle.timeline(), before); + + let entry = IntentClipEntry { + media_ref: "asset-1".into(), + media_type: ClipType::Video, + source_clip_type: ClipType::Video, + track_index: None, + start_frame: 0, + duration_frames: 5, + trim_start_frame: None, + trim_end_frame: None, + has_audio: true, + add_linked_audio: true, + transform: None, + }; + let beat_plan = + plan_beat_sync_placement(&before, vec![entry.clone()], &[3]).expect("beat placement plan"); + assert_eq!(beat_plan.commands.len(), 1); + assert!(matches!( + beat_plan.commands[0], + EditCommand::AddClipsAutoTrack { .. } + )); + let bad_beats = + plan_beat_sync_placement(&before, vec![entry], &[]).expect_err("missing beat must fail"); + assert!(bad_beats.to_string().contains("Need at least 1 beat")); + assert_eq!(handle.timeline(), before); +} diff --git a/crates/opentake-agent/tests/tool_argument_contract.rs b/crates/opentake-agent/tests/tool_argument_contract.rs index 96c54b61..cfd4fec1 100644 --- a/crates/opentake-agent/tests/tool_argument_contract.rs +++ b/crates/opentake-agent/tests/tool_argument_contract.rs @@ -48,7 +48,7 @@ fn dispatcher() -> (Dispatcher, Arc) { } #[test] -fn all_tool_schemas_reject_unknown_missing_wrong_type_and_nonfinite() { +fn all_tool_schemas_reject_unknown_missing_wrong_type() { let (dispatcher, apply_calls) = dispatcher(); for tool in ToolName::ALL { diff --git a/crates/opentake-core/src/core.rs b/crates/opentake-core/src/core.rs index cfe4f0b7..74f28689 100644 --- a/crates/opentake-core/src/core.rs +++ b/crates/opentake-core/src/core.rs @@ -31,8 +31,10 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard}; -use opentake_domain::{MediaManifest, MediaManifestEntry, Timeline}; -use opentake_ops::command::{EditCommand, EditResult}; +use opentake_domain::{ + ClipType, GenerationInput, MediaAsset, MediaManifest, MediaManifestEntry, MediaProxy, Timeline, +}; +use opentake_ops::command::{ClipEntry, EditCommand, EditResult}; use opentake_ops::IdGen; use opentake_project::{GenerationLog, ProjectCompatibility}; use same_file::Handle; @@ -41,8 +43,8 @@ use crate::deps::CoreDeps; use crate::error::{CoreError, Result}; use crate::events::{CoreEvent, EventBus, SubscriptionId}; use crate::session::{ - EditorSession, GenerationJobCommit, GenerationStateUpdate, PreparedGenerationJob, - PreparedGenerationOutput, ProbedMedia, + DerivedStemProvenance, EditorSession, GenerationJobCommit, GenerationStateUpdate, + PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, }; type ProjectIdentityTransitionListener = Arc; @@ -126,6 +128,31 @@ pub struct ProjectRuntimeSnapshot { pub version: u64, } +/// Placement half of a project-managed motion render commit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MotionPlacement { + Add { + start_frame: i32, + duration_frames: i32, + track_index: Option, + }, + Replace { + clip_id: String, + }, + /// Replace a clip with a derivative that already contains the result of + /// its masks, then clear those editable masks in the same undo snapshot. + ReplaceAndClearMasks { + clip_id: String, + }, +} + +/// Result of atomically registering a rendered video and placing/replacing it. +#[derive(Clone, Debug)] +pub struct MotionMediaCommit { + pub media: MediaManifestEntry, + pub edit: EditResult, +} + /// A folder target in a prepared media-import plan. Planned folders are keyed /// by the scanner without consuming application ids; existing folders retain /// their authoritative project id. @@ -150,6 +177,12 @@ pub enum PreparedMediaImportOp { probe: ProbedMedia, folder: Option, }, + ImportDerivedStem { + path: PathBuf, + name: String, + probe: ProbedMedia, + provenance: DerivedStemProvenance, + }, } /// One file admitted by a successful durable batch import. @@ -509,6 +542,61 @@ impl AppCore { self.apply_with_revision(command, Some(expected)) } + /// Apply one revision-bound edit and durably save the project under the + /// same session lock. Persistence failure restores document, history, and + /// version exactly before returning. + pub fn apply_at_revision_persisted( + &self, + expected: ProjectRevision, + command: EditCommand, + ) -> Result { + let (result, project_epoch, media_count, written) = { + let mut session = self.lock(); + if session.project_epoch != expected.project_epoch + || session.editor.version() != expected.version + { + return Err(CoreError::Media( + "project changed while preparing a deferred edit".to_string(), + )); + } + let before = session.editor.checkpoint_editor_state(); + let outcome = (|| { + let result = session.editor.apply(command, self.ids.as_ref())?; + let media_count = result + .manifest_changed + .then(|| session.editor.media().entries.len()); + let written = session.editor.save_project(None)?; + Ok((result, media_count, written)) + })(); + match outcome { + Ok((result, media_count, written)) => { + (result, session.project_epoch, media_count, written) + } + Err(error) => { + session.editor.restore_editor_state(before); + return Err(error); + } + } + }; + if result.changed { + self.events.emit(&CoreEvent::TimelineChanged { + project_epoch, + version: result.timeline_version, + }); + } + if let Some(count) = media_count { + self.events.emit(&CoreEvent::MediaChanged { + project_epoch, + count, + }); + } + self.events.emit(&CoreEvent::ProjectSaved { + path: written.to_string_lossy().into_owned(), + project_epoch, + }); + Ok(result) + } + fn apply_with_revision( &self, command: EditCommand, @@ -922,6 +1010,178 @@ impl AppCore { Ok(entry) } + /// Atomically register a completed project-managed motion render and place + /// or replace its timeline clip. + #[allow(clippy::too_many_arguments)] + pub fn commit_motion_media_for_project( + &self, + expected_project_epoch: u64, + expected_version: u64, + expected_project_dir: &Path, + path: impl AsRef, + name: impl Into, + probe: &ProbedMedia, + provenance: GenerationInput, + placement: MotionPlacement, + ) -> Result { + self.commit_generated_media_for_project( + expected_project_epoch, + expected_version, + expected_project_dir, + path, + name, + ClipType::Video, + probe, + provenance, + placement, + "Add Motion Graphic", + ) + } + + /// Atomically register a completed generated audio/video file and place or + /// replace its timeline clip. The generated file must already be a + /// regular, non-symlink child of the active bundle's `media/` directory. + /// The document command and project save share the session lock; any command + /// or persistence failure restores timeline, manifest, undo/redo, and + /// version exactly. A stale document version is rejected under that same + /// lock. Events are emitted only after the durable save succeeds. + #[allow(clippy::too_many_arguments)] + pub fn commit_generated_media_for_project( + &self, + expected_project_epoch: u64, + expected_version: u64, + expected_project_dir: &Path, + path: impl AsRef, + name: impl Into, + kind: ClipType, + probe: &ProbedMedia, + provenance: GenerationInput, + placement: MotionPlacement, + action_name: &str, + ) -> Result { + let path = path.as_ref(); + let media_dir = expected_project_dir.join(opentake_project::layout::MEDIA_DIR); + if path.parent() != Some(media_dir.as_path()) + || path.file_name().is_none() + || path.components().any(|component| { + matches!( + component, + std::path::Component::ParentDir | std::path::Component::CurDir + ) + }) + { + return Err(CoreError::Media( + "generated output must be one direct child of the active project media directory" + .into(), + )); + } + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| CoreError::Media(format!("motion output metadata failed: {error}")))?; + if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { + return Err(CoreError::Media( + "generated output must be a regular non-symlink file".into(), + )); + } + + let (commit, count, written) = { + let mut session = self.lock(); + ensure_project_identity(&session, expected_project_epoch, expected_project_dir)?; + if session.editor.version() != expected_version { + return Err(CoreError::Media( + "project changed while preparing a generated-media edit".into(), + )); + } + session.editor.ensure_mutable()?; + let before = session.editor.checkpoint_editor_state(); + let id = loop { + let candidate = self.ids.next_id(); + if session.editor.media_entry(&candidate).is_none() { + break candidate; + } + }; + + let result = (|| { + if !matches!(kind, ClipType::Audio | ClipType::Video) { + return Err(CoreError::Media( + "generated placement supports audio or video".into(), + )); + } + let mut asset = MediaAsset::new(id, path, kind, name, probe.duration_secs); + asset.source_width = probe.width; + asset.source_height = probe.height; + asset.source_fps = probe.fps; + asset.color = probe.color.clone(); + asset.has_audio = probe.has_audio; + asset.generation_input = Some(provenance); + let media = asset.to_manifest_entry(Some(expected_project_dir), 0.0); + + let command = match placement { + MotionPlacement::Add { + start_frame, + duration_frames, + track_index, + } => EditCommand::RegisterMediaAndAddClip { + entry: ClipEntry { + media_ref: media.id.clone(), + media_type: kind, + source_clip_type: kind, + track_index: track_index.unwrap_or(0), + start_frame, + duration_frames, + trim_start_frame: None, + trim_end_frame: None, + has_audio: probe.has_audio, + add_linked_audio: false, + transform: None, + }, + media: media.clone(), + auto_track: track_index.is_none(), + }, + MotionPlacement::Replace { clip_id } => EditCommand::RegisterMediaAndSwapClip { + media: media.clone(), + clip_id, + }, + MotionPlacement::ReplaceAndClearMasks { clip_id } => { + EditCommand::RegisterMediaAndSwapClipClearingMasks { + media: media.clone(), + clip_id, + } + } + }; + let mut edit = session.editor.apply(command, self.ids.as_ref())?; + edit.action_name = action_name.to_string(); + edit.summary = format!("{} generated media clip(s)", edit.affected_clip_ids.len()); + let written = session.editor.save_project(None)?; + Ok((MotionMediaCommit { media, edit }, written)) + })(); + + match result { + Ok((commit, written)) => { + let count = session.editor.media().entries.len(); + (commit, count, written) + } + Err(error) => { + session.editor.restore_editor_state(before); + return Err(error); + } + } + }; + + self.events.emit(&CoreEvent::TimelineChanged { + project_epoch: expected_project_epoch, + version: commit.edit.timeline_version, + }); + self.events.emit(&CoreEvent::MediaChanged { + project_epoch: expected_project_epoch, + count, + }); + self.events.emit(&CoreEvent::ProjectSaved { + path: written.to_string_lossy().into_owned(), + project_epoch: expected_project_epoch, + }); + Ok(commit) + } + /// Commit a fully probed media-import plan as one project-bound durable /// transaction. The session lock covers identity validation, all manifest /// edits, and the atomic `media.json` write. Any failure restores the exact @@ -1008,6 +1268,18 @@ impl AppCore { } imports.push(CommittedMediaImport { path, entry }); } + PreparedMediaImportOp::ImportDerivedStem { + path, + name, + probe, + provenance, + } => { + let id = self.ids.next_id(); + let entry = session + .editor + .import_derived_stem_file(&path, id, name, &probe, provenance)?; + imports.push(CommittedMediaImport { path, entry }); + } } } @@ -1330,6 +1602,41 @@ impl AppCore { Ok(changed) } + /// Persist one asset's playback proxy as an atomic manifest mutation. + /// Failure to write restores the in-memory manifest before the lock is + /// released, so UI and disk never disagree about proxy availability. + pub fn set_media_proxy_for_project( + &self, + expected_project_epoch: u64, + expected_project_dir: &Path, + asset_id: &str, + proxy: Option, + ) -> Result { + let (entry, count, written) = { + let mut session = self.lock(); + ensure_project_identity(&session, expected_project_epoch, expected_project_dir)?; + let before = session.editor.media(); + let entry = session.editor.set_media_proxy(asset_id, proxy)?; + let count = session.editor.media().entries.len(); + match session.editor.save_media_manifest() { + Ok(written) => (entry, count, written), + Err(error) => { + session.editor.restore_media(before); + return Err(error); + } + } + }; + self.events.emit(&CoreEvent::MediaChanged { + project_epoch: expected_project_epoch, + count, + }); + self.events.emit(&CoreEvent::ProjectSaved { + path: written.to_string_lossy().into_owned(), + project_epoch: expected_project_epoch, + }); + Ok(entry) + } + /// Set or clear one project's global-favorite mapping, emitting the same /// media-change signal used by other manifest mutations. pub fn set_media_global_favorite( @@ -1547,16 +1854,34 @@ impl AppCore { path: impl AsRef, probe: &ProbedMedia, ) -> Result { - let (entry, count, project_epoch) = { + let (entry, count, project_epoch, saved) = { let mut session = self.lock(); + let before = session.editor.media(); let entry = session.editor.relink_media_file(asset_id, path, probe)?; let count = session.editor.media().entries.len(); - (entry, count, session.project_epoch) + let saved = if session.editor.project_dir().is_some() { + match session.editor.save_media_manifest() { + Ok(path) => Some(path), + Err(error) => { + session.editor.restore_media(before); + return Err(error); + } + } + } else { + None + }; + (entry, count, session.project_epoch, saved) }; self.events.emit(&CoreEvent::MediaChanged { project_epoch, count, }); + if let Some(path) = saved { + self.events.emit(&CoreEvent::ProjectSaved { + path: path.to_string_lossy().into_owned(), + project_epoch, + }); + } Ok(entry) } @@ -1576,7 +1901,7 @@ impl AppCore { #[cfg(test)] mod tests { use super::*; - use opentake_domain::{ClipType, Timeline, Track}; + use opentake_domain::{ClipType, MediaColorMetadata, MediaProxy, Timeline, Track}; use opentake_ops::command::ClipEntry; use std::sync::Mutex; @@ -1592,6 +1917,189 @@ mod tests { core } + #[test] + fn motion_media_commit_is_durable_atomic_and_one_step_undoable() { + let bundle = project_bundle("motion-commit"); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let media_dir = opentake_project::layout::media_dir(&bundle); + std::fs::create_dir_all(&media_dir).unwrap(); + let rendered = media_dir.join("motion-a.mp4"); + std::fs::write(&rendered, b"validated-render-fixture").unwrap(); + let snapshot = core.runtime_snapshot(); + let probe = ProbedMedia { + duration_secs: 1.0, + width: Some(64), + height: Some(36), + fps: Some(30.0), + has_audio: false, + color: None, + }; + + let committed = core + .commit_motion_media_for_project( + snapshot.project_epoch, + snapshot.version, + &bundle, + &rendered, + "Motion A", + &probe, + GenerationInput { + prompt: "{\"templateId\":\"title-card\"}".into(), + model: "opentake.motion-canvas".into(), + duration: 30, + aspect_ratio: "64:36".into(), + provider: Some("opentake-motion".into()), + status: Some(opentake_domain::GenerationJobStatus::Ready), + ..GenerationInput::default() + }, + MotionPlacement::Add { + start_frame: 0, + duration_frames: 30, + track_index: Some(0), + }, + ) + .unwrap(); + assert_eq!(committed.edit.action_name, "Add Motion Graphic"); + assert_eq!(core.media().entries.len(), 2); + assert_eq!(core.get_timeline().timeline.tracks[0].clips.len(), 1); + + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + assert_eq!(reopened.media().entries.len(), 2); + assert_eq!(reopened.get_timeline().timeline.tracks[0].clips.len(), 1); + + core.undo().unwrap(); + assert_eq!(core.media().entries.len(), 1); + assert!(core.get_timeline().timeline.tracks[0].clips.is_empty()); + let _ = std::fs::remove_dir_all(bundle); + } + + #[test] + fn generated_media_commit_refuses_version_drift_without_mutation() { + let bundle = project_bundle("generated-version-drift"); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let media_dir = opentake_project::layout::media_dir(&bundle); + std::fs::create_dir_all(&media_dir).unwrap(); + let rendered = media_dir.join("stale-render.mp4"); + std::fs::write(&rendered, b"validated-render-fixture").unwrap(); + let stale = core.runtime_snapshot(); + core.apply(EditCommand::SetTimelineSettings { + fps: 24, + width: 1280, + height: 720, + }) + .unwrap(); + let before_commit = core.runtime_snapshot(); + + let error = core + .commit_motion_media_for_project( + stale.project_epoch, + stale.version, + &bundle, + &rendered, + "Stale Render", + &ProbedMedia::default(), + GenerationInput::default(), + MotionPlacement::Add { + start_frame: 0, + duration_frames: 1, + track_index: Some(0), + }, + ) + .unwrap_err(); + assert!(error + .to_string() + .contains("project changed while preparing a generated-media edit")); + let after_commit = core.runtime_snapshot(); + assert_eq!(after_commit.timeline, before_commit.timeline); + assert_eq!(after_commit.media, before_commit.media); + assert_eq!(after_commit.version, before_commit.version); + + let _ = std::fs::remove_dir_all(bundle); + } + + #[test] + fn motion_media_commit_rejects_output_outside_active_bundle_without_mutation() { + let bundle = project_bundle("motion-outside"); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let outside = std::env::temp_dir().join(format!( + "opentake-motion-outside-{}-{}.mp4", + std::process::id(), + core.runtime_snapshot().project_epoch + )); + std::fs::write(&outside, b"outside").unwrap(); + let before = core.runtime_snapshot(); + + let error = core + .commit_motion_media_for_project( + before.project_epoch, + before.version, + &bundle, + &outside, + "Outside", + &ProbedMedia::default(), + GenerationInput::default(), + MotionPlacement::Add { + start_frame: 0, + duration_frames: 1, + track_index: Some(0), + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("active project media")); + let after = core.runtime_snapshot(); + assert_eq!(after.timeline, before.timeline); + assert_eq!(after.media, before.media); + assert_eq!(after.version, before.version); + + let _ = std::fs::remove_file(outside); + let _ = std::fs::remove_dir_all(bundle); + } + + #[cfg(unix)] + #[test] + fn motion_media_commit_rejects_symlink_without_mutation() { + use std::os::unix::fs::symlink; + + let bundle = project_bundle("motion-symlink"); + let core = AppCore::new(); + core.open_project(&bundle).unwrap(); + let media_dir = opentake_project::layout::media_dir(&bundle); + std::fs::create_dir_all(&media_dir).unwrap(); + let target = media_dir.join("motion-target.mp4"); + let linked = media_dir.join("motion-linked.mp4"); + std::fs::write(&target, b"validated-render-fixture").unwrap(); + symlink(&target, &linked).unwrap(); + let before = core.runtime_snapshot(); + + let error = core + .commit_motion_media_for_project( + before.project_epoch, + before.version, + &bundle, + &linked, + "Linked", + &ProbedMedia::default(), + GenerationInput::default(), + MotionPlacement::Add { + start_frame: 0, + duration_frames: 1, + track_index: Some(0), + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("regular non-symlink")); + let after = core.runtime_snapshot(); + assert_eq!(after.timeline, before.timeline); + assert_eq!(after.media, before.media); + assert_eq!(after.version, before.version); + + let _ = std::fs::remove_dir_all(bundle); + } + #[test] fn project_identity_workflow_blocks_project_replacement_until_release() { let core = AppCore::new(); @@ -2021,6 +2529,7 @@ mod tests { height: Some(480), fps: Some(24.0), has_audio: false, + color: None, }; let entry = core.import_media_file("/abs/a.mp4", "a", &probe).unwrap(); @@ -2038,6 +2547,58 @@ mod tests { ); } + #[test] + fn hdr_and_proxy_metadata_persist_across_project_reopen() { + let temp = tempfile::tempdir().unwrap(); + let bundle = temp.path().join("ColorProxy.opentake"); + let source = temp.path().join("source.mp4"); + std::fs::write(&source, b"source").unwrap(); + let core = AppCore::new(); + core.save_project(Some(bundle.clone())).unwrap(); + let entry = core + .import_media_file( + &source, + "source", + &ProbedMedia { + duration_secs: 1.0, + width: Some(1920), + height: Some(1080), + fps: Some(24.0), + has_audio: false, + color: Some(MediaColorMetadata { + primaries: Some("bt2020".into()), + transfer: Some("smpte2084".into()), + matrix: Some("bt2020nc".into()), + range: Some("tv".into()), + }), + }, + ) + .unwrap(); + core.save_project(None).unwrap(); + let proxy_relative = "media/proxies/source.mp4"; + std::fs::create_dir_all(bundle.join("media/proxies")).unwrap(); + std::fs::write(bundle.join(proxy_relative), b"proxy").unwrap(); + let revision = core.runtime_snapshot(); + core.set_media_proxy_for_project( + revision.project_epoch, + &bundle, + &entry.id, + Some(MediaProxy { + relative_path: proxy_relative.into(), + source_sha256: "a".repeat(64), + width: 640, + height: 360, + }), + ) + .unwrap(); + + let reopened = AppCore::new(); + reopened.open_project(&bundle).unwrap(); + let restored = reopened.media().entries.into_iter().next().unwrap(); + assert!(restored.color.as_ref().is_some_and(|color| color.is_hdr())); + assert_eq!(restored.proxy.unwrap().relative_path, proxy_relative); + } + #[test] fn import_media_unsupported_errors_and_emits_nothing() { let core = AppCore::new(); diff --git a/crates/opentake-core/src/dto.rs b/crates/opentake-core/src/dto.rs index 8bb1b6a2..eadc0854 100644 --- a/crates/opentake-core/src/dto.rs +++ b/crates/opentake-core/src/dto.rs @@ -35,9 +35,23 @@ pub struct CmdError { impl From for CmdError { fn from(err: CoreError) -> Self { + let code = err.code(); + let message = match &err { + CoreError::Edit(_) + | CoreError::Media(_) + | CoreError::Project(opentake_project::ProjectError::CompatibilityReadOnly { + .. + }) + | CoreError::NoProjectOpen + | CoreError::Unsupported(_) => err.to_string(), + CoreError::Project(_) => { + eprintln!("project command failed: {err}"); + "Project operation failed".to_string() + } + }; CmdError { - code: err.code().to_string(), - message: err.to_string(), + code: code.to_string(), + message, } } } @@ -239,9 +253,42 @@ mod tests { #[test] fn edit_apply_handler_maps_validation_error() { let core = core_with_track(); + let before = core.get_timeline(); + let events = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed = events.clone(); + core.subscribe(move |_| { + observed.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }); + let err = handle_edit_apply(&core, EditCommand::AddClips { entries: vec![] }).unwrap_err(); + assert_eq!(err.code, "validation"); assert!(!err.message.is_empty()); + let after = core.get_timeline(); + assert_eq!( + after.timeline, before.timeline, + "failed edit mutated timeline" + ); + assert_eq!(after.version, before.version, "failed edit mutated version"); + assert_eq!( + after.project_epoch, before.project_epoch, + "failed edit mutated project identity" + ); + assert_eq!(events.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + #[test] + fn internal_command_error_does_not_expose_project_paths() { + let error = CmdError::from(CoreError::Project( + opentake_project::ProjectError::MissingTimeline { + file: "project.json", + bundle: "/private/customer/secret.opentake".into(), + }, + )); + + assert_eq!(error.code, "internal"); + assert_eq!(error.message, "Project operation failed"); + assert!(!error.message.contains("/private")); } #[test] diff --git a/crates/opentake-core/src/events.rs b/crates/opentake-core/src/events.rs index f7c0a42e..fa318957 100644 --- a/crates/opentake-core/src/events.rs +++ b/crates/opentake-core/src/events.rs @@ -225,15 +225,41 @@ mod tests { #[test] fn core_event_serializes_with_kind_tag() { - let json = serde_json::to_string(&CoreEvent::TimelineChanged { - project_epoch: 3, - version: 7, - }) - .unwrap(); - assert_eq!( - json, - r#"{"kind":"timeline_changed","projectEpoch":3,"version":7}"# - ); + let cases = [ + ( + CoreEvent::TimelineChanged { + project_epoch: 3, + version: 7, + }, + r#"{"kind":"timeline_changed","projectEpoch":3,"version":7}"#, + ), + ( + CoreEvent::ProjectOpened { + path: "/project.otk".into(), + project_epoch: 4, + version: 0, + }, + r#"{"kind":"project_opened","path":"/project.otk","projectEpoch":4,"version":0}"#, + ), + ( + CoreEvent::ProjectSaved { + path: "/project.otk".into(), + project_epoch: 4, + }, + r#"{"kind":"project_saved","path":"/project.otk","projectEpoch":4}"#, + ), + ( + CoreEvent::MediaChanged { + project_epoch: 4, + count: 2, + }, + r#"{"kind":"media_changed","projectEpoch":4,"count":2}"#, + ), + ]; + + for (event, expected) in cases { + assert_eq!(serde_json::to_string(&event).unwrap(), expected); + } } #[test] diff --git a/crates/opentake-core/src/lib.rs b/crates/opentake-core/src/lib.rs index 070c06eb..641854da 100644 --- a/crates/opentake-core/src/lib.rs +++ b/crates/opentake-core/src/lib.rs @@ -45,13 +45,14 @@ pub mod session; // --- Assembly façade --- pub use crate::core::{ AppCore, BundleExportSnapshot, CapabilityImportCommit, CommittedMediaImport, - DeferredCoreEvents, ImportCommitWarning, PreparedMediaFolderRef, PreparedMediaImportOp, - ProjectRevision, ProjectRuntimeSnapshot, TimelineSnapshot, + DeferredCoreEvents, ImportCommitWarning, MotionMediaCommit, MotionPlacement, + PreparedMediaFolderRef, PreparedMediaImportOp, ProjectRevision, ProjectRuntimeSnapshot, + TimelineSnapshot, }; pub use session::{ - importable_clip_type, EditorSession, GenerationJobCommit, GenerationStateUpdate, - PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, SUPPORTED_AUDIO_EXTENSIONS, - SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, + importable_clip_type, DerivedStemProvenance, EditorSession, GenerationJobCommit, + GenerationStateUpdate, PreparedGenerationJob, PreparedGenerationOutput, ProbedMedia, + SUPPORTED_AUDIO_EXTENSIONS, SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, }; // --- Events --- diff --git a/crates/opentake-core/src/session.rs b/crates/opentake-core/src/session.rs index ced3d523..1aa4a535 100644 --- a/crates/opentake-core/src/session.rs +++ b/crates/opentake-core/src/session.rs @@ -35,8 +35,8 @@ use std::path::{Path, PathBuf}; use opentake_domain::{ - ClipType, GenerationInput, GenerationJobStatus, MediaAsset, MediaManifest, MediaManifestEntry, - MediaSource, Timeline, + ClipType, GenerationInput, GenerationJobStatus, MediaAsset, MediaColorMetadata, MediaManifest, + MediaManifestEntry, MediaProxy, MediaSource, Timeline, }; use opentake_ops::command::{self, EditCommand, EditResult}; use opentake_ops::{EditorState, IdGen}; @@ -68,6 +68,20 @@ pub struct ProbedMedia { pub fps: Option, /// Whether the file carries an audio track. pub has_audio: bool, + /// Source color signalling for HDR-aware decode and durable project state. + pub color: Option, +} + +/// Non-secret provenance attached when a separated audio stem re-enters the +/// ordinary media manifest. Content/model hashes make the derivation auditable +/// without persisting provider credentials or result URLs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DerivedStemProvenance { + pub source_asset_id: String, + pub source_sha256: String, + pub execution: String, + pub model_sha256: Option, + pub stem: String, } /// Validated provider-neutral generation job prepared by the Agent/Tauri host. @@ -167,6 +181,10 @@ fn safe_provider_prefix(value: &str) -> bool { .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') } +fn valid_sha256(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + fn safe_generation_error_code(value: &str) -> bool { !value.is_empty() && value.len() <= 80 @@ -477,6 +495,98 @@ impl EditorSession { self.import_media_file_checked(path, id, name, probe, || Ok(())) } + /// Import a ready vocals/accompaniment file through the shared media path + /// and attach durable provenance. The original asset remains immutable. + pub fn import_derived_stem_file( + &mut self, + path: impl AsRef, + id: impl Into, + name: impl Into, + probe: &ProbedMedia, + provenance: DerivedStemProvenance, + ) -> Result { + self.ensure_mutable()?; + let source = self + .state + .manifest + .entries + .iter() + .find(|entry| entry.id == provenance.source_asset_id) + .ok_or_else(|| { + CoreError::Media(format!( + "stem source asset does not exist: {}", + provenance.source_asset_id + )) + })?; + if !matches!(source.kind, ClipType::Audio | ClipType::Video) + || !source.has_audio.unwrap_or(source.kind == ClipType::Audio) + { + return Err(CoreError::Media( + "stem source asset has no audio".to_string(), + )); + } + if !valid_sha256(&provenance.source_sha256) + || provenance + .model_sha256 + .as_deref() + .is_some_and(|digest| !valid_sha256(digest)) + { + return Err(CoreError::Media( + "stem provenance checksum is invalid".to_string(), + )); + } + let (provider, model) = provenance.execution.split_once(':').ok_or_else(|| { + CoreError::Media("stem execution must be ':'".to_string()) + })?; + if !safe_provider_prefix(provider) || model.trim().is_empty() { + return Err(CoreError::Media( + "stem execution provider or model is invalid".to_string(), + )); + } + let output_index = match provenance.stem.as_str() { + "vocals" => 0, + "accompaniment" => 1, + _ => { + return Err(CoreError::Media( + "stem kind must be vocals or accompaniment".to_string(), + )) + } + }; + + let before = self.state.manifest.clone(); + let result = (|| { + let entry = self.import_media_file(path, id, name, probe)?; + let target = self + .state + .manifest + .entries + .iter_mut() + .find(|candidate| candidate.id == entry.id) + .ok_or_else(|| CoreError::Media("imported stem disappeared".to_string()))?; + target.generation_input = Some(GenerationInput { + prompt: format!("stem:{}", provenance.stem), + model: model.to_string(), + duration: probe.duration_secs.max(0.0).round() as i32, + aspect_ratio: "audio".to_string(), + quality: provenance + .model_sha256 + .map(|digest| format!("model-sha256:{digest}")), + reference_audio_urls: Some(vec![format!("sha256:{}", provenance.source_sha256)]), + provider: Some(provider.to_string()), + status: Some(GenerationJobStatus::Ready), + progress: Some(1.0), + output_index: Some(output_index), + source_asset_id: Some(provenance.source_asset_id), + ..GenerationInput::default() + }); + Ok(target.clone()) + })(); + if result.is_err() { + self.state.manifest = before; + } + result + } + /// Import one file and roll the manifest back if `postcondition` fails. /// Save-as-media uses this to bind its final retained-file identity check to /// the live manifest mutation: an attacker-triggered swap can never leave a @@ -498,6 +608,7 @@ impl EditorSession { asset.source_width = probe.width; asset.source_height = probe.height; asset.source_fps = probe.fps; + asset.color = probe.color.clone(); // Video defaults to having audio (MediaAsset::new); refine from the probe. // Non-video never carries a video-track-linked audio flag upstream. asset.has_audio = match kind { @@ -572,6 +683,8 @@ impl EditorSession { entry.source_width = probe.width; entry.source_height = probe.height; entry.source_fps = probe.fps; + entry.color = probe.color.clone(); + entry.proxy = None; entry.has_audio = Some(match kind { ClipType::Audio => true, ClipType::Video => probe.has_audio, @@ -590,6 +703,47 @@ impl EditorSession { Ok(self.state.manifest.set_favorites(asset_ids, favorite)) } + /// Attach or clear a project-local playback proxy without changing the + /// authoritative source used by export. The proxy path is deliberately + /// constrained to `media/proxies/` and the source digest is fixed-width so + /// corrupt or externally-authored manifests cannot redirect playback. + pub fn set_media_proxy( + &mut self, + asset_id: &str, + proxy: Option, + ) -> Result { + self.ensure_mutable()?; + if let Some(proxy) = proxy.as_ref() { + let path = Path::new(&proxy.relative_path); + let components: Vec<_> = path.components().collect(); + if path.is_absolute() + || components.len() != 3 + || components[0] != std::path::Component::Normal("media".as_ref()) + || components[1] != std::path::Component::Normal("proxies".as_ref()) + || !matches!(components[2], std::path::Component::Normal(_)) + || path.extension().and_then(|extension| extension.to_str()) != Some("mp4") + || proxy.width == 0 + || proxy.height == 0 + || proxy.source_sha256.len() != 64 + || !proxy + .source_sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(CoreError::Media("invalid media proxy metadata".to_string())); + } + } + let entry = self + .state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| CoreError::Media(format!("unknown media asset: {asset_id}")))?; + entry.proxy = proxy; + Ok(entry.clone()) + } + /// Set or clear one asset's content-addressed global favorite id. This is a /// manifest mutation outside undo, matching [`Self::set_media_favorite`]. pub fn set_media_global_favorite( @@ -778,6 +932,8 @@ impl EditorSession { plan.kind == ClipType::Audio || (plan.kind == ClipType::Video && input.generate_audio.unwrap_or(true)), ), + color: None, + proxy: None, folder_id: plan.folder_id.clone(), cached_remote_url: None, cached_remote_url_expires_at: None, @@ -894,6 +1050,8 @@ impl EditorSession { entry.source_height = output.probe.height; entry.source_fps = output.probe.fps; entry.has_audio = Some(output.probe.has_audio); + entry.color = output.probe.color.clone(); + entry.proxy = None; input.status = Some(GenerationJobStatus::Ready); input.progress = Some(1.0); input.error_code = None; @@ -1366,6 +1524,7 @@ mod tests { height: Some(1080), fps: Some(30.0), has_audio: true, + color: None, }; let entry = s .import_media_file("/abs/clip.mp4", "asset-1", "clip", &probe) @@ -1397,6 +1556,48 @@ mod tests { assert_eq!(s.version(), 0); } + #[test] + fn derived_stem_import_reuses_media_path_and_persists_provenance() { + let mut session = EditorSession::new_project(); + let source_probe = ProbedMedia { + duration_secs: 5.0, + has_audio: true, + ..ProbedMedia::default() + }; + session + .import_media_file("/abs/source.wav", "source-asset", "Source", &source_probe) + .unwrap(); + let stem = session + .import_derived_stem_file( + "/abs/source-vocals.wav", + "stem-asset", + "Source Vocals", + &source_probe, + DerivedStemProvenance { + source_asset_id: "source-asset".into(), + source_sha256: "a".repeat(64), + execution: "local:opentake-center-v1".into(), + model_sha256: Some("b".repeat(64)), + stem: "vocals".into(), + }, + ) + .unwrap(); + let provenance = stem.generation_input.expect("derived provenance"); + assert_eq!(provenance.prompt, "stem:vocals"); + assert_eq!(provenance.provider.as_deref(), Some("local")); + assert_eq!(provenance.model, "opentake-center-v1"); + assert_eq!(provenance.source_asset_id.as_deref(), Some("source-asset")); + assert_eq!( + provenance.reference_audio_urls, + Some(vec![format!("sha256:{}", "a".repeat(64))]) + ); + assert_eq!( + provenance.quality, + Some(format!("model-sha256:{}", "b".repeat(64))) + ); + assert_eq!(provenance.status, Some(GenerationJobStatus::Ready)); + } + #[test] fn reimporting_the_same_file_reuses_the_entry_instead_of_duplicating() { // #91: importing a file already in the manifest must not append a second @@ -1409,6 +1610,7 @@ mod tests { height: Some(480), fps: Some(24.0), has_audio: true, + color: None, }; let first = s .import_media_file("/abs/clip.mp4", "asset-1", "clip", &probe) @@ -1424,6 +1626,55 @@ mod tests { assert_eq!(second.source, first.source); } + #[test] + fn media_proxy_metadata_is_confined_and_never_replaces_source() { + let mut session = EditorSession::new_project(); + let source = "/abs/source.mp4"; + let entry = session + .import_media_file(source, "asset", "source", &ProbedMedia::default()) + .unwrap(); + let original_source = entry.source; + let proxy = MediaProxy { + relative_path: "media/proxies/asset.mp4".into(), + source_sha256: "a".repeat(64), + width: 640, + height: 360, + }; + let updated = session + .set_media_proxy("asset", Some(proxy.clone())) + .unwrap(); + assert_eq!(updated.source, original_source); + assert_eq!(updated.proxy, Some(proxy)); + + assert!(session + .set_media_proxy( + "asset", + Some(MediaProxy { + relative_path: "../outside.mp4".into(), + source_sha256: "a".repeat(64), + width: 640, + height: 360, + }), + ) + .is_err()); + assert!(session + .set_media_proxy( + "asset", + Some(MediaProxy { + relative_path: "media/proxies/nested/asset.mp4".into(), + source_sha256: "a".repeat(64), + width: 640, + height: 360, + }), + ) + .is_err()); + assert!(session + .set_media_proxy("asset", None) + .unwrap() + .proxy + .is_none()); + } + #[test] fn import_image_has_no_audio_regardless_of_probe() { let mut s = EditorSession::new_project(); @@ -1433,6 +1684,7 @@ mod tests { height: Some(600), fps: None, has_audio: true, // probe lies; an image never has audio + color: None, }; let entry = s .import_media_file("/abs/pic.png", "img-1", "pic", &probe) @@ -1534,6 +1786,8 @@ mod tests { source_height: Some(2), source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-core/tests/generation_persistence.rs b/crates/opentake-core/tests/generation_persistence.rs index 94bf0eec..6db20e06 100644 --- a/crates/opentake-core/tests/generation_persistence.rs +++ b/crates/opentake-core/tests/generation_persistence.rs @@ -25,6 +25,8 @@ fn saved_project() -> (tempfile::TempDir, std::path::PathBuf) { source_height: Some(3), source_fps: None, has_audio: Some(false), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -140,6 +142,7 @@ fn placeholders_job_events_and_finalized_output_survive_restart() { height: Some(6), fps: None, has_audio: false, + color: None, }, created_at: Some(800_000_002.0), }, @@ -283,6 +286,7 @@ fn cancelling_a_partially_finalized_job_preserves_ready_outputs() { height: Some(6), fps: None, has_audio: false, + color: None, }, created_at: Some(800_000_002.0), }, diff --git a/crates/opentake-core/tests/project_open.rs b/crates/opentake-core/tests/project_open.rs index 6d1903e5..449ca8c9 100644 --- a/crates/opentake-core/tests/project_open.rs +++ b/crates/opentake-core/tests/project_open.rs @@ -60,6 +60,8 @@ fn manifest_entry(id: &str, generation_input: Option) -> MediaM source_height: None, source_fps: None, has_audio: Some(true), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-core/tests/schema_compat.rs b/crates/opentake-core/tests/schema_compat.rs index 07901630..694d97b0 100644 --- a/crates/opentake-core/tests/schema_compat.rs +++ b/crates/opentake-core/tests/schema_compat.rs @@ -47,6 +47,8 @@ fn external_entry(id: &str, name: &str, source: &Path) -> MediaManifestEntry { source_height: Some(240), source_fps: Some(30.0), has_audio: Some(false), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-domain/src/audio.rs b/crates/opentake-domain/src/audio.rs new file mode 100644 index 00000000..91e98344 --- /dev/null +++ b/crates/opentake-domain/src/audio.rs @@ -0,0 +1,86 @@ +use serde::{Deserialize, Serialize}; + +/// Deterministic local denoise profiles. Both use the same spectral processing +/// owner; `Voice` applies stronger subtraction for spoken-word material. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DenoiseMode { + #[default] + Adaptive, + Voice, +} + +/// Non-destructive denoise parameters persisted on one clip. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AudioDenoise { + pub mode: DenoiseMode, + pub strength: f64, + /// Inspector A/B toggle. Export always applies the configured operation; + /// native preview applies it only while this flag is true. + pub preview_enabled: bool, +} + +impl AudioDenoise { + pub fn validate(&self) -> Result<(), &'static str> { + if !self.strength.is_finite() || !(0.0..=1.0).contains(&self.strength) { + return Err("denoise strength must be finite and between 0 and 1"); + } + Ok(()) + } +} + +/// Persisted result of one clip loudness analysis. The measured values make the +/// operation reproducible; playback/export consume only `gain_db` and never +/// need to re-read the source. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LoudnessNormalization { + pub target_lufs: f64, + pub true_peak_ceiling_dbtp: f64, + pub input_integrated_lufs: f64, + pub input_true_peak_dbtp: f64, + pub gain_db: f64, + pub output_integrated_lufs: f64, + pub output_true_peak_dbtp: f64, +} + +impl LoudnessNormalization { + pub fn validate(&self) -> Result<(), &'static str> { + let values = [ + self.target_lufs, + self.true_peak_ceiling_dbtp, + self.input_integrated_lufs, + self.input_true_peak_dbtp, + self.gain_db, + self.output_integrated_lufs, + self.output_true_peak_dbtp, + ]; + if values.iter().any(|value| !value.is_finite()) { + return Err("loudness values must be finite"); + } + if self.true_peak_ceiling_dbtp > 0.0 { + return Err("true-peak ceiling must be at most 0 dBTP"); + } + if !(-70.0..=0.0).contains(&self.target_lufs) + || !(-20.0..=0.0).contains(&self.true_peak_ceiling_dbtp) + || !(-120.0..=60.0).contains(&self.gain_db) + { + return Err("loudness target, ceiling, or gain is outside the supported range"); + } + if (self.output_integrated_lufs - self.target_lufs).abs() > 1.0 { + return Err("normalized loudness does not reach its target"); + } + if self.output_true_peak_dbtp > self.true_peak_ceiling_dbtp + 0.05 { + return Err("normalized true peak exceeds its ceiling"); + } + Ok(()) + } + + pub fn linear_gain(self) -> f64 { + if !self.gain_db.is_finite() { + return 1.0; + } + 10.0_f64.powf(self.gain_db.clamp(-120.0, 60.0) / 20.0) + } +} diff --git a/crates/opentake-domain/src/clip.rs b/crates/opentake-domain/src/clip.rs index d8570097..be8c495f 100644 --- a/crates/opentake-domain/src/clip.rs +++ b/crates/opentake-domain/src/clip.rs @@ -17,10 +17,26 @@ use crate::clip_wire::{ deserialize_one_on_error, deserialize_optional_crop_track_on_error, deserialize_optional_f64_track_on_error, deserialize_optional_pair_track_on_error, }; -use crate::grade::{ChromaKey, ColorGrade, Effect, Mask}; +use crate::grade::{ChromaKey, ColorGrade, ColorMatchInput, Effect, Mask}; use crate::keyframe::{AnimPair, AnimatableProperty, Interpolation, KeyframeTrack}; +use crate::lut::LutReference; +use crate::stabilization::StabilizationTrack; use crate::text::TextStyle; use crate::transform::{Crop, Point, Transform}; +use crate::transition::Transition; + +/// Persisted provenance for one accepted caption translation. The original +/// text remains available for review/recovery and manual text edits clear this +/// record so the project never attributes authored text to a provider. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CaptionTranslationInput { + pub source_text: String, + pub source_locale: String, + pub target_locale: String, + pub provider: String, + pub model: String, +} /// Linear amplitude <-> dB mapping for the volume slider. 1:1 port of /// upstream `VolumeScale`. Below the floor we snap to true 0 (hard mute). @@ -147,6 +163,15 @@ pub struct Clip { skip_serializing_if = "Option::is_none" )] pub caption_group_id: Option, + /// ID of an editable child timeline in the root timeline's + /// `nestedSequences` registry. Nested clips never overload `mediaRef` with + /// a sentinel value, so media and sequence identity remain unambiguous. + #[serde( + default, + deserialize_with = "deserialize_default_on_error", + skip_serializing_if = "Option::is_none" + )] + pub nested_sequence_id: Option, // Text clips only. #[serde( @@ -161,6 +186,8 @@ pub struct Clip { skip_serializing_if = "Option::is_none" )] pub text_style: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub caption_translation_input: Option, // Keyframe tracks for each animatable property. None when no animation exists. #[serde( @@ -200,12 +227,29 @@ pub struct Clip { )] pub volume_track: Option>, + /// Source analysis plus the static gain used identically by preview and + /// export. `None` leaves authored volume behavior unchanged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loudness_normalization: Option, + + /// Local non-destructive denoise configuration. Source PCM is never + /// rewritten; native preview and export process decoded copies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_denoise: Option, + // Advanced pixel-effect fields (A-tier; `docs/ADVANCED-FEATURES.md`). All // `#[serde(default)]` + Option/Vec, so older projects (without these keys) // decode unchanged, and an all-default clip omits them on the way out. /// High-end floating-point color grade (linear-light chain). `None` = no grade. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_grade: Option, + /// Sampling inputs and measured result for an automatically matched grade. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color_match_input: Option, + /// Project-managed 3D LUT reference. The persisted id maps only to the + /// bundle's canonical `media/luts/.cube` location. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lut: Option, /// Green/blue-screen chroma key. `None` = no keying. #[serde(default, skip_serializing_if = "Option::is_none")] pub chroma_key: Option, @@ -215,6 +259,17 @@ pub struct Clip { /// Generic named-effect chain. Empty = no effects. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub effects: Vec, + /// Non-destructive camera compensation composed with authored transform tracks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stabilization: Option, + /// Optional visual transition into one exact adjacent successor. Pair + /// identity prevents stale transitions from rebinding after moves/deletes. + #[serde( + default, + deserialize_with = "deserialize_default_on_error", + skip_serializing_if = "Option::is_none" + )] + pub transition_out: Option, /// Reverse playback. When true, video clips sample their referenced source /// window in reverse order. Non-video sources ignore this flag. #[serde(default, skip_serializing_if = "is_false")] @@ -245,18 +300,25 @@ impl Clip { "crop", "linkGroupId", "captionGroupId", + "nestedSequenceId", "textContent", "textStyle", + "captionTranslationInput", "opacityTrack", "positionTrack", "scaleTrack", "rotationTrack", "cropTrack", "volumeTrack", + "loudnessNormalization", + "audioDenoise", "colorGrade", + "lut", "chromaKey", "masks", "effects", + "stabilization", + "transitionOut", "reversed", ]; pub const TOLERANT_SCALAR_WIRE_FIELDS: &'static [&'static str] = &[ @@ -334,22 +396,42 @@ impl Clip { crop: Crop::default(), link_group_id: None, caption_group_id: None, + nested_sequence_id: None, text_content: None, text_style: None, + caption_translation_input: None, opacity_track: None, position_track: None, scale_track: None, rotation_track: None, crop_track: None, volume_track: None, + loudness_normalization: None, + audio_denoise: None, color_grade: None, + color_match_input: None, + lut: None, chroma_key: None, masks: Vec::new(), effects: Vec::new(), + stabilization: None, + transition_out: None, reversed: false, } } + /// Construct a clip that references one editable nested sequence. + pub fn new_nested( + id: impl Into, + sequence_id: impl Into, + start_frame: i32, + duration_frames: i32, + ) -> Self { + let mut clip = Self::new(id, "", start_frame, duration_frames); + clip.nested_sequence_id = Some(sequence_id.into()); + clip + } + /// Frame where this clip ends on the timeline (exclusive end). pub fn end_frame(&self) -> i32 { self.start_frame + self.duration_frames @@ -486,7 +568,7 @@ impl Clip { } _ => 1.0, }; - self.volume * kf_gain * self.fade_multiplier(frame) + self.volume * kf_gain * self.loudness_gain() * self.fade_multiplier(frame) } /// Linear volume without the fade envelope. @@ -497,7 +579,13 @@ impl Clip { } _ => 1.0, }; - self.volume * kf_gain + self.volume * kf_gain * self.loudness_gain() + } + + pub fn loudness_gain(&self) -> f64 { + self.loudness_normalization + .map(crate::LoudnessNormalization::linear_gain) + .unwrap_or(1.0) } /// 0..=1 envelope from the fade head/tail ramps. `min(in, out)`. Returns 0 @@ -903,6 +991,27 @@ mod tests { approx(c.raw_volume_at(105), 1.0); } + #[test] + fn persisted_loudness_gain_is_shared_by_raw_and_effective_volume() { + let mut c = base_clip(); + c.volume = 0.5; + c.loudness_normalization = Some(crate::LoudnessNormalization { + target_lufs: -16.0, + true_peak_ceiling_dbtp: -1.0, + input_integrated_lufs: -22.0, + input_true_peak_dbtp: -8.0, + gain_db: 6.0, + output_integrated_lufs: -16.0, + output_true_peak_dbtp: -2.0, + }); + let expected = 0.5 * 10.0_f64.powf(6.0 / 20.0); + approx(c.raw_volume_at(110), expected); + approx(c.volume_at(110), expected); + let json = serde_json::to_string(&c).unwrap(); + let roundtrip: Clip = serde_json::from_str(&json).unwrap(); + assert_eq!(roundtrip.loudness_normalization, c.loudness_normalization); + } + #[test] fn live_volume_kf_db_requires_active_track_and_membership() { let mut c = base_clip(); @@ -1149,6 +1258,42 @@ mod tests { assert_eq!(c, back); } + #[test] + fn clip_transition_roundtrips_and_legacy_projects_default_to_none() { + let mut c = base_clip(); + c.transition_out = Some(crate::transition::Transition { + from_clip_id: "c1".into(), + to_clip_id: "c2".into(), + kind: crate::transition::TransitionKind::CrossDissolve, + duration_frames: 12, + }); + let json = serde_json::to_string(&c).unwrap(); + assert!(json.contains( + r#""transitionOut":{"fromClipId":"c1","toClipId":"c2","kind":"crossDissolve","durationFrames":12}"# + )); + let back: Clip = serde_json::from_str(&json).unwrap(); + assert_eq!(c, back); + + let legacy: Clip = serde_json::from_str( + r#"{"id":"old","mediaRef":"m","startFrame":0,"durationFrames":12}"#, + ) + .unwrap(); + assert!(legacy.transition_out.is_none()); + + let legacy_transition: Clip = serde_json::from_str( + r#"{"id":"c1","mediaRef":"m","startFrame":0,"durationFrames":12,"transitionOut":{"toClipId":"c2","kind":"crossDissolve","durationFrames":4}}"#, + ) + .unwrap(); + assert_eq!( + legacy_transition + .transition_out + .as_ref() + .unwrap() + .from_clip_id, + "" + ); + } + #[test] fn clip_decodes_with_missing_optional_fields() { // Only the required keys present; everything else falls back to defaults. @@ -1221,8 +1366,9 @@ mod tests { }, feather: 0.05, invert: false, + ..Mask::default() }]; - c.effects = vec![Effect::new("gaussianBlur").with_param("radius", 4.0)]; + c.effects = vec![Effect::new("grayscale").with_param("amount", 0.4)]; let json = serde_json::to_string(&c).unwrap(); assert!(json.contains("\"colorGrade\"")); assert!(json.contains("\"chromaKey\"")); diff --git a/crates/opentake-domain/src/grade.rs b/crates/opentake-domain/src/grade.rs index bac2593b..408c0658 100644 --- a/crates/opentake-domain/src/grade.rs +++ b/crates/opentake-domain/src/grade.rs @@ -22,6 +22,11 @@ use serde::{Deserialize, Serialize}; +/// Fixed GPU contract shared by editor validation and the compositor uniform. +pub const MAX_MASKS_PER_CLIP: usize = 4; +/// Maximum authored vertices in one pen/polygon mask. +pub const MAX_POLYGON_MASK_POINTS: usize = 16; + // =========================================================================== // Small numeric helpers (shared by the reference pixel math). // =========================================================================== @@ -134,23 +139,132 @@ impl LiftGammaGain { self.lift.is_zero() && self.gamma.is_one() && self.gain.is_one() } - /// Apply one channel: `gain * (x + lift)` then `^(1/gamma)`. Matches the - /// classic lift/gamma/gain operator (gamma applied last, as a display power). + /// Apply one channel: `gain * (x + lift * (1 - x))^(1/gamma)`. + /// Lift rolls off toward highlights, gamma shapes mid-tones, and gain remains + /// an independent highlight multiplier. #[inline] fn apply_channel(x: f64, lift: f64, gamma: f64, gain: f64) -> f64 { - let v = gain * (x + lift); + let shaped = x + lift * (1.0 - x); if gamma > 0.0 && (gamma - 1.0).abs() > f64::EPSILON { - // `v` can be negative after lift; guard the power. - v.max(0.0).powf(1.0 / gamma) + // `shaped` can be negative after lift; guard fractional powers. + gain * shaped.max(0.0).powf(1.0 / gamma) } else { - v + gain * shaped + } + } +} + +/// One feathered HSL qualifier applied after the primary color controls. +/// Hue values are normalized turns, so the range wraps continuously across +/// red (`0 == 1`) without a seam. +#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HslSecondary { + /// Center of the selected hue range in normalized turns (`0..=1`). + pub hue_center: f64, + /// Full width of the selected hue range (`0 < width <= 1`). + pub hue_width: f64, + /// Soft edge width measured inward from both range boundaries (`0..=0.5`). + pub feather: f64, + /// Hue rotation in normalized turns (`-0.5..=0.5`). + pub hue_shift: f64, + /// Relative saturation adjustment (`-1..=1`). + pub saturation: f64, + /// Additive lightness adjustment (`-1..=1`). + pub lightness: f64, +} + +impl Default for HslSecondary { + fn default() -> Self { + Self { + hue_center: 0.0, + hue_width: 0.24, + feather: 0.08, + hue_shift: 0.0, + saturation: 0.0, + lightness: 0.0, + } + } +} + +impl HslSecondary { + fn is_identity(&self) -> bool { + self.hue_shift == 0.0 && self.saturation == 0.0 && self.lightness == 0.0 + } + + fn weight(&self, hue: f64, saturation: f64) -> f64 { + // Achromatic pixels have no stable hue and must not be selected. + if saturation <= f64::EPSILON { + return 0.0; + } + let distance = ((hue - self.hue_center + 0.5).rem_euclid(1.0) - 0.5).abs(); + let outer = self.hue_width * 0.5; + if distance > outer { + return 0.0; + } + if self.feather <= f64::EPSILON { + return 1.0; + } + let inner = (outer - self.feather).max(0.0); + 1.0 - smoothstep01(inner, outer, distance) + } + + fn apply(&self, r: f64, g: f64, b: f64) -> (f64, f64, f64) { + let (mut hue, mut saturation, mut lightness) = rgb_to_hsl(r, g, b); + let weight = self.weight(hue, saturation); + if weight <= f64::EPSILON { + return (r, g, b); } + hue = (hue + self.hue_shift * weight).rem_euclid(1.0); + saturation = clamp01(saturation * (1.0 + self.saturation * weight)); + lightness = clamp01(lightness + self.lightness * weight); + hsl_to_rgb(hue, saturation, lightness) } } +fn rgb_to_hsl(r: f64, g: f64, b: f64) -> (f64, f64, f64) { + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let delta = max - min; + let lightness = (max + min) * 0.5; + if delta <= f64::EPSILON { + return (0.0, 0.0, lightness); + } + let saturation = delta / (1.0 - (2.0 * lightness - 1.0).abs()).max(f64::EPSILON); + let sector = if max == r { + ((g - b) / delta).rem_euclid(6.0) + } else if max == g { + (b - r) / delta + 2.0 + } else { + (r - g) / delta + 4.0 + }; + (sector / 6.0, saturation, lightness) +} + +fn hsl_to_rgb(hue: f64, saturation: f64, lightness: f64) -> (f64, f64, f64) { + let chroma = (1.0 - (2.0 * lightness - 1.0).abs()) * saturation; + let sector = hue.rem_euclid(1.0) * 6.0; + let x = chroma * (1.0 - (sector.rem_euclid(2.0) - 1.0).abs()); + let (r1, g1, b1) = if sector < 1.0 { + (chroma, x, 0.0) + } else if sector < 2.0 { + (x, chroma, 0.0) + } else if sector < 3.0 { + (0.0, chroma, x) + } else if sector < 4.0 { + (0.0, x, chroma) + } else if sector < 5.0 { + (x, 0.0, chroma) + } else { + (chroma, 0.0, x) + }; + let m = lightness - chroma * 0.5; + (r1 + m, g1 + m, b1 + m) +} + /// High-end floating-point color grade, applied in **linear light** in the order /// locked by the spec: -/// `exposure -> white balance -> lift/gamma/gain -> contrast -> saturation`. +/// `exposure -> white balance -> lift/gamma/gain -> contrast -> saturation -> HSL secondary`. /// /// Every field defaults to a no-op, so `ColorGrade::default()` is the identity /// transform (verified by [`ColorGrade::is_identity`] and a unit test). @@ -176,8 +290,48 @@ pub struct ColorGrade { /// Saturation multiplier (identity `1`; `0` = greyscale, `>1` = boosted). #[serde(default = "default_one")] pub saturation: f64, + /// Optional feathered hue qualifier. Absence preserves legacy projects and + /// avoids uploading an active secondary block for identity grades. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hsl_secondary: Option, } +/// Persisted provenance for an automatically generated reference color match. +/// The grade remains fully editable; a later manual grade change clears this +/// record so projects never claim an edited grade is still the sampled match. +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColorMatchInput { + pub reference_media_ref: String, + pub reference_frame: i32, + pub target_frame: i32, + pub algorithm: String, + pub algorithm_version: u32, + pub target_mean_linear: Rgb, + pub reference_mean_linear: Rgb, + pub delta_e_before: f64, + pub delta_e_after: f64, + pub target_luma_before: f64, + pub target_luma_after: f64, +} + +/// Stable validation failure for authored color-grade parameters. The bounds +/// mirror the Inspector controls and keep malformed persisted data out of the +/// command and GPU paths. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ColorGradeValidationError { + pub field: &'static str, + pub rule: &'static str, +} + +impl std::fmt::Display for ColorGradeValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} must be {}", self.field, self.rule) + } +} + +impl std::error::Error for ColorGradeValidationError {} + impl Default for ColorGrade { fn default() -> Self { ColorGrade { @@ -187,6 +341,7 @@ impl Default for ColorGrade { lift_gamma_gain: LiftGammaGain::default(), contrast: 0.0, saturation: 1.0, + hsl_secondary: None, } } } @@ -204,6 +359,120 @@ impl ColorGrade { && self.lift_gamma_gain.is_identity() && self.contrast == 0.0 && self.saturation == 1.0 + && self + .hsl_secondary + .is_none_or(|secondary| secondary.is_identity()) + } + + /// Validate the persisted grade against the finite parameter ranges exposed + /// by the editor. Gamma is strictly positive because it is used as a power + /// denominator in both the CPU reference and WGSL shader. + pub fn validate(&self) -> Result<(), ColorGradeValidationError> { + fn inclusive( + field: &'static str, + value: f64, + range: std::ops::RangeInclusive, + rule: &'static str, + ) -> Result<(), ColorGradeValidationError> { + if !value.is_finite() || !range.contains(&value) { + return Err(ColorGradeValidationError { field, rule }); + } + Ok(()) + } + + inclusive( + "exposure", + self.exposure, + -5.0..=5.0, + "finite and within [-5, 5]", + )?; + inclusive( + "temperature", + self.temperature, + -1.0..=1.0, + "finite and within [-1, 1]", + )?; + inclusive("tint", self.tint, -1.0..=1.0, "finite and within [-1, 1]")?; + for (field, value) in [ + ("liftGammaGain.lift.r", self.lift_gamma_gain.lift.r), + ("liftGammaGain.lift.g", self.lift_gamma_gain.lift.g), + ("liftGammaGain.lift.b", self.lift_gamma_gain.lift.b), + ] { + inclusive(field, value, -1.0..=1.0, "finite and within [-1, 1]")?; + } + for (field, value) in [ + ("liftGammaGain.gamma.r", self.lift_gamma_gain.gamma.r), + ("liftGammaGain.gamma.g", self.lift_gamma_gain.gamma.g), + ("liftGammaGain.gamma.b", self.lift_gamma_gain.gamma.b), + ] { + if !value.is_finite() || value <= 0.0 || value > 4.0 { + return Err(ColorGradeValidationError { + field, + rule: "finite and within (0, 4]", + }); + } + } + for (field, value) in [ + ("liftGammaGain.gain.r", self.lift_gamma_gain.gain.r), + ("liftGammaGain.gain.g", self.lift_gamma_gain.gain.g), + ("liftGammaGain.gain.b", self.lift_gamma_gain.gain.b), + ] { + inclusive(field, value, 0.0..=4.0, "finite and within [0, 4]")?; + } + inclusive( + "contrast", + self.contrast, + -1.0..=2.0, + "finite and within [-1, 2]", + )?; + inclusive( + "saturation", + self.saturation, + 0.0..=3.0, + "finite and within [0, 3]", + )?; + if let Some(secondary) = self.hsl_secondary { + inclusive( + "hslSecondary.hueCenter", + secondary.hue_center, + 0.0..=1.0, + "finite and within [0, 1]", + )?; + if !secondary.hue_width.is_finite() + || secondary.hue_width <= 0.0 + || secondary.hue_width > 1.0 + { + return Err(ColorGradeValidationError { + field: "hslSecondary.hueWidth", + rule: "finite and within (0, 1]", + }); + } + inclusive( + "hslSecondary.feather", + secondary.feather, + 0.0..=0.5, + "finite and within [0, 0.5]", + )?; + inclusive( + "hslSecondary.hueShift", + secondary.hue_shift, + -0.5..=0.5, + "finite and within [-0.5, 0.5]", + )?; + inclusive( + "hslSecondary.saturation", + secondary.saturation, + -1.0..=1.0, + "finite and within [-1, 1]", + )?; + inclusive( + "hslSecondary.lightness", + secondary.lightness, + -1.0..=1.0, + "finite and within [-1, 1]", + )?; + } + Ok(()) } /// Per-channel white-balance gain derived from `temperature` / `tint`. A @@ -266,6 +535,14 @@ impl ColorGrade { bb = l + (bb - l) * self.saturation; } + // 6. Feathered HSL secondary qualifier. + if let Some(secondary) = self + .hsl_secondary + .filter(|secondary| !secondary.is_identity()) + { + (rr, gg, bb) = secondary.apply(rr, gg, bb); + } + (clamp01(rr), clamp01(gg), clamp01(bb)) } } @@ -421,7 +698,7 @@ pub enum MaskShape { /// [`crate::transform::Point`] only to keep mask serialization self-contained and /// `Serialize`/`Deserialize`-derivable (the transform `Point` has hand-written /// (de)serialization elsewhere; here a plain derive is what we want). -#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, PartialEq, Debug, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Point2 { #[serde(default)] @@ -436,6 +713,56 @@ impl Point2 { } } +/// Optional whole-mask transform applied around the canvas center after the +/// shape's own geometry. Shape coordinates remain editable and portable while +/// offset/scale/rotation can move the complete mask non-destructively. +#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MaskTransform { + #[serde(default)] + pub offset: Point2, + #[serde(default = "default_mask_scale")] + pub scale: Point2, + #[serde(default)] + pub rotation_degrees: f64, +} + +fn default_mask_scale() -> Point2 { + Point2::new(1.0, 1.0) +} + +impl Default for MaskTransform { + fn default() -> Self { + MaskTransform { + offset: Point2::new(0.0, 0.0), + scale: default_mask_scale(), + rotation_degrees: 0.0, + } + } +} + +impl MaskTransform { + pub fn is_identity(&self) -> bool { + self.offset.x == 0.0 + && self.offset.y == 0.0 + && self.scale.x == 1.0 + && self.scale.y == 1.0 + && self.rotation_degrees == 0.0 + } + + fn inverse_point(&self, x: f64, y: f64) -> (f64, f64) { + let sx = self.scale.x.abs().max(f64::EPSILON); + let sy = self.scale.y.abs().max(f64::EPSILON); + let radians = self.rotation_degrees.to_radians(); + let (sin, cos) = radians.sin_cos(); + let dx = x - 0.5 - self.offset.x; + let dy = y - 0.5 - self.offset.y; + let unrotated_x = cos * dx + sin * dy; + let unrotated_y = -sin * dx + cos * dy; + (unrotated_x / sx + 0.5, unrotated_y / sy + 0.5) + } +} + /// A vector mask that generates a per-pixel alpha coverage. `feather` softens the /// edge; `invert` flips inside/outside. /// @@ -452,6 +779,9 @@ pub struct Mask { /// Invert coverage (mask out the inside instead of the outside). #[serde(default)] pub invert: bool, + /// Non-destructive whole-mask translation, scale, and rotation. + #[serde(default, skip_serializing_if = "MaskTransform::is_identity")] + pub transform: MaskTransform, } fn default_mask_shape() -> MaskShape { @@ -468,6 +798,7 @@ impl Default for Mask { shape: default_mask_shape(), feather: 0.0, invert: false, + transform: MaskTransform::default(), } } } @@ -478,6 +809,7 @@ impl Mask { /// polygon variant returns the unsigned distance with an inside/outside sign /// from an even-odd test (an exact polygon SDF is overkill for feathering). pub fn signed_distance(&self, x: f64, y: f64) -> f64 { + let (x, y) = self.transform.inverse_point(x, y); match &self.shape { MaskShape::Linear { point, normal } => { // Signed distance along the (assumed unit-ish) normal. We @@ -590,20 +922,129 @@ fn point_segment_dist2(px: f64, py: f64, ax: f64, ay: f64, bx: f64, by: f64) -> // Effect (generic named-parameter effect chain) // =========================================================================== -/// A generic named pixel effect with a flat parameter map — the extensible chain -/// the spec calls for (`Clip.effects: Vec`, each = one wgpu pass). The -/// `name` selects a shader/kernel; `params` are its named scalar inputs and -/// `enabled` lets a clip carry a disabled effect without removing it. -/// -/// Concrete effects (blur, glow, sharpen, ...) are deferred (see module TODO); -/// this type and its serde/round-trip are the stable contract that ops + agent -/// tools target now, and the render layer can grow per-name handling -/// incrementally without further domain changes. +/// Maximum authored effects evaluated for one clip. A fixed bound keeps the +/// persisted contract aligned with the portable GPU uniform layout. +pub const MAX_EFFECTS_PER_CLIP: usize = 8; + +/// One persisted scalar in an advertised effect's closed schema. +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct EffectParameterDescriptor { + pub name: &'static str, + pub default: f64, + pub min: f64, + pub max: f64, +} + +/// An effect available to the editor, agent tools, preview, and export. +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct EffectDescriptor { + pub name: &'static str, + pub parameters: &'static [EffectParameterDescriptor], +} + +const AMOUNT_PARAMETER: [EffectParameterDescriptor; 1] = [EffectParameterDescriptor { + name: "amount", + default: 1.0, + min: 0.0, + max: 1.0, +}]; + +const EFFECT_REGISTRY: [EffectDescriptor; 3] = [ + EffectDescriptor { + name: "grayscale", + parameters: &AMOUNT_PARAMETER, + }, + EffectDescriptor { + name: "sepia", + parameters: &AMOUNT_PARAMETER, + }, + EffectDescriptor { + name: "invert", + parameters: &AMOUNT_PARAMETER, + }, +]; + +/// The complete effect list advertised by every product surface. +pub fn effect_registry() -> &'static [EffectDescriptor] { + &EFFECT_REGISTRY +} + +/// Typed rejection for invalid persisted effect data. +#[derive(Clone, PartialEq, Debug)] +pub enum EffectValidationError { + TooManyEffects { + count: usize, + limit: usize, + }, + UnknownEffect { + name: String, + }, + UnknownParameter { + effect: String, + parameter: String, + }, + NonFiniteParameter { + effect: String, + parameter: String, + }, + ParameterOutOfRange { + effect: String, + parameter: String, + value: f64, + min: f64, + max: f64, + }, +} + +impl std::fmt::Display for EffectValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooManyEffects { count, limit } => { + write!(f, "effect chain contains {count} entries; limit is {limit}") + } + Self::UnknownEffect { name } => write!(f, "unknown effect `{name}`"), + Self::UnknownParameter { effect, parameter } => { + write!(f, "unknown parameter `{parameter}` for effect `{effect}`") + } + Self::NonFiniteParameter { effect, parameter } => write!( + f, + "parameter `{parameter}` for effect `{effect}` must be finite" + ), + Self::ParameterOutOfRange { + effect, + parameter, + value, + min, + max, + } => write!( + f, + "parameter `{parameter}` for effect `{effect}` is {value}; expected {min}..={max}" + ), + } + } +} + +impl std::error::Error for EffectValidationError {} + +/// Validate a complete authored chain before an edit or render boundary. +pub fn validate_effect_chain(effects: &[Effect]) -> Result<(), EffectValidationError> { + if effects.len() > MAX_EFFECTS_PER_CLIP { + return Err(EffectValidationError::TooManyEffects { + count: effects.len(), + limit: MAX_EFFECTS_PER_CLIP, + }); + } + effects.iter().try_for_each(Effect::validate) +} + +/// A named pixel effect with a flat parameter map. Names and parameters remain +/// string-keyed for stable JSON compatibility, but every edit and render path +/// validates them against [`effect_registry`] rather than silently ignoring an +/// unknown value. #[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Effect { - /// Effect identifier (e.g. `"gaussianBlur"`). Free-form; the render layer maps - /// known names to passes and ignores unknown ones. + /// Identifier from [`effect_registry`]. pub name: String, /// Named scalar parameters. Insertion-stable ordering is not required; the /// render layer reads by key. @@ -638,6 +1079,60 @@ impl Effect { pub fn param(&self, key: &str, default: f64) -> f64 { self.params.get(key).copied().unwrap_or(default) } + + /// Validate this persisted value against the advertised closed schema. + pub fn validate(&self) -> Result<(), EffectValidationError> { + let descriptor = effect_registry() + .iter() + .find(|candidate| candidate.name == self.name) + .ok_or_else(|| EffectValidationError::UnknownEffect { + name: self.name.clone(), + })?; + for (name, value) in &self.params { + let parameter = descriptor + .parameters + .iter() + .find(|candidate| candidate.name == name) + .ok_or_else(|| EffectValidationError::UnknownParameter { + effect: self.name.clone(), + parameter: name.clone(), + })?; + if !value.is_finite() { + return Err(EffectValidationError::NonFiniteParameter { + effect: self.name.clone(), + parameter: name.clone(), + }); + } + if *value < parameter.min || *value > parameter.max { + return Err(EffectValidationError::ParameterOutOfRange { + effect: self.name.clone(), + parameter: name.clone(), + value: *value, + min: parameter.min, + max: parameter.max, + }); + } + } + Ok(()) + } + + /// Read a registered scalar using its schema default. + pub fn registered_param(&self, key: &str) -> Result { + self.validate()?; + let descriptor = effect_registry() + .iter() + .find(|candidate| candidate.name == self.name) + .expect("validated effect is registered"); + let parameter = descriptor + .parameters + .iter() + .find(|candidate| candidate.name == key) + .ok_or_else(|| EffectValidationError::UnknownParameter { + effect: self.name.clone(), + parameter: key.to_owned(), + })?; + Ok(self.param(key, parameter.default)) + } } #[cfg(test)] @@ -748,6 +1243,21 @@ mod tests { let (r, gg, _) = g.apply_linear(0.4, 0.4, 0.0); approx(r, 0.2); // 0.4 * 0.5 approx(gg, 0.4); // unchanged + + // The authored color-wheel contract is: + // gain * (x + lift * (1 - x)) ^ (1 / gamma). Lift therefore rolls off + // toward the highlights instead of adding the same offset everywhere, + // and gain remains an independent highlight multiplier outside gamma. + let combined = ColorGrade { + lift_gamma_gain: LiftGammaGain { + lift: Rgb::new(0.1, 0.0, 0.0), + gamma: Rgb::new(2.0, 1.0, 1.0), + gain: Rgb::new(0.8, 1.0, 1.0), + }, + ..Default::default() + }; + let (combined_r, _, _) = combined.apply_linear(0.25, 0.0, 0.0); + approx(combined_r, 0.8 * 0.325_f64.sqrt()); } #[test] @@ -761,6 +1271,32 @@ mod tests { }; let (r, _, _) = g.apply_linear(0.0, 0.0, 0.0); approx(r, 0.1); + let (white, _, _) = g.apply_linear(1.0, 0.0, 0.0); + approx(white, 1.0); + } + + #[test] + fn color_grade_rejects_non_finite_and_zero_gamma() { + let zero_gamma = ColorGrade { + lift_gamma_gain: LiftGammaGain { + gamma: Rgb::new(0.0, 1.0, 1.0), + ..Default::default() + }, + ..Default::default() + }; + assert_eq!( + zero_gamma.validate().unwrap_err().to_string(), + "liftGammaGain.gamma.r must be finite and within (0, 4]" + ); + + let non_finite = ColorGrade { + exposure: f64::NAN, + ..Default::default() + }; + assert_eq!( + non_finite.validate().unwrap_err().to_string(), + "exposure must be finite and within [-5, 5]" + ); } // --- ColorGrade serde --- @@ -778,9 +1314,18 @@ mod tests { }, contrast: 0.3, saturation: 1.2, + hsl_secondary: Some(HslSecondary { + hue_center: 0.98, + hue_width: 0.2, + feather: 0.05, + hue_shift: 0.1, + saturation: -0.2, + lightness: 0.05, + }), }; let json = serde_json::to_string(&g).unwrap(); assert!(json.contains("\"liftGammaGain\"")); + assert!(json.contains("\"hslSecondary\"")); assert!(json.contains("\"exposure\":0.5")); let back: ColorGrade = serde_json::from_str(&json).unwrap(); assert_eq!(g, back); @@ -792,6 +1337,42 @@ mod tests { assert!(g.is_identity()); } + #[test] + fn hsl_secondary_wraps_red_and_isolates_other_hues() { + let grade = ColorGrade { + hsl_secondary: Some(HslSecondary { + hue_center: 0.98, + hue_width: 0.16, + feather: 0.04, + hue_shift: 0.2, + ..Default::default() + }), + ..Default::default() + }; + grade.validate().unwrap(); + let red = grade.apply_linear(1.0, 0.0, 0.0); + assert!( + red.1 > 0.2 || red.2 > 0.2, + "wrapped red must rotate: {red:?}" + ); + let green = grade.apply_linear(0.0, 1.0, 0.0); + approx(green.0, 0.0); + approx(green.1, 1.0); + approx(green.2, 0.0); + + let invalid = ColorGrade { + hsl_secondary: Some(HslSecondary { + hue_width: 0.0, + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + invalid.validate().unwrap_err().to_string(), + "hslSecondary.hueWidth must be finite and within (0, 1]" + ); + } + #[test] fn color_grade_partial_decode_keeps_other_defaults() { let g: ColorGrade = serde_json::from_str(r#"{"exposure":1.0}"#).unwrap(); @@ -901,6 +1482,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; approx(m.coverage(0.5, 0.5), 1.0); // center approx(m.coverage(0.5, 0.9), 0.0); // outside radius @@ -915,6 +1497,7 @@ mod tests { }, feather: 0.0, invert: true, + ..Mask::default() }; approx(m.coverage(0.5, 0.5), 0.0); // center now masked out approx(m.coverage(0.5, 0.9), 1.0); // outside now covered @@ -929,6 +1512,7 @@ mod tests { }, feather: 0.1, invert: false, + ..Mask::default() }; // Exactly on the boundary -> ~0.5 coverage. let c = m.coverage(0.7, 0.5); @@ -947,6 +1531,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; approx(m.coverage(0.8, 0.5), 1.0); // +normal side covered approx(m.coverage(0.2, 0.5), 0.0); // -normal side not @@ -964,6 +1549,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; approx(m.coverage(0.5, 0.3), 1.0); // inside the triangle approx(m.coverage(0.05, 0.05), 0.0); // outside (a corner region) @@ -977,6 +1563,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; approx(m.coverage(0.5, 0.5), 0.0); } @@ -990,6 +1577,7 @@ mod tests { }, feather: 0.05, invert: true, + ..Mask::default() }; let json = serde_json::to_string(&m).unwrap(); assert!(json.contains("\"kind\":\"circle\"")); @@ -1007,6 +1595,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; let json = serde_json::to_string(&m).unwrap(); assert!(json.contains("\"kind\":\"linear\"")); @@ -1026,6 +1615,7 @@ mod tests { }, feather: 0.0, invert: false, + ..Mask::default() }; let json = serde_json::to_string(&m).unwrap(); assert!(json.contains("\"kind\":\"poly\"")); @@ -1033,28 +1623,68 @@ mod tests { assert_eq!(m, back); } + #[test] + fn mask_transform_is_non_destructive_and_roundtrips() { + let base = Mask { + shape: MaskShape::Circle { + center: Point2::new(0.5, 0.5), + radius: Point2::new(0.1, 0.2), + }, + ..Mask::default() + }; + let transformed = Mask { + transform: MaskTransform { + offset: Point2::new(0.2, -0.1), + scale: Point2::new(2.0, 0.5), + rotation_degrees: 90.0, + }, + ..base.clone() + }; + + // The authored shape stays unchanged while its whole-mask transform + // moves the local center (0.5, 0.5) to display point (0.7, 0.4). + assert_eq!(transformed.shape, base.shape); + approx(transformed.coverage(0.7, 0.4), 1.0); + approx(transformed.coverage(0.5, 0.5), 0.0); + + let json = serde_json::to_string(&transformed).unwrap(); + assert!(json.contains("\"transform\"")); + assert!(json.contains("\"rotationDegrees\":90.0")); + assert_eq!(serde_json::from_str::(&json).unwrap(), transformed); + + // Legacy/default projects do not gain noisy transform payloads and + // deserialize to the identity transform. + let default_json = serde_json::to_string(&base).unwrap(); + assert!(!default_json.contains("\"transform\"")); + let legacy: Mask = serde_json::from_str( + r#"{"shape":{"kind":"circle","center":{"x":0.5,"y":0.5},"radius":{"x":0.1,"y":0.2}},"feather":0.0,"invert":false}"#, + ) + .unwrap(); + assert!(legacy.transform.is_identity()); + } + // --- Effect --- #[test] fn effect_new_is_enabled_no_params() { - let e = Effect::new("gaussianBlur"); - assert_eq!(e.name, "gaussianBlur"); + let e = Effect::new("grayscale"); + assert_eq!(e.name, "grayscale"); assert!(e.enabled); assert!(e.params.is_empty()); - approx(e.param("radius", 3.0), 3.0); // default fallback + approx(e.registered_param("amount").unwrap(), 1.0); } #[test] fn effect_with_param_and_read() { - let e = Effect::new("glow").with_param("intensity", 0.8); - approx(e.param("intensity", 0.0), 0.8); + let e = Effect::new("sepia").with_param("amount", 0.8); + approx(e.registered_param("amount").unwrap(), 0.8); } #[test] fn effect_roundtrip_with_params() { - let e = Effect::new("sharpen").with_param("amount", 0.5); + let e = Effect::new("invert").with_param("amount", 0.5); let json = serde_json::to_string(&e).unwrap(); - assert!(json.contains("\"name\":\"sharpen\"")); + assert!(json.contains("\"name\":\"invert\"")); assert!(json.contains("\"amount\":0.5")); let back: Effect = serde_json::from_str(&json).unwrap(); assert_eq!(e, back); diff --git a/crates/opentake-domain/src/lib.rs b/crates/opentake-domain/src/lib.rs index 18bb40c8..c687ea3a 100644 --- a/crates/opentake-domain/src/lib.rs +++ b/crates/opentake-domain/src/lib.rs @@ -20,43 +20,59 @@ //! Zero IO, pure logic, fully unit-testable. The only runtime dependency is //! `serde`; persistence-side UUID repair belongs to `opentake-project`. +pub mod audio; pub mod caption_sync; pub mod clip; pub mod clip_type; mod clip_wire; pub mod grade; pub mod keyframe; +pub mod lut; pub mod media; pub mod signal; pub mod split; +pub mod stabilization; pub mod subtitle_export; pub mod text; mod text_wire; pub mod timeline; pub mod transform; +pub mod transition; // Flat re-export of the public domain API for ergonomic downstream use. +pub use audio::{AudioDenoise, DenoiseMode, LoudnessNormalization}; pub use caption_sync::{caption_group_ids, clips_in_group, sync_caption_group_style}; -pub use clip::{Clip, FadeEdge, KeyframeTrackWireField, KeyframeValueWireShape, VolumeScale}; +pub use clip::{ + CaptionTranslationInput, Clip, FadeEdge, KeyframeTrackWireField, KeyframeValueWireShape, + VolumeScale, +}; pub use clip_type::ClipType; pub use grade::{ - chroma_cb_cr, luma709, smoothstep01, ChromaKey, ColorGrade, Effect, LiftGammaGain, Mask, - MaskShape, Point2, Rgb, + chroma_cb_cr, effect_registry, luma709, smoothstep01, validate_effect_chain, ChromaKey, + ColorGrade, ColorGradeValidationError, ColorMatchInput, Effect, EffectDescriptor, + EffectParameterDescriptor, EffectValidationError, HslSecondary, LiftGammaGain, Mask, MaskShape, + MaskTransform, Point2, Rgb, MAX_EFFECTS_PER_CLIP, MAX_MASKS_PER_CLIP, MAX_POLYGON_MASK_POINTS, }; pub use keyframe::{ smoothstep, split_keyframe_track, AnimPair, AnimatableProperty, Interpolation, Keyframe, KeyframeInterpolatable, KeyframeTrack, }; +pub use lut::{CubeLut, CubeLutError, LutReference, LutReferenceValidationError}; pub use media::{ - GenerationInput, GenerationJobStatus, GenerationStatus, MediaAsset, MediaFolder, MediaManifest, - MediaManifestEntry, MediaResolver, MediaSource, + GenerationInput, GenerationJobStatus, GenerationStatus, MediaAsset, MediaColorMetadata, + MediaFolder, MediaManifest, MediaManifestEntry, MediaProxy, MediaResolver, MediaSource, }; pub use signal::{ ContextSignal, EditingSkeleton, EditingStage, StageGuidance, TrackHint, TrackRole, TrackRoleAssignment, VideoType, }; pub use split::split_clip; +pub use stabilization::{StabilizationKeyframe, StabilizationTrack, StabilizationTransform}; pub use subtitle_export::{collect_caption_cues, export_srt, export_vtt, SubtitleCue}; pub use text::{Fill, Rgba, Shadow, TextAlignment, TextLayout, TextStyle}; -pub use timeline::{ClipLocation, Timeline, Track}; +pub use timeline::{ + ClipLocation, NestedSequence, ScriptAssemblyPlan, ScriptAssemblySegment, Timeline, Track, + VoiceModelRecord, +}; pub use transform::{Crop, CropAspectLock, Point, Transform}; +pub use transition::{Transition, TransitionKind}; diff --git a/crates/opentake-domain/src/lut.rs b/crates/opentake-domain/src/lut.rs new file mode 100644 index 00000000..b1e29f13 --- /dev/null +++ b/crates/opentake-domain/src/lut.rs @@ -0,0 +1,350 @@ +//! Pure domain model and bounded parser for project-managed 3D `.cube` LUTs. +//! +//! File I/O belongs to the desktop/project layers. This module accepts an +//! already-bounded byte slice, validates the complete table, and exposes only +//! finite data suitable for GPU upload. + +use serde::{Deserialize, Serialize}; + +/// Authored reference persisted on a clip. The content hash is also the only +/// allowed storage key; no ambient source path is retained in project JSON. +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LutReference { + pub id: String, + pub name: String, + pub intensity: f64, +} + +impl LutReference { + pub fn new( + id: impl Into, + name: impl Into, + intensity: f64, + ) -> Result { + let reference = Self { + id: id.into(), + name: name.into(), + intensity, + }; + reference.validate()?; + Ok(reference) + } + + pub fn validate(&self) -> Result<(), LutReferenceValidationError> { + if self.id.len() != 64 + || !self + .id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(LutReferenceValidationError::InvalidId); + } + if self.name.is_empty() || self.name.len() > 128 || self.name.chars().any(char::is_control) + { + return Err(LutReferenceValidationError::InvalidName); + } + if !self.intensity.is_finite() || !(0.0..=1.0).contains(&self.intensity) { + return Err(LutReferenceValidationError::InvalidIntensity); + } + Ok(()) + } + + /// Canonical bundle-relative location. It is derived, never deserialized. + pub fn relative_path(&self) -> String { + format!("media/luts/{}.cube", self.id) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum LutReferenceValidationError { + InvalidId, + InvalidName, + InvalidIntensity, +} + +impl std::fmt::Display for LutReferenceValidationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::InvalidId => "id must be a lowercase 64-character SHA-256 digest", + Self::InvalidName => "name must contain 1..=128 bytes without control characters", + Self::InvalidIntensity => "intensity must be finite and within [0, 1]", + }) + } +} + +impl std::error::Error for LutReferenceValidationError {} + +/// Fully validated 3D table in `.cube` red-fastest order. +#[derive(Clone, PartialEq, Debug)] +pub struct CubeLut { + title: Option, + size: u32, + domain_min: [f32; 3], + domain_max: [f32; 3], + table: Vec<[f32; 3]>, +} + +impl CubeLut { + /// Hard read/parse ceiling for an untrusted input file. + pub const MAX_BYTES: usize = 4 * 1024 * 1024; + + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > Self::MAX_BYTES { + return Err(CubeLutError::TooLarge { + actual: bytes.len(), + maximum: Self::MAX_BYTES, + }); + } + let text = std::str::from_utf8(bytes).map_err(|_| CubeLutError::InvalidUtf8)?; + let mut title = None; + let mut size = None; + let mut domain_min = None; + let mut domain_max = None; + let mut table = Vec::new(); + + for (zero_line, raw) in text.lines().enumerate() { + let line_number = zero_line + 1; + let line = raw.split('#').next().unwrap_or_default().trim(); + if line.is_empty() { + continue; + } + let fields = line.split_whitespace().collect::>(); + match fields[0] { + "TITLE" => { + if title.is_some() { + return Err(CubeLutError::DuplicateDirective { directive: "TITLE" }); + } + let value = line["TITLE".len()..].trim().trim_matches('"'); + if value.is_empty() || value.len() > 128 || value.chars().any(char::is_control) + { + return Err(CubeLutError::InvalidDirective { + line: line_number, + directive: "TITLE", + }); + } + title = Some(value.to_owned()); + } + "LUT_3D_SIZE" => { + if size.is_some() { + return Err(CubeLutError::DuplicateDirective { + directive: "LUT_3D_SIZE", + }); + } + if fields.len() != 2 { + return Err(CubeLutError::InvalidDirective { + line: line_number, + directive: "LUT_3D_SIZE", + }); + } + let parsed = + fields[1] + .parse::() + .map_err(|_| CubeLutError::InvalidDirective { + line: line_number, + directive: "LUT_3D_SIZE", + })?; + if !matches!(parsed, 17 | 33) { + return Err(CubeLutError::UnsupportedSize(parsed)); + } + size = Some(parsed); + table.reserve(parsed as usize * parsed as usize * parsed as usize); + } + "DOMAIN_MIN" => { + if domain_min.is_some() { + return Err(CubeLutError::DuplicateDirective { + directive: "DOMAIN_MIN", + }); + } + domain_min = Some(parse_triplet(&fields, line_number, "DOMAIN_MIN")?); + } + "DOMAIN_MAX" => { + if domain_max.is_some() { + return Err(CubeLutError::DuplicateDirective { + directive: "DOMAIN_MAX", + }); + } + domain_max = Some(parse_triplet(&fields, line_number, "DOMAIN_MAX")?); + } + directive if directive.as_bytes()[0].is_ascii_alphabetic() => { + return Err(CubeLutError::UnsupportedDirective { + line: line_number, + directive: directive.to_owned(), + }); + } + _ => { + if size.is_none() { + return Err(CubeLutError::TableBeforeSize { line: line_number }); + } + let value = parse_triplet(&fields, line_number, "table row")?; + if value.iter().any(|channel| channel.abs() > 16.0) { + return Err(CubeLutError::OutOfRangeValue { line: line_number }); + } + table.push(value); + } + } + } + + let size = size.ok_or(CubeLutError::MissingSize)?; + let expected = size as usize * size as usize * size as usize; + if table.len() != expected { + return Err(CubeLutError::WrongTableLength { + expected, + actual: table.len(), + }); + } + let domain_min = domain_min.unwrap_or([0.0; 3]); + let domain_max = domain_max.unwrap_or([1.0; 3]); + if (0..3).any(|channel| domain_min[channel] >= domain_max[channel]) { + return Err(CubeLutError::InvalidDomain); + } + Ok(Self { + title, + size, + domain_min, + domain_max, + table, + }) + } + + pub fn title(&self) -> Option<&str> { + self.title.as_deref() + } + + pub fn size(&self) -> u32 { + self.size + } + + pub fn domain_min(&self) -> [f32; 3] { + self.domain_min + } + + pub fn domain_max(&self) -> [f32; 3] { + self.domain_max + } + + pub fn table(&self) -> &[[f32; 3]] { + &self.table + } +} + +fn parse_triplet( + fields: &[&str], + line: usize, + directive: &'static str, +) -> Result<[f32; 3], CubeLutError> { + if fields.len() != 4 && directive != "table row" + || fields.len() != 3 && directive == "table row" + { + return Err(CubeLutError::InvalidDirective { line, directive }); + } + let offset = usize::from(directive != "table row"); + let mut value = [0.0; 3]; + for channel in 0..3 { + value[channel] = fields[channel + offset] + .parse::() + .map_err(|_| CubeLutError::InvalidNumber { line })?; + if !value[channel].is_finite() { + return Err(CubeLutError::InvalidNumber { line }); + } + } + Ok(value) +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum CubeLutError { + TooLarge { + actual: usize, + maximum: usize, + }, + InvalidUtf8, + DuplicateDirective { + directive: &'static str, + }, + InvalidDirective { + line: usize, + directive: &'static str, + }, + UnsupportedDirective { + line: usize, + directive: String, + }, + UnsupportedSize(u32), + TableBeforeSize { + line: usize, + }, + MissingSize, + InvalidNumber { + line: usize, + }, + OutOfRangeValue { + line: usize, + }, + InvalidDomain, + WrongTableLength { + expected: usize, + actual: usize, + }, +} + +impl std::fmt::Display for CubeLutError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooLarge { actual, maximum } => { + write!(formatter, "LUT is {actual} bytes; maximum is {maximum}") + } + Self::InvalidUtf8 => formatter.write_str("LUT is not valid UTF-8 text"), + Self::DuplicateDirective { directive } => { + write!(formatter, "duplicate {directive} directive") + } + Self::InvalidDirective { line, directive } => { + write!(formatter, "invalid {directive} on line {line}") + } + Self::UnsupportedDirective { line, directive } => write!( + formatter, + "unsupported directive {directive} on line {line}" + ), + Self::UnsupportedSize(size) => { + write!(formatter, "unsupported LUT size {size}; expected 17 or 33") + } + Self::TableBeforeSize { line } => { + write!(formatter, "table row before LUT_3D_SIZE on line {line}") + } + Self::MissingSize => formatter.write_str("missing LUT_3D_SIZE"), + Self::InvalidNumber { line } => { + write!(formatter, "invalid finite number on line {line}") + } + Self::OutOfRangeValue { line } => { + write!(formatter, "table value outside [-16, 16] on line {line}") + } + Self::InvalidDomain => { + formatter.write_str("each DOMAIN_MIN channel must be less than DOMAIN_MAX") + } + Self::WrongTableLength { expected, actual } => write!( + formatter, + "LUT table has {actual} rows; expected {expected}" + ), + } + } +} + +impl std::error::Error for CubeLutError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_duplicate_metadata_and_non_finite_domain() { + let duplicate = b"LUT_3D_SIZE 17\nLUT_3D_SIZE 17\n"; + assert!(matches!( + CubeLut::parse(duplicate), + Err(CubeLutError::DuplicateDirective { .. }) + )); + let non_finite = b"LUT_3D_SIZE 17\nDOMAIN_MIN NaN 0 0\n"; + assert!(matches!( + CubeLut::parse(non_finite), + Err(CubeLutError::InvalidNumber { .. }) + )); + } +} diff --git a/crates/opentake-domain/src/media.rs b/crates/opentake-domain/src/media.rs index 36c27d15..09622812 100644 --- a/crates/opentake-domain/src/media.rs +++ b/crates/opentake-domain/src/media.rs @@ -50,6 +50,54 @@ pub enum MediaSource { Project { relative_path: String }, } +/// Source color signalling retained from the first playable video stream. +/// Values use FFmpeg's stable tokens (`bt709`, `bt2020`, `smpte2084`, +/// `arib-std-b67`, ...). Keeping the original tokens makes older/newer codecs +/// forward-compatible while helpers can still identify the HDR transfers that +/// require explicit tone mapping in the current SDR compositor. +#[derive(Clone, PartialEq, Eq, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaColorMetadata { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub primaries: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transfer: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matrix: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range: Option, +} + +impl MediaColorMetadata { + pub fn is_hdr(&self) -> bool { + self.transfer.as_deref().is_some_and(|transfer| { + matches!( + transfer.to_ascii_lowercase().as_str(), + "smpte2084" | "pq" | "arib-std-b67" | "hlg" + ) + }) + } + + pub fn is_empty(&self) -> bool { + self.primaries.is_none() + && self.transfer.is_none() + && self.matrix.is_none() + && self.range.is_none() + } +} + +/// Project-local low-resolution media used only for interactive playback. +/// Export always resolves [`MediaManifestEntry::source`]. The source digest +/// prevents a stale proxy being paired with bytes that changed in place. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaProxy { + pub relative_path: String, + pub source_sha256: String, + pub width: u32, + pub height: u32, +} + /// Full serializable input snapshot for a generated asset. 1:1 port of /// `GenerationInput`. `prompt` / `model` / `duration` / `aspect_ratio` are /// required upstream; everything else is optional. @@ -150,6 +198,13 @@ pub struct GenerationInput { /// recorded once in the generation log when a provider supplies it. #[serde(default, skip_serializing_if = "Option::is_none")] pub estimated_cost_credits: Option, + /// Explicit user-consent record supplied for identity-bearing generation. + /// This is an opaque local audit id, never a credential. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub consent_id: Option, + /// SHA-256 of the canonical, non-secret provider request inputs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_hash: Option, } /// Serializable manifest entry. 1:1 port of `MediaManifestEntry`. @@ -173,6 +228,10 @@ pub struct MediaManifestEntry { #[serde(default, skip_serializing_if = "Option::is_none")] pub has_audio: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub folder_id: Option, #[serde( rename = "cachedRemoteURL", @@ -188,6 +247,17 @@ pub struct MediaManifestEntry { pub cached_remote_url_expires_at: Option, } +impl MediaManifestEntry { + /// Generated local matting derivatives contain straight RGBA from FFmpeg's + /// ProRes 4444 decoder. The render adapters use this non-secret provenance + /// to request one premultiplication before blending. + pub fn carries_straight_alpha(&self) -> bool { + self.generation_input.as_ref().is_some_and(|input| { + input.provider.as_deref() == Some("opentake-matting") && input.model.starts_with("rvm-") + }) + } +} + /// A media library folder. 1:1 port of `MediaFolder`. #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -342,6 +412,9 @@ impl MediaManifest { } } +/// Decode a persisted manifest without confusing the current constructor +/// version with the legacy wire fallback: an omitted version means schema 1, +/// while every explicitly stored version is retained verbatim. impl<'de> Deserialize<'de> for MediaManifest { fn deserialize(deserializer: D) -> Result where @@ -446,6 +519,10 @@ pub struct MediaAsset { #[serde(default)] pub has_audio: bool, #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub generation_input: Option, #[serde(default)] pub generation_status: GenerationStatus, @@ -487,6 +564,8 @@ impl MediaAsset { source_height: None, source_fps: None, has_audio: kind == ClipType::Video, + color: None, + proxy: None, generation_input: None, generation_status: GenerationStatus::None, folder_id: None, @@ -509,6 +588,8 @@ impl MediaAsset { source_height: entry.source_height, source_fps: entry.source_fps, has_audio: entry.has_audio.unwrap_or(false), + color: entry.color.clone(), + proxy: entry.proxy.clone(), generation_input: entry.generation_input.clone(), generation_status: match entry .generation_input @@ -602,6 +683,8 @@ impl MediaAsset { source_height: self.source_height, source_fps: self.source_fps, has_audio: Some(self.has_audio), + color: self.color.clone(), + proxy: self.proxy.clone(), folder_id: self.folder_id.clone(), cached_remote_url: fresh, cached_remote_url_expires_at: expires, @@ -719,6 +802,8 @@ mod tests { source_height: Some(1080), source_fps: Some(30.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: None, cached_remote_url: Some("https://x".into()), cached_remote_url_expires_at: Some(700_000_000.0), @@ -854,6 +939,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -871,6 +958,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -1010,6 +1099,8 @@ mod tests { source_height: Some(720), source_fps: Some(24.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: Some("f1".into()), cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-domain/src/split.rs b/crates/opentake-domain/src/split.rs index 68577607..0f9c1c3c 100644 --- a/crates/opentake-domain/src/split.rs +++ b/crates/opentake-domain/src/split.rs @@ -43,6 +43,7 @@ pub fn split_clip(clip: &Clip, at_frame: i32, right_id: impl Into) -> Op left.trim_end_frame = clip.trim_end_frame + right_source; } left.fade_out_frames = 0; + left.loudness_normalization = None; left.clamp_fades_to_duration(); let mut right = clip.clone(); @@ -55,6 +56,7 @@ pub fn split_clip(clip: &Clip, at_frame: i32, right_id: impl Into) -> Op right.trim_start_frame = clip.trim_start_frame + left_source; } right.fade_in_frames = 0; + right.loudness_normalization = None; right.clamp_fades_to_duration(); // Split every animatable track at the cut, inserting a boundary keyframe so diff --git a/crates/opentake-domain/src/stabilization.rs b/crates/opentake-domain/src/stabilization.rs new file mode 100644 index 00000000..bf77bc13 --- /dev/null +++ b/crates/opentake-domain/src/stabilization.rs @@ -0,0 +1,188 @@ +//! Persisted, editable video-stabilization solution. +//! +//! The track is deliberately separate from the user's authored position/scale/ +//! rotation keyframes. Renderers compose both tracks, so applying or resetting +//! stabilization never destroys manual animation or source media identity. + +use serde::{Deserialize, Serialize}; + +fn default_strength() -> f64 { + 1.0 +} + +fn default_model_version() -> u32 { + 1 +} + +#[derive(Clone, Copy, PartialEq, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StabilizationTransform { + pub translation_x: f64, + pub translation_y: f64, + pub rotation_degrees: f64, +} + +#[derive(Clone, Copy, PartialEq, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StabilizationKeyframe { + pub frame: i32, + pub translation_x: f64, + pub translation_y: f64, + pub rotation_degrees: f64, +} + +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StabilizationTrack { + pub model: String, + #[serde(default = "default_model_version")] + pub model_version: u32, + pub source_identity: String, + #[serde(default = "default_strength")] + pub strength: f64, + #[serde(default)] + pub crop_margin: f64, + #[serde(default)] + pub keyframes: Vec, +} + +impl StabilizationTrack { + /// Linearly sample the correction track at one clip-relative frame. + pub fn sample(&self, frame: i32) -> StabilizationTransform { + let Some(first) = self.keyframes.first() else { + return StabilizationTransform::default(); + }; + let strength = self.strength.clamp(0.0, 1.0); + let raw = if frame <= first.frame { + keyframe_transform(*first) + } else if let Some(last) = self.keyframes.last().filter(|last| frame >= last.frame) { + keyframe_transform(*last) + } else { + let pair = self + .keyframes + .windows(2) + .find(|pair| frame >= pair[0].frame && frame <= pair[1].frame) + .expect("a sorted stabilization track covers an interior sample"); + let span = (pair[1].frame - pair[0].frame).max(1) as f64; + let t = (frame - pair[0].frame) as f64 / span; + StabilizationTransform { + translation_x: lerp(pair[0].translation_x, pair[1].translation_x, t), + translation_y: lerp(pair[0].translation_y, pair[1].translation_y, t), + rotation_degrees: lerp(pair[0].rotation_degrees, pair[1].rotation_degrees, t), + } + }; + StabilizationTransform { + translation_x: raw.translation_x * strength, + translation_y: raw.translation_y * strength, + rotation_degrees: raw.rotation_degrees * strength, + } + } + + /// Conservative uniform zoom needed to keep every output corner covered. + /// `aspect_ratio` is output width / height. + pub fn crop_scale(&self, aspect_ratio: f64) -> f64 { + let aspect = aspect_ratio.max(1e-6); + let required = self + .keyframes + .iter() + .map(|keyframe| { + let correction = self.sample(keyframe.frame); + coverage_scale(correction, aspect) + }) + .fold(1.0_f64, f64::max); + required + self.crop_margin.max(0.0) * 2.0 + } + + pub fn guarantees_coverage(&self, aspect_ratio: f64) -> bool { + let scale = self.crop_scale(aspect_ratio); + self.keyframes.iter().all(|keyframe| { + scale + 1e-12 >= coverage_scale(self.sample(keyframe.frame), aspect_ratio.max(1e-6)) + }) + } + + pub fn validate(&self) -> Result<(), String> { + if self.model.trim().is_empty() || self.model_version == 0 { + return Err("stabilization model and version are required".to_string()); + } + if self.source_identity.trim().is_empty() { + return Err("stabilization source identity is required".to_string()); + } + if !(0.0..=1.0).contains(&self.strength) || !self.strength.is_finite() { + return Err("stabilization strength must be finite and within 0..=1".to_string()); + } + if !(0.0..=0.5).contains(&self.crop_margin) || !self.crop_margin.is_finite() { + return Err("stabilization crop margin must be finite and within 0..=0.5".to_string()); + } + if self.keyframes.len() < 2 { + return Err("stabilization requires at least two keyframes".to_string()); + } + let mut previous = None; + for keyframe in &self.keyframes { + if previous.is_some_and(|frame| keyframe.frame <= frame) { + return Err("stabilization keyframes must be strictly increasing".to_string()); + } + if !keyframe.translation_x.is_finite() + || !keyframe.translation_y.is_finite() + || !keyframe.rotation_degrees.is_finite() + { + return Err("stabilization keyframes must be finite".to_string()); + } + previous = Some(keyframe.frame); + } + Ok(()) + } +} + +fn keyframe_transform(keyframe: StabilizationKeyframe) -> StabilizationTransform { + StabilizationTransform { + translation_x: keyframe.translation_x, + translation_y: keyframe.translation_y, + rotation_degrees: keyframe.rotation_degrees, + } +} + +fn lerp(a: f64, b: f64, t: f64) -> f64 { + a + (b - a) * t.clamp(0.0, 1.0) +} + +fn coverage_scale(correction: StabilizationTransform, aspect: f64) -> f64 { + let radians = correction.rotation_degrees.to_radians(); + let (sin, cos) = radians.sin_cos(); + let sin = sin.abs(); + let cos = cos.abs(); + let translation_x = correction.translation_x.abs(); + let translation_y = correction.translation_y.abs(); + let cover_width = cos + sin / aspect + 2.0 * (translation_x + translation_y / aspect); + let cover_height = cos + sin * aspect + 2.0 * (translation_y + translation_x * aspect); + cover_width.max(cover_height).max(1.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sampling_scales_correction_by_editable_strength() { + let track = StabilizationTrack { + model: "test".into(), + model_version: 1, + source_identity: "asset".into(), + strength: 0.5, + crop_margin: 0.0, + keyframes: vec![ + StabilizationKeyframe::default(), + StabilizationKeyframe { + frame: 10, + translation_x: 0.2, + translation_y: -0.1, + rotation_degrees: 4.0, + }, + ], + }; + let sample = track.sample(5); + assert!((sample.translation_x - 0.05).abs() < 1e-12); + assert!((sample.translation_y + 0.025).abs() < 1e-12); + assert!((sample.rotation_degrees - 1.0).abs() < 1e-12); + assert!(track.guarantees_coverage(16.0 / 9.0)); + } +} diff --git a/crates/opentake-domain/src/timeline.rs b/crates/opentake-domain/src/timeline.rs index 0759c8c0..3a2ac0eb 100644 --- a/crates/opentake-domain/src/timeline.rs +++ b/crates/opentake-domain/src/timeline.rs @@ -6,12 +6,59 @@ //! boundary owns UUID repair because it retains the raw JSON needed to //! distinguish that placeholder from an explicitly encoded empty string. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; use crate::clip::Clip; use crate::clip_type::ClipType; +use crate::transition::TransitionKind; + +/// One reviewed script-to-video segment. Exact media identities and frame +/// duration are persisted before assembly so applying never repeats creative +/// selection or frame arithmetic. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptAssemblySegment { + pub script: String, + pub media_ref: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub narration_media_ref: Option, + pub duration_frames: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transition: Option, +} + +/// Persisted, reviewable assembly plan. `plan_hash` is the SHA-256 of the +/// canonical segment payload; planner provenance is deliberately non-secret. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptAssemblyPlan { + pub id: String, + pub plan_hash: String, + pub planner: String, + pub planner_version: u32, + pub start_frame: i32, + pub segments: Vec, +} + +/// Durable non-secret record for a provider-hosted cloned voice. Provider +/// credentials and reference audio bytes never enter the project document. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceModelRecord { + pub id: String, + pub provider: String, + pub provider_voice_id: String, + pub model: String, + pub consent_id: String, + pub source_audio_asset_id: String, + pub source_audio_sha256: String, + pub request_hash: String, + pub voice_name: String, + #[serde(default)] + pub revoked: bool, +} /// Clip location inside track storage. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -50,10 +97,45 @@ pub struct Timeline { pub height: i32, #[serde(default)] pub settings_configured: bool, + /// Editable child timelines referenced by clips through + /// `Clip::nested_sequence_id`. The registry lives on the root timeline so + /// every reference has one stable identity and graph cycles are detectable. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub nested_sequences: Vec, + /// Reviewed script assembly plans. Bounded by the command layer and + /// ignored by render/export until explicitly applied. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub script_assembly_plans: Vec, + /// Consent-bearing provider voice identities. Revoked records remain for + /// audit and are rejected by every generation path. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub voice_models: Vec, #[serde(default)] pub tracks: Vec, } +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NestedSequence { + pub id: String, + pub name: String, + pub timeline: Timeline, +} + +impl NestedSequence { + pub const WIRE_FIELDS: &'static [&'static str] = &["id", "name", "timeline"]; + pub const ID_WIRE_FIELD: &'static str = "id"; + pub const TIMELINE_WIRE_FIELD: &'static str = "timeline"; + + pub fn new(id: impl Into, name: impl Into, timeline: Timeline) -> Self { + Self { + id: id.into(), + name: name.into(), + timeline, + } + } +} + impl Default for Timeline { fn default() -> Self { Timeline { @@ -61,6 +143,9 @@ impl Default for Timeline { width: 1920, height: 1080, settings_configured: false, + nested_sequences: Vec::new(), + script_assembly_plans: Vec::new(), + voice_models: Vec::new(), tracks: Vec::new(), } } @@ -68,6 +153,9 @@ impl Default for Timeline { impl Timeline { pub const TRACKS_WIRE_FIELD: &'static str = "tracks"; + pub const NESTED_SEQUENCES_WIRE_FIELD: &'static str = "nestedSequences"; + pub const SCRIPT_ASSEMBLY_PLANS_WIRE_FIELD: &'static str = "scriptAssemblyPlans"; + pub const VOICE_MODELS_WIRE_FIELD: &'static str = "voiceModels"; pub fn new() -> Self { Timeline::default() @@ -77,6 +165,121 @@ impl Timeline { pub fn total_frames(&self) -> i32 { self.tracks.iter().map(|t| t.end_frame()).max().unwrap_or(0) } + + /// Validate unique sequence identities, every reference, and graph cycles. + /// This is a pure preflight used before edits are committed or plans built. + pub fn validate_nested_sequences(&self) -> Result<(), String> { + let mut registry = HashMap::new(); + for sequence in &self.nested_sequences { + if sequence.id.is_empty() { + return Err("nested sequence id must not be empty".to_string()); + } + if registry.insert(sequence.id.as_str(), sequence).is_some() { + return Err(format!("duplicate nested sequence id: {}", sequence.id)); + } + if !sequence.timeline.nested_sequences.is_empty() { + return Err(format!( + "nested sequence {} contains a nestedSequences registry; child references must use the root registry", + sequence.id + )); + } + } + + // Several cross-cutting consumers (text resolution, selection, and + // edit commands) address clips by id without a sequence namespace. + // Once a project has nested timelines, ids therefore must be unique + // across the entire stored graph rather than only inside one track. + let uses_nested_graph = !self.nested_sequences.is_empty() + || self + .tracks + .iter() + .flat_map(|track| &track.clips) + .any(|clip| clip.nested_sequence_id.is_some()); + if uses_nested_graph { + let mut clip_ids = HashSet::new(); + for timeline in std::iter::once(self).chain( + self.nested_sequences + .iter() + .map(|sequence| &sequence.timeline), + ) { + for track in &timeline.tracks { + for clip in &track.clips { + if clip.id.is_empty() { + return Err( + "clip id must not be empty in a nested timeline graph".to_string() + ); + } + if !clip_ids.insert(clip.id.as_str()) { + return Err(format!( + "duplicate clip id in nested timeline graph: {}", + clip.id + )); + } + if clip.nested_sequence_id.is_some() + && (track.kind == ClipType::Audio + || clip.media_type != ClipType::Video + || !clip.media_ref.is_empty() + || clip.start_frame < 0 + || clip.duration_frames < 1 + || clip.trim_start_frame < 0) + { + return Err(format!( + "invalid compound clip representation: {}", + clip.id + )); + } + } + } + } + } + + fn references(timeline: &Timeline) -> impl Iterator { + timeline.tracks.iter().flat_map(|track| { + track + .clips + .iter() + .filter_map(|clip| clip.nested_sequence_id.as_deref()) + }) + } + + for reference in references(self) { + if !registry.contains_key(reference) { + return Err(format!("missing nested sequence reference: {reference}")); + } + } + + fn visit<'a>( + id: &'a str, + registry: &HashMap<&'a str, &'a NestedSequence>, + visiting: &mut Vec<&'a str>, + complete: &mut HashSet<&'a str>, + ) -> Result<(), String> { + if complete.contains(id) { + return Ok(()); + } + if let Some(index) = visiting.iter().position(|candidate| *candidate == id) { + let mut cycle = visiting[index..].to_vec(); + cycle.push(id); + return Err(format!("nested sequence cycle: {}", cycle.join(" -> "))); + } + let sequence = registry + .get(id) + .ok_or_else(|| format!("missing nested sequence reference: {id}"))?; + visiting.push(id); + for reference in references(&sequence.timeline) { + visit(reference, registry, visiting, complete)?; + } + visiting.pop(); + complete.insert(id); + Ok(()) + } + + let mut complete = HashSet::new(); + for sequence in &self.nested_sequences { + visit(&sequence.id, ®istry, &mut Vec::new(), &mut complete)?; + } + Ok(()) + } } fn default_sync_locked() -> bool { @@ -305,6 +508,55 @@ mod tests { assert!(back.settings_configured); } + #[test] + fn script_assembly_plan_roundtrips_and_legacy_timelines_default_empty() { + let mut timeline = Timeline::new(); + timeline.script_assembly_plans.push(ScriptAssemblyPlan { + id: "plan-1".into(), + plan_hash: "a".repeat(64), + planner: "opentake-script-assembly".into(), + planner_version: 1, + start_frame: 42, + segments: vec![ScriptAssemblySegment { + script: "Opening".into(), + media_ref: "visual".into(), + narration_media_ref: Some("voice".into()), + duration_frames: 30, + transition: Some(TransitionKind::CrossDissolve), + }], + }); + let json = serde_json::to_string(&timeline).unwrap(); + assert!(json.contains("\"scriptAssemblyPlans\"")); + assert_eq!(serde_json::from_str::(&json).unwrap(), timeline); + let legacy: Timeline = serde_json::from_str( + r#"{"fps":30,"width":1920,"height":1080,"settingsConfigured":true,"tracks":[]}"#, + ) + .unwrap(); + assert!(legacy.script_assembly_plans.is_empty()); + assert!(legacy.voice_models.is_empty()); + } + + #[test] + fn voice_model_record_roundtrips_without_secret_material() { + let mut timeline = Timeline::new(); + timeline.voice_models.push(VoiceModelRecord { + id: "voice-local-1".into(), + provider: "elevenlabs".into(), + provider_voice_id: "provider-voice-1".into(), + model: "eleven_multilingual_v2".into(), + consent_id: "consent-1".into(), + source_audio_asset_id: "audio-1".into(), + source_audio_sha256: "a".repeat(64), + request_hash: "b".repeat(64), + voice_name: "Narrator".into(), + revoked: false, + }); + let json = serde_json::to_string(&timeline).unwrap(); + assert!(json.contains("\"voiceModels\"")); + assert!(!json.contains("apiKey")); + assert_eq!(serde_json::from_str::(&json).unwrap(), timeline); + } + #[test] fn timeline_decode_defaults() { let tl: Timeline = serde_json::from_str("{}").unwrap(); @@ -326,6 +578,115 @@ mod tests { assert_eq!(tl, back); } + #[test] + fn nested_sequence_roundtrip_and_legacy_omission_are_stable() { + let legacy: Timeline = serde_json::from_str(r#"{"fps":24,"tracks":[]}"#).unwrap(); + assert!(legacy.nested_sequences.is_empty()); + assert!(!serde_json::to_string(&legacy) + .unwrap() + .contains("nestedSequences")); + + let mut child = Timeline::new(); + child + .tracks + .push(Track::new("child-track", ClipType::Video)); + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence-a", "Scene A", child)); + let encoded = serde_json::to_string(&root).unwrap(); + assert!(encoded.contains("\"nestedSequences\"")); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), root); + } + + #[test] + fn nested_sequence_validation_is_deterministic_and_fail_closed() { + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("", "Empty", Timeline::new())); + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "nested sequence id must not be empty" + ); + + root.nested_sequences = vec![ + NestedSequence::new("duplicate", "A", Timeline::new()), + NestedSequence::new("duplicate", "B", Timeline::new()), + ]; + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "duplicate nested sequence id: duplicate" + ); + + let mut missing_track = Track::new("root-track", ClipType::Video); + missing_track + .clips + .push(Clip::new_nested("compound", "missing", 0, 10)); + root.nested_sequences.clear(); + root.tracks = vec![missing_track]; + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "missing nested sequence reference: missing" + ); + + let mut a = Timeline::new(); + let mut a_track = Track::new("a-track", ClipType::Video); + a_track.clips.push(Clip::new_nested("a-to-b", "b", 0, 10)); + a.tracks.push(a_track); + let mut b = Timeline::new(); + let mut b_track = Track::new("b-track", ClipType::Video); + b_track.clips.push(Clip::new_nested("b-to-a", "a", 0, 10)); + b.tracks.push(b_track); + root.tracks.clear(); + root.nested_sequences = vec![ + NestedSequence::new("a", "A", a), + NestedSequence::new("b", "B", b), + ]; + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "nested sequence cycle: a -> b -> a" + ); + } + + #[test] + fn nested_sequence_validation_rejects_graph_wide_clip_id_collisions() { + let mut child = Timeline::new(); + let mut child_track = Track::new("child-track", ClipType::Video); + child_track.clips.push(clip("shared-id", 0, 10)); + child.tracks.push(child_track); + + let mut root = Timeline::new(); + let mut root_track = Track::new("root-track", ClipType::Video); + root_track.clips.push(clip("shared-id", 0, 10)); + root_track + .clips + .push(Clip::new_nested("compound", "sequence", 10, 10)); + root.tracks.push(root_track); + root.nested_sequences + .push(NestedSequence::new("sequence", "Scene", child)); + + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "duplicate clip id in nested timeline graph: shared-id" + ); + } + + #[test] + fn nested_sequence_validation_rejects_compound_on_audio_track() { + let mut root = Timeline::new(); + let mut track = Track::new("audio-track", ClipType::Audio); + track + .clips + .push(Clip::new_nested("compound", "sequence", 0, 10)); + root.tracks.push(track); + root.nested_sequences + .push(NestedSequence::new("sequence", "Scene", Timeline::new())); + + assert_eq!( + root.validate_nested_sequences().unwrap_err(), + "invalid compound clip representation: compound" + ); + } + #[test] fn clip_location_fields() { let loc = ClipLocation::new(2, 5); diff --git a/crates/opentake-domain/src/transition.rs b/crates/opentake-domain/src/transition.rs new file mode 100644 index 00000000..9281121f --- /dev/null +++ b/crates/opentake-domain/src/transition.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; + +/// Visual transition applied at the cut from one clip to its exact adjacent +/// successor. V1 intentionally starts with the lossless baseline required by +/// the product plan; additional shader-backed kinds can extend this enum later. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TransitionKind { + #[default] + CrossDissolve, +} + +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Transition { + /// Both sides are persisted so a transition cannot silently rebind when a + /// project is reordered. Empty is accepted only for legacy project files; + /// the next validated edit normalizes it to the containing clip id. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub from_clip_id: String, + pub to_clip_id: String, + pub kind: TransitionKind, + pub duration_frames: i32, +} diff --git a/crates/opentake-domain/tests/wire_schema.rs b/crates/opentake-domain/tests/wire_schema.rs index cfbd0d9f..ca230836 100644 --- a/crates/opentake-domain/tests/wire_schema.rs +++ b/crates/opentake-domain/tests/wire_schema.rs @@ -1,8 +1,10 @@ use std::collections::BTreeSet; use opentake_domain::{ - AnimPair, ChromaKey, Clip, ClipType, ColorGrade, Crop, Effect, Fill, Interpolation, Keyframe, - KeyframeTrack, Mask, Rgba, Shadow, TextAlignment, TextStyle, Track, Transform, + AnimPair, AudioDenoise, CaptionTranslationInput, ChromaKey, Clip, ClipType, ColorGrade, Crop, + DenoiseMode, Effect, Fill, Interpolation, Keyframe, KeyframeTrack, LoudnessNormalization, + LutReference, Mask, Rgba, Shadow, StabilizationKeyframe, StabilizationTrack, TextAlignment, + TextStyle, Track, Transform, Transition, TransitionKind, }; use serde::Serialize; @@ -73,8 +75,16 @@ fn full_clip() -> Clip { }; clip.link_group_id = Some("link".to_owned()); clip.caption_group_id = Some("caption".to_owned()); + clip.nested_sequence_id = Some("sequence".to_owned()); clip.text_content = Some("text".to_owned()); clip.text_style = Some(full_text_style()); + clip.caption_translation_input = Some(CaptionTranslationInput { + source_text: "source".into(), + source_locale: "en-US".into(), + target_locale: "zh-CN".into(), + provider: "wire".into(), + model: "wire-v1".into(), + }); clip.opacity_track = Some(KeyframeTrack::from_keyframes(vec![ Keyframe::with_interpolation(0, 0.5, Interpolation::Linear), ])); @@ -97,10 +107,50 @@ fn full_clip() -> Clip { }, )])); clip.volume_track = Some(KeyframeTrack::from_keyframes(vec![Keyframe::new(0, 0.9)])); + clip.loudness_normalization = Some(LoudnessNormalization { + target_lufs: -16.0, + true_peak_ceiling_dbtp: -1.0, + input_integrated_lufs: -24.0, + input_true_peak_dbtp: -12.0, + gain_db: 8.0, + output_integrated_lufs: -16.0, + output_true_peak_dbtp: -2.0, + }); + clip.audio_denoise = Some(AudioDenoise { + mode: DenoiseMode::Voice, + strength: 0.75, + preview_enabled: true, + }); clip.color_grade = Some(ColorGrade::default()); + clip.lut = Some( + LutReference::new("0123456789abcdef".repeat(4), "Wire schema LUT", 0.75) + .expect("wire LUT reference is valid"), + ); clip.chroma_key = Some(ChromaKey::default()); clip.masks = vec![Mask::default()]; clip.effects = vec![Effect::new("wire").with_param("amount", 1.0)]; + clip.stabilization = Some(StabilizationTrack { + model: "wire".to_owned(), + model_version: 1, + source_identity: "asset".to_owned(), + strength: 0.75, + crop_margin: 0.02, + keyframes: vec![ + StabilizationKeyframe::default(), + StabilizationKeyframe { + frame: 1, + translation_x: 0.01, + translation_y: -0.01, + rotation_degrees: 0.5, + }, + ], + }); + clip.transition_out = Some(Transition { + from_clip_id: "clip".to_owned(), + to_clip_id: "next".to_owned(), + kind: TransitionKind::CrossDissolve, + duration_frames: 5, + }); clip.reversed = true; clip } diff --git a/crates/opentake-gen/src/job.rs b/crates/opentake-gen/src/job.rs index b2747b3c..103ad8dd 100644 --- a/crates/opentake-gen/src/job.rs +++ b/crates/opentake-gen/src/job.rs @@ -113,7 +113,9 @@ mod tests { assert_eq!(job.id, "j1"); assert_eq!(job.status, JobStatus::Running); assert_eq!(job.result_urls, None); + assert_eq!(job.error_message, None); assert_eq!(job.cost_credits, None); + assert_eq!(job.completed_at, None); } #[test] diff --git a/crates/opentake-gen/src/lib.rs b/crates/opentake-gen/src/lib.rs index f7b0cdef..a6bdd820 100644 --- a/crates/opentake-gen/src/lib.rs +++ b/crates/opentake-gen/src/lib.rs @@ -21,6 +21,7 @@ pub mod job; pub mod keys; pub mod params; pub mod provider; +pub mod stems; pub mod transport; // Public API surface. @@ -43,6 +44,7 @@ pub use provider::{ content_type_for, ElevenLabsAdapter, FalAdapter, ModelRoute, OpenAiAdapter, ProviderAdapter, ProviderRegistry, ReplicateAdapter, }; +pub use stems::{resolve_stem_execution, StemExecutionPlan, StemProviderSelection}; pub use transport::{ Body, HttpRequest, HttpResponse, HttpTransport, Method, MockTransport, ReqwestTransport, }; diff --git a/crates/opentake-gen/src/stems.rs b/crates/opentake-gen/src/stems.rs new file mode 100644 index 00000000..2dc9b996 --- /dev/null +++ b/crates/opentake-gen/src/stems.rs @@ -0,0 +1,144 @@ +//! Explicit routing policy for stem separation. +//! +//! Local execution never uploads media. Hosted execution is available only +//! after the user selects a concrete provider/model, acknowledges upload, and +//! the normal generation registry proves that provider is configured. + +use crate::{GenError, ModelRoute, ProviderRegistry}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StemProviderSelection { + Local, + Hosted { + provider: String, + model: String, + upload_confirmed: bool, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StemExecutionPlan { + Local, + Hosted { + provider: String, + model: String, + vendor_model: String, + }, +} + +pub fn resolve_stem_execution( + selection: StemProviderSelection, + registry: &ProviderRegistry, +) -> Result { + match selection { + StemProviderSelection::Local => Ok(StemExecutionPlan::Local), + StemProviderSelection::Hosted { + provider, + model, + upload_confirmed, + } => { + if provider.trim().is_empty() || model.trim().is_empty() { + return Err(GenError::Other(anyhow::anyhow!( + "stem provider and model must be selected explicitly" + ))); + } + if !upload_confirmed { + return Err(GenError::Other(anyhow::anyhow!( + "stem upload requires explicit privacy confirmation" + ))); + } + if !registry.has_prefix(&provider) { + return Err(GenError::NotConfigured); + } + let route = ModelRoute::parse(&model)?; + if route.prefix != provider { + return Err(GenError::Other(anyhow::anyhow!( + "stem model prefix does not match selected provider" + ))); + } + Ok(StemExecutionPlan::Hosted { + provider, + model, + vendor_model: route.vendor_model, + }) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use async_trait::async_trait; + + use super::*; + use crate::{GenerationJob, GenerationParams, ProviderAdapter}; + + struct StemCapableProvider; + + #[async_trait] + impl ProviderAdapter for StemCapableProvider { + fn prefix(&self) -> &'static str { + "stemhost" + } + + async fn submit( + &self, + _route: &ModelRoute, + _params: &GenerationParams, + ) -> Result { + unreachable!("routing test does not submit") + } + + async fn poll(&self, _job_id: &str) -> Result { + unreachable!("routing test does not poll") + } + + async fn upload( + &self, + _path: &std::path::Path, + _content_type: &str, + ) -> Result { + unreachable!("routing test does not upload") + } + } + + #[test] + fn local_never_requires_a_provider() { + assert_eq!( + resolve_stem_execution(StemProviderSelection::Local, &ProviderRegistry::new()).unwrap(), + StemExecutionPlan::Local + ); + } + + #[test] + fn hosted_requires_confirmation_configuration_and_matching_prefix() { + let registry = ProviderRegistry::new().with(Arc::new(StemCapableProvider)); + let unconfirmed = resolve_stem_execution( + StemProviderSelection::Hosted { + provider: "stemhost".into(), + model: "stemhost:separate-v1".into(), + upload_confirmed: false, + }, + ®istry, + ); + assert!(unconfirmed.is_err()); + let plan = resolve_stem_execution( + StemProviderSelection::Hosted { + provider: "stemhost".into(), + model: "stemhost:separate-v1".into(), + upload_confirmed: true, + }, + ®istry, + ) + .unwrap(); + assert_eq!( + plan, + StemExecutionPlan::Hosted { + provider: "stemhost".into(), + model: "stemhost:separate-v1".into(), + vendor_model: "separate-v1".into(), + } + ); + } +} diff --git a/crates/opentake-media/Cargo.toml b/crates/opentake-media/Cargo.toml index a7ba54bf..57a8d4a4 100644 --- a/crates/opentake-media/Cargo.toml +++ b/crates/opentake-media/Cargo.toml @@ -25,12 +25,14 @@ ndarray = "0.16" tracing = "0.1" unicode-normalization = "0.1" tempfile = "3" +rustfft = "6" # Media IO. We drive the system ffmpeg/ffprobe over the CLI rather than linking # libav*: the local toolchain is ffmpeg 8.1 (libavcodec 62), which the C-binding # crates (ffmpeg-next / ffmpeg-the-third) do not support, and pkg-config is not -# installed. ffmpeg-sidecar shells out to the binaries on PATH — zero native -# linkage, and it never auto-downloads here (we only use its command/parse API). +# installed. ffmpeg-sidecar shells out to OpenTake's checksum-pinned packaged +# binaries (or PATH during development) — zero native linkage, and it never +# auto-downloads here (we only use its command/parse API). # `default-features = false` drops the `download_ffmpeg` feature, whose ureq + # rustls + zip/tar/xz2 stack is the crate's only HTTP/TLS dependency and is dead # weight for us; the FfmpegCommand command/parse API lives in the always-on core. @@ -69,10 +71,11 @@ default = [] # Test-only cross-crate fault boundaries. Shipped builds do not expose these # hooks; integration tests opt in through a dev-dependency feature union. test-faults = [] -# Real SigLIP2 inference via ONNX Runtime. `download-binaries` lets ort fetch a -# prebuilt onnxruntime *when this feature is explicitly enabled* — it is off by -# default, so plain `cargo build`/`cargo test` never touch the network. -ort-backend = ["dep:ort"] +# Real SigLIP2 inference through ort. Windows uses ort's official pure-Rust +# tract backend so the installed product does not depend on Server-only +# DirectX/ONNX Runtime entry points. Other platforms keep the pinned native +# ONNX Runtime backend. Both remain off in the default, fully-offline build. +ort-backend = ["dep:ort", "dep:ort-tract"] # Real on-device transcription via whisper.cpp (compiles native C++ on enable). whisper-backend = ["dep:whisper-rs"] # Model weight download/verify/unzip (reqwest + zip + sha1). Off by default so @@ -80,10 +83,20 @@ whisper-backend = ["dep:whisper-rs"] # ggml downloads against whisper.cpp's published SHA-1 checksums. model-download = ["dep:reqwest", "dep:zip", "dep:futures-util", "dep:sha1"] -[dependencies.ort] +[target.'cfg(not(windows))'.dependencies.ort] version = "=2.0.0-rc.10" default-features = false -features = ["std", "ndarray", "download-binaries"] +features = ["std", "ndarray", "download-binaries", "copy-dylibs"] +optional = true + +[target.'cfg(windows)'.dependencies.ort] +version = "=2.0.0-rc.10" +default-features = false +features = ["std", "ndarray", "alternative-backend"] +optional = true + +[target.'cfg(windows)'.dependencies.ort-tract] +version = "=0.1.0" optional = true [dependencies.whisper-rs] diff --git a/crates/opentake-media/src/analysis/beat.rs b/crates/opentake-media/src/analysis/beat.rs index 5ae1f97b..74e471df 100644 --- a/crates/opentake-media/src/analysis/beat.rs +++ b/crates/opentake-media/src/analysis/beat.rs @@ -28,6 +28,11 @@ pub struct BeatOnset { pub strength: f32, } +// Normalized PCM below this onset-energy delta is treated as low-level speech, +// room tone, or codec noise. Relative normalization alone would otherwise turn +// an inaudible fluctuation into a full-strength "beat". +const MIN_ABSOLUTE_ONSET_ENERGY_DELTA: f32 = 0.0001; + pub fn detect_beats(samples: &[f32], config: BeatDetectionConfig) -> Vec { if samples.is_empty() || config.sample_rate == 0 || !config.fps.is_finite() || config.fps <= 0.0 { @@ -45,7 +50,7 @@ pub fn detect_beats(samples: &[f32], config: BeatDetectionConfig) -> Vec 0.0); } + + #[test] + fn low_energy_speech_is_not_overdetected() { + let samples = (0..1_000) + .map(|index| if (index / 100) % 2 == 0 { 0.005 } else { 0.006 }) + .collect::>(); + let config = BeatDetectionConfig { + sample_rate: 1_000, + fps: 10.0, + window_size_samples: 100, + hop_size_samples: 100, + min_onset_strength: 0.05, + min_gap_frames: 1, + }; + + assert!(detect_beats(&samples, config).is_empty()); + } } diff --git a/crates/opentake-media/src/analysis/denoise.rs b/crates/opentake-media/src/analysis/denoise.rs new file mode 100644 index 00000000..6a4cfed3 --- /dev/null +++ b/crates/opentake-media/src/analysis/denoise.rs @@ -0,0 +1,232 @@ +//! Deterministic local STFT noise suppression shared by preview and export. + +use std::sync::Arc; + +use opentake_domain::{AudioDenoise, DenoiseMode}; +use rustfft::{num_complex::Complex32, Fft, FftPlanner}; + +use crate::MediaCancelToken; + +pub type DenoiseProgressCallback = Arc; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum DenoiseError { + #[error("denoise_invalid_config: {0}")] + InvalidConfig(String), + #[error("denoise_cancelled")] + Cancelled, +} + +/// Process interleaved PCM without mutating the source. A zero-strength config +/// is a bit-exact bypass. Each channel is transformed independently so stereo +/// placement is preserved, while all call sites share identical parameters and +/// math. +pub fn denoise_interleaved( + samples: &[f32], + channels: usize, + sample_rate: u32, + config: AudioDenoise, + cancel: &MediaCancelToken, + progress: Option, +) -> Result, DenoiseError> { + config + .validate() + .map_err(|error| DenoiseError::InvalidConfig(error.to_string()))?; + if channels == 0 + || channels > 8 + || sample_rate < 8_000 + || !samples.len().is_multiple_of(channels) + { + return Err(DenoiseError::InvalidConfig( + "channels, sample rate, or interleaving is unsupported".to_string(), + )); + } + if cancel.checkpoint() { + return Err(DenoiseError::Cancelled); + } + if samples.is_empty() || config.strength == 0.0 { + return Ok(samples.to_vec()); + } + + let frame_len = if sample_rate >= 32_000 { 1_024 } else { 512 }; + let hop = frame_len / 4; + let audio_frames = samples.len() / channels; + let windows = if audio_frames <= frame_len { + 1 + } else { + 1 + (audio_frames - 1) / hop + }; + let total_steps = channels.saturating_mul(windows).saturating_mul(2).max(1); + let mut completed = 0usize; + + let window = (0..frame_len) + .map(|index| { + let phase = std::f32::consts::TAU * index as f32 / frame_len as f32; + 0.5 - 0.5 * phase.cos() + }) + .collect::>(); + let mut planner = FftPlanner::::new(); + let forward = planner.plan_fft_forward(frame_len); + let inverse = planner.plan_fft_inverse(frame_len); + let mut output = vec![0.0_f32; samples.len()]; + + for channel in 0..channels { + let mono = samples + .iter() + .skip(channel) + .step_by(channels) + .copied() + .collect::>(); + let processed = process_channel( + &mono, + &window, + hop, + windows, + config, + cancel, + &progress, + total_steps, + &mut completed, + &forward, + &inverse, + )?; + for (frame, value) in processed.into_iter().enumerate() { + output[frame * channels + channel] = value.clamp(-1.0, 1.0); + } + } + + if let Some(report) = progress { + report(total_steps, total_steps); + } + Ok(output) +} + +#[allow(clippy::too_many_arguments)] +fn process_channel( + samples: &[f32], + window: &[f32], + hop: usize, + windows: usize, + config: AudioDenoise, + cancel: &MediaCancelToken, + progress: &Option, + total_steps: usize, + completed: &mut usize, + forward: &Arc>, + inverse: &Arc>, +) -> Result, DenoiseError> { + let frame_len = window.len(); + let bins = frame_len / 2 + 1; + const MAX_NOISE_ESTIMATE_WINDOWS: usize = 512; + let estimate_stride = windows.div_ceil(MAX_NOISE_ESTIMATE_WINDOWS).max(1); + let mut powers = (0..bins) + .map(|_| Vec::with_capacity(windows.min(MAX_NOISE_ESTIMATE_WINDOWS))) + .collect::>(); + let mut spectrum = vec![Complex32::new(0.0, 0.0); frame_len]; + + for frame_index in 0..windows { + if cancel.checkpoint() { + return Err(DenoiseError::Cancelled); + } + load_window(samples, window, frame_index * hop, &mut spectrum); + forward.process(&mut spectrum); + if frame_index.is_multiple_of(estimate_stride) { + for bin in 0..bins { + powers[bin].push(spectrum[bin].norm_sqr()); + } + } + report_step(progress, total_steps, completed); + } + + let noise_power = powers + .into_iter() + .map(|mut values| { + values.sort_by(f32::total_cmp); + let index = ((values.len().saturating_sub(1)) as f32 * 0.15).round() as usize; + values[index].max(1.0e-12) + }) + .collect::>(); + let strength = config.strength as f32; + let oversubtraction = match config.mode { + DenoiseMode::Adaptive => 1.0 + 4.5 * strength, + DenoiseMode::Voice => 1.0 + 6.0 * strength, + }; + let floor_gain = 1.0 - 0.92 * strength; + let mut prior_gain = vec![1.0_f32; bins]; + let mut raw_gain = vec![1.0_f32; bins]; + let mut out = vec![0.0_f32; samples.len()]; + let mut norm = vec![0.0_f32; samples.len()]; + + for frame_index in 0..windows { + if cancel.checkpoint() { + return Err(DenoiseError::Cancelled); + } + let start = frame_index * hop; + load_window(samples, window, start, &mut spectrum); + forward.process(&mut spectrum); + for bin in 0..bins { + let power = spectrum[bin].norm_sqr().max(1.0e-12); + let clean_ratio = (1.0 - oversubtraction * noise_power[bin] / power).max(0.0); + raw_gain[bin] = clean_ratio.sqrt().max(floor_gain); + } + for bin in 0..bins { + let lo = bin.saturating_sub(1); + let hi = (bin + 1).min(bins - 1); + let frequency_smoothed = raw_gain[lo..=hi].iter().sum::() / (hi - lo + 1) as f32; + let gain = (prior_gain[bin] * 0.25 + frequency_smoothed * 0.75).clamp(floor_gain, 1.0); + prior_gain[bin] = gain; + spectrum[bin] *= gain; + if bin > 0 && bin < frame_len / 2 { + spectrum[frame_len - bin] *= gain; + } + } + inverse.process(&mut spectrum); + for index in 0..frame_len { + let output_index = start + index; + if output_index >= out.len() { + break; + } + let weight = window[index]; + out[output_index] += spectrum[index].re / frame_len as f32 * weight; + norm[output_index] += weight * weight; + } + report_step(progress, total_steps, completed); + } + + let input_peak = samples + .iter() + .map(|sample| sample.abs()) + .fold(0.0_f32, f32::max) + .min(1.0); + let edge_span = (frame_len / 2).min(samples.len().saturating_sub(1)).max(1); + for (index, (value, weight)) in out.iter_mut().zip(norm).enumerate() { + let normalized = if weight > 1.0e-6 { + *value / weight + } else { + samples[index] + }; + // A centered STFT would normally pad both ends before analysis. Keep + // the implementation allocation-bounded by crossfading the unpadded + // edge into the processed signal instead. The peak guard prevents + // low Hann-normalization weights from creating a click or a new peak. + let edge_distance = index.min(samples.len() - 1 - index); + let processed_mix = (edge_distance as f32 / edge_span as f32).min(1.0); + *value = (samples[index] * (1.0 - processed_mix) + normalized * processed_mix) + .clamp(-input_peak, input_peak); + } + Ok(out) +} + +fn load_window(samples: &[f32], window: &[f32], start: usize, target: &mut [Complex32]) { + for (index, complex) in target.iter_mut().enumerate() { + let value = samples.get(start + index).copied().unwrap_or(0.0); + *complex = Complex32::new(value * window[index], 0.0); + } +} + +fn report_step(progress: &Option, total: usize, completed: &mut usize) { + *completed = completed.saturating_add(1); + if let Some(report) = progress { + report((*completed).min(total), total); + } +} diff --git a/crates/opentake-media/src/analysis/loudness.rs b/crates/opentake-media/src/analysis/loudness.rs new file mode 100644 index 00000000..e5a022ea --- /dev/null +++ b/crates/opentake-media/src/analysis/loudness.rs @@ -0,0 +1,384 @@ +//! Deterministic EBU R128 / ITU-R BS.1770 loudness analysis for mono PCM. +//! +//! OpenTake decodes clip windows to 48 kHz mono before analysis. The same +//! computed gain is persisted on the clip and consumed by preview and export; +//! analysis is never repeated during playback or rendering. + +use std::sync::Arc; + +use thiserror::Error; + +use crate::MediaCancelToken; + +const ABSOLUTE_GATE_LUFS: f64 = -70.0; +const RELATIVE_GATE_LU: f64 = -10.0; +const LOUDNESS_OFFSET: f64 = -0.691; +const BLOCK_MILLIS: u64 = 400; +const BLOCK_STEP_MILLIS: u64 = 100; +const SILENCE_EPSILON: f64 = 1.0e-12; + +pub type LoudnessProgressCallback = Arc; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct LoudnessNormalizationConfig { + pub target_lufs: f64, + pub true_peak_ceiling_dbtp: f64, +} + +impl Default for LoudnessNormalizationConfig { + fn default() -> Self { + Self { + target_lufs: -16.0, + true_peak_ceiling_dbtp: -1.0, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct LoudnessAnalysis { + pub input_integrated_lufs: f64, + pub input_true_peak_dbtp: f64, + pub target_lufs: f64, + pub true_peak_ceiling_dbtp: f64, + pub gain_db: f64, + pub output_integrated_lufs: f64, + pub output_true_peak_dbtp: f64, +} + +#[derive(Clone, Debug, Error, PartialEq)] +pub enum LoudnessError { + #[error("loudness_invalid_config: target LUFS and true-peak ceiling must be finite, with ceiling <= 0 dBTP")] + InvalidConfig, + #[error("loudness_unreadable_audio: sample rate must be positive and PCM must contain finite samples")] + UnreadableAudio, + #[error("loudness_silent_audio: no block passed the EBU R128 absolute gate")] + SilentAudio, + #[error("loudness_target_unreachable: the requested target cannot be reached under the true-peak ceiling")] + TargetUnreachable, + #[error("loudness_cancelled")] + Cancelled, +} + +#[derive(Clone, Copy)] +struct Biquad { + b0: f64, + b1: f64, + b2: f64, + a1: f64, + a2: f64, + x1: f64, + x2: f64, + y1: f64, + y2: f64, +} + +impl Biquad { + fn process(&mut self, input: f64) -> f64 { + let output = self.b0 * input + self.b1 * self.x1 + self.b2 * self.x2 + - self.a1 * self.y1 + - self.a2 * self.y2; + self.x2 = self.x1; + self.x1 = input; + self.y2 = self.y1; + self.y1 = output; + output + } +} + +/// Analyze mono PCM with EBU R128 gating and a 4x inter-sample peak estimate. +pub fn analyze_loudness( + samples: &[f32], + sample_rate: u32, + config: LoudnessNormalizationConfig, +) -> Result { + analyze_loudness_with_progress(samples, sample_rate, config, &MediaCancelToken::new(), None) +} + +pub fn analyze_loudness_with_progress( + samples: &[f32], + sample_rate: u32, + config: LoudnessNormalizationConfig, + cancel: &MediaCancelToken, + progress: Option, +) -> Result { + validate(samples, sample_rate, config)?; + if cancel.checkpoint() { + return Err(LoudnessError::Cancelled); + } + + let input_integrated_lufs = + integrated_loudness(samples, sample_rate, cancel, progress.as_deref())?; + let input_true_peak_dbtp = true_peak_dbtp(samples); + if !input_integrated_lufs.is_finite() || !input_true_peak_dbtp.is_finite() { + return Err(LoudnessError::SilentAudio); + } + + // Compensate for the shared ceiling stage instead of sacrificing program + // loudness on high-crest-factor speech. Three correction passes converge + // the exact persisted gain against the same hard ceiling preview/export + // use, while remaining deterministic and bounded. + let mut gain_db = (config.target_lufs - input_integrated_lufs).clamp(-120.0, 60.0); + let mut output_integrated_lufs = input_integrated_lufs; + let mut output_true_peak_dbtp = input_true_peak_dbtp; + for _ in 0..4 { + if cancel.checkpoint() { + return Err(LoudnessError::Cancelled); + } + let mut normalized = apply_loudness_gain(samples, gain_db); + crate::encode::mix::apply_true_peak_ceiling( + &mut normalized, + Some(config.true_peak_ceiling_dbtp), + ); + output_integrated_lufs = integrated_loudness(&normalized, sample_rate, cancel, None)?; + output_true_peak_dbtp = true_peak_dbtp(&normalized); + let correction = config.target_lufs - output_integrated_lufs; + if correction.abs() <= 0.05 { + break; + } + gain_db = (gain_db + correction).clamp(-120.0, 60.0); + } + if (output_integrated_lufs - config.target_lufs).abs() > 1.0 { + return Err(LoudnessError::TargetUnreachable); + } + if let Some(report) = &progress { + report(samples.len(), samples.len()); + } + Ok(LoudnessAnalysis { + input_integrated_lufs, + input_true_peak_dbtp, + target_lufs: config.target_lufs, + true_peak_ceiling_dbtp: config.true_peak_ceiling_dbtp, + gain_db, + output_integrated_lufs, + output_true_peak_dbtp, + }) +} + +fn integrated_loudness( + samples: &[f32], + sample_rate: u32, + cancel: &MediaCancelToken, + progress: Option<&(dyn Fn(usize, usize) + Send + Sync)>, +) -> Result { + let weighted = k_weight(samples, sample_rate, cancel, progress)?; + let block_len = + (((u64::from(sample_rate) * BLOCK_MILLIS) / 1_000) as usize).min(weighted.len()); + let block_step = + (((u64::from(sample_rate) * BLOCK_STEP_MILLIS) / 1_000) as usize).min(weighted.len()); + if block_len == 0 || block_step == 0 { + return Err(LoudnessError::SilentAudio); + } + + let mut block_powers = Vec::with_capacity((weighted.len() - block_len) / block_step + 1); + let block_count = (weighted.len() - block_len) / block_step + 1; + for (block_index, start) in (0..=weighted.len() - block_len) + .step_by(block_step) + .enumerate() + { + if block_index.is_multiple_of(32) { + if cancel.checkpoint() { + return Err(LoudnessError::Cancelled); + } + if let Some(report) = progress { + report(weighted.len() + block_index, weighted.len() + block_count); + } + } + let power = weighted[start..start + block_len] + .iter() + .map(|sample| sample * sample) + .sum::() + / block_len as f64; + if power_to_lufs(power) >= ABSOLUTE_GATE_LUFS { + block_powers.push(power); + } + } + if block_powers.is_empty() { + return Err(LoudnessError::SilentAudio); + } + + let absolute_mean = mean(&block_powers); + let relative_gate = power_to_lufs(absolute_mean) + RELATIVE_GATE_LU; + let relative_powers = block_powers + .iter() + .copied() + .filter(|power| power_to_lufs(*power) >= relative_gate) + .collect::>(); + if relative_powers.is_empty() { + return Err(LoudnessError::SilentAudio); + } + let integrated = power_to_lufs(mean(&relative_powers)); + if let Some(report) = progress { + let total = weighted.len() + block_count; + report(total, total); + } + Ok(integrated) +} + +pub fn apply_loudness_gain(samples: &[f32], gain_db: f64) -> Vec { + let gain_db = if gain_db.is_finite() { + gain_db.clamp(-120.0, 60.0) + } else { + 0.0 + }; + let gain = 10.0_f64.powf(gain_db / 20.0) as f32; + samples.iter().map(|sample| *sample * gain).collect() +} + +fn validate( + samples: &[f32], + sample_rate: u32, + config: LoudnessNormalizationConfig, +) -> Result<(), LoudnessError> { + if !config.target_lufs.is_finite() + || !config.true_peak_ceiling_dbtp.is_finite() + || config.true_peak_ceiling_dbtp > 0.0 + || !(-70.0..=0.0).contains(&config.target_lufs) + || !(-20.0..=0.0).contains(&config.true_peak_ceiling_dbtp) + { + return Err(LoudnessError::InvalidConfig); + } + if sample_rate == 0 || samples.is_empty() || samples.iter().any(|sample| !sample.is_finite()) { + return Err(LoudnessError::UnreadableAudio); + } + Ok(()) +} + +fn k_weight( + samples: &[f32], + sample_rate: u32, + cancel: &MediaCancelToken, + progress: Option<&(dyn Fn(usize, usize) + Send + Sync)>, +) -> Result, LoudnessError> { + let mut shelf = shelf_filter(sample_rate as f64); + let mut high_pass = high_pass_filter(sample_rate as f64); + let mut output = Vec::with_capacity(samples.len()); + for (index, sample) in samples.iter().enumerate() { + if index.is_multiple_of(16_384) { + if cancel.checkpoint() { + return Err(LoudnessError::Cancelled); + } + if let Some(report) = progress { + report(index, samples.len()); + } + } + output.push(high_pass.process(shelf.process(f64::from(*sample)))); + } + Ok(output) +} + +// Coefficients are generated from the analog transfer functions in ITU-R +// BS.1770, allowing analysis at device rates other than 48 kHz. +fn shelf_filter(sample_rate: f64) -> Biquad { + let f0 = 1_681.974_450_955_533; + let gain_db = 3.999_843_853_973_347; + let q = 0.707_175_236_955_419_6; + let k = (std::f64::consts::PI * f0 / sample_rate).tan(); + let vh = 10.0_f64.powf(gain_db / 20.0); + let vb = vh.powf(0.499_666_774_154_541_6); + let a0 = 1.0 + k / q + k * k; + Biquad { + b0: (vh + vb * k / q + k * k) / a0, + b1: 2.0 * (k * k - vh) / a0, + b2: (vh - vb * k / q + k * k) / a0, + a1: 2.0 * (k * k - 1.0) / a0, + a2: (1.0 - k / q + k * k) / a0, + x1: 0.0, + x2: 0.0, + y1: 0.0, + y2: 0.0, + } +} + +fn high_pass_filter(sample_rate: f64) -> Biquad { + let f0 = 38.135_470_876_024_44; + let q = 0.500_327_037_323_877_3; + let k = (std::f64::consts::PI * f0 / sample_rate).tan(); + let a0 = 1.0 + k / q + k * k; + Biquad { + b0: 1.0 / a0, + b1: -2.0 / a0, + b2: 1.0 / a0, + a1: 2.0 * (k * k - 1.0) / a0, + a2: (1.0 - k / q + k * k) / a0, + x1: 0.0, + x2: 0.0, + y1: 0.0, + y2: 0.0, + } +} + +fn true_peak_dbtp(samples: &[f32]) -> f64 { + let mut peak = samples + .iter() + .map(|sample| f64::from(sample.abs())) + .fold(0.0_f64, f64::max); + // Four-times cubic interpolation catches inter-sample peaks without adding + // a heavyweight DSP dependency. End points are extended constantly. + for index in 0..samples.len().saturating_sub(1) { + let p0 = f64::from(samples[index.saturating_sub(1)]); + let p1 = f64::from(samples[index]); + let p2 = f64::from(samples[index + 1]); + let p3 = f64::from(samples[(index + 2).min(samples.len() - 1)]); + for phase in 1..4 { + let t = phase as f64 / 4.0; + let t2 = t * t; + let t3 = t2 * t; + let value = 0.5 + * ((2.0 * p1) + + (-p0 + p2) * t + + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); + peak = peak.max(value.abs()); + } + } + 20.0 * peak.max(SILENCE_EPSILON).log10() +} + +fn mean(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} + +fn power_to_lufs(power: f64) -> f64 { + LOUDNESS_OFFSET + 10.0 * power.max(SILENCE_EPSILON).log10() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn silence_is_a_typed_error() { + let error = analyze_loudness( + &vec![0.0; 48_000], + 48_000, + LoudnessNormalizationConfig::default(), + ) + .unwrap_err(); + assert_eq!(error, LoudnessError::SilentAudio); + } + + #[test] + fn pre_cancelled_analysis_stops_before_work() { + let cancel = MediaCancelToken::new(); + cancel.cancel(); + let error = analyze_loudness_with_progress( + &vec![0.1; 48_000], + 48_000, + LoudnessNormalizationConfig::default(), + &cancel, + None, + ) + .unwrap_err(); + assert_eq!(error, LoudnessError::Cancelled); + } + + #[test] + fn audible_clip_shorter_than_one_r128_block_is_supported() { + let samples = (0..4_800) + .map(|index| (index as f32 * 440.0 * std::f32::consts::TAU / 48_000.0).sin() * 0.1) + .collect::>(); + let analysis = analyze_loudness(&samples, 48_000, LoudnessNormalizationConfig::default()) + .expect("short audible clip"); + assert!(analysis.input_integrated_lufs.is_finite()); + } +} diff --git a/crates/opentake-media/src/analysis/matting.rs b/crates/opentake-media/src/analysis/matting.rs new file mode 100644 index 00000000..226d0c2c --- /dev/null +++ b/crates/opentake-media/src/analysis/matting.rs @@ -0,0 +1,367 @@ +//! Verified local Robust Video Matting (RVM) model and frame inference. + +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +#[cfg(feature = "model-download")] +use std::sync::Arc; + +use sha2::{Digest, Sha256}; + +#[cfg(any(feature = "ort-backend", feature = "model-download"))] +use crate::MediaCancelToken; +#[cfg(feature = "ort-backend")] +use crate::RgbaFrame; +use crate::{MediaError, Result}; + +pub const RVM_MODEL_ID: &str = "rvm-mobilenetv3-fp32-v1.0.0"; +pub const RVM_MODEL_FILE: &str = "rvm_mobilenetv3_fp32.onnx"; +pub const RVM_MODEL_SHA256: &str = + "88d4531297118f595bf2fd60f6f566aec2e559393802d1f436c380f0cbbd2828"; +pub const RVM_MODEL_BYTES: u64 = 14_975_696; +pub const RVM_MODEL_URL: &str = "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx"; + +#[cfg(feature = "model-download")] +pub type MattingDownloadProgress = Arc; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InstalledMattingModel { + pub id: String, + pub path: PathBuf, + pub sha256: String, + pub bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlphaMatteFrame { + pub width: u32, + pub height: u32, + pub alpha: Vec, + /// Model-cleaned straight foreground RGB, three bytes per pixel. + pub foreground_rgb: Vec, +} + +pub fn matting_model_path(model_dir: &Path) -> PathBuf { + model_dir.join("matting").join(RVM_MODEL_FILE) +} + +pub fn verify_rvm_model(model_dir: &Path) -> Result { + let path = matting_model_path(model_dir); + let metadata = std::fs::symlink_metadata(&path).map_err(|error| { + MediaError::ModelInstall(format!("matting_model_not_installed:{error}")) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(MediaError::ModelInstall( + "matting_model_must_be_a_regular_file".to_string(), + )); + } + if metadata.len() != RVM_MODEL_BYTES { + return Err(MediaError::Checksum(format!( + "matting_model_size_mismatch: expected {RVM_MODEL_BYTES}, got {}", + metadata.len() + ))); + } + let mut file = File::open(&path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + let actual = format!("{:x}", digest.finalize()); + if actual != RVM_MODEL_SHA256 { + return Err(MediaError::Checksum(format!( + "matting_model_integrity_failed: expected {RVM_MODEL_SHA256}, got {actual}" + ))); + } + Ok(InstalledMattingModel { + id: RVM_MODEL_ID.to_string(), + path, + sha256: actual, + bytes: metadata.len(), + }) +} + +#[cfg(feature = "model-download")] +pub async fn download_rvm_model( + model_dir: &Path, + cancel: &MediaCancelToken, + progress: Option, +) -> Result { + use std::io::Write; + + use futures_util::StreamExt; + + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let destination = matting_model_path(model_dir); + if destination.exists() { + return verify_rvm_model(model_dir); + } + let parent = destination + .parent() + .ok_or_else(|| MediaError::ModelInstall("matting_model_path_invalid".to_string()))?; + std::fs::create_dir_all(parent)?; + let parent_metadata = std::fs::symlink_metadata(parent)?; + if !parent_metadata.is_dir() || parent_metadata.file_type().is_symlink() { + return Err(MediaError::ModelInstall( + "matting_model_directory_must_be_regular".to_string(), + )); + } + let install = async { + let response = reqwest::Client::new() + .get(RVM_MODEL_URL) + .send() + .await + .map_err(|error| MediaError::ModelInstall(format!("matting_model_download:{error}")))? + .error_for_status() + .map_err(|error| MediaError::ModelInstall(format!("matting_model_download:{error}")))?; + if response + .content_length() + .is_some_and(|bytes| bytes != RVM_MODEL_BYTES) + { + return Err(MediaError::Checksum( + "matting_model_content_length_mismatch".to_string(), + )); + } + let mut partial = tempfile::Builder::new() + .prefix(".rvm-model-") + .suffix(".partial") + .tempfile_in(parent) + .map_err(|error| { + MediaError::ModelInstall(format!("matting_model_partial_create:{error}")) + })?; + let mut digest = Sha256::new(); + let mut downloaded = 0_u64; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let chunk = chunk.map_err(|error| { + MediaError::ModelInstall(format!("matting_model_download:{error}")) + })?; + downloaded = downloaded + .checked_add(chunk.len() as u64) + .ok_or_else(|| MediaError::ModelInstall("matting_model_too_large".to_string()))?; + if downloaded > RVM_MODEL_BYTES { + return Err(MediaError::Checksum( + "matting_model_download_exceeds_manifest".to_string(), + )); + } + digest.update(&chunk); + partial.as_file_mut().write_all(&chunk)?; + if let Some(progress) = &progress { + progress(downloaded, RVM_MODEL_BYTES); + } + } + partial.as_file().sync_all()?; + if downloaded != RVM_MODEL_BYTES { + return Err(MediaError::Checksum(format!( + "matting_model_size_mismatch: expected {RVM_MODEL_BYTES}, got {downloaded}" + ))); + } + let actual = format!("{:x}", digest.finalize()); + if actual != RVM_MODEL_SHA256 { + return Err(MediaError::Checksum(format!( + "matting_model_integrity_failed: expected {RVM_MODEL_SHA256}, got {actual}" + ))); + } + partial.persist_noclobber(&destination).map_err(|error| { + MediaError::ModelInstall(format!("matting_model_publish:{}", error.error)) + })?; + Ok(()) + } + .await; + install?; + verify_rvm_model(model_dir) +} + +#[cfg(feature = "ort-backend")] +pub struct RvmMattingSession { + model: crate::ort_worker::OrtModel, + recurrent: [ndarray::ArrayD; 4], + pub installed: InstalledMattingModel, +} + +#[cfg(feature = "ort-backend")] +impl RvmMattingSession { + pub fn load(model_dir: &Path) -> Result { + use ndarray::{ArrayD, IxDyn}; + + let installed = verify_rvm_model(model_dir)?; + let model = crate::ort_worker::OrtModel::load( + &installed.path, + crate::ort_worker::ExecutionProvider::platform_default(), + )?; + let (inputs, outputs) = model.io_contract(); + let input_names = inputs + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(); + let output_names = outputs + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(); + if input_names != ["src", "r1i", "r2i", "r3i", "r4i", "downsample_ratio"] + || output_names != ["fgr", "pha", "r1o", "r2o", "r3o", "r4o"] + { + return Err(MediaError::ModelInstall( + "matting_model_io_contract_mismatch".to_string(), + )); + } + let empty = || ArrayD::zeros(IxDyn(&[1, 1, 1, 1])); + Ok(Self { + model, + recurrent: [empty(), empty(), empty(), empty()], + installed, + }) + } + + pub fn reset_temporal_state(&mut self) { + use ndarray::{ArrayD, IxDyn}; + self.recurrent = std::array::from_fn(|_| ArrayD::zeros(IxDyn(&[1, 1, 1, 1]))); + } + + pub fn infer( + &mut self, + frame: &RgbaFrame, + cancel: &MediaCancelToken, + ) -> Result { + use ndarray::{ArrayD, IxDyn}; + + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let width = frame.width as usize; + let height = frame.height as usize; + let mut src = vec![0.0_f32; 3 * width * height]; + for (index, pixel) in frame.rgba.chunks_exact(4).enumerate() { + src[index] = pixel[0] as f32 / 255.0; + src[width * height + index] = pixel[1] as f32 / 255.0; + src[2 * width * height + index] = pixel[2] as f32 / 255.0; + } + let src = ArrayD::from_shape_vec(IxDyn(&[1, 3, height, width]), src) + .map_err(|error| MediaError::Decode(format!("matting_input_shape:{error}")))?; + let mut outputs = self.model.run_f32(vec![ + ("src".to_string(), src), + ("r1i".to_string(), self.recurrent[0].clone()), + ("r2i".to_string(), self.recurrent[1].clone()), + ("r3i".to_string(), self.recurrent[2].clone()), + ("r4i".to_string(), self.recurrent[3].clone()), + ( + "downsample_ratio".to_string(), + ArrayD::from_shape_vec(IxDyn(&[1]), vec![0.25]).expect("fixed ratio shape"), + ), + ])?; + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + for (index, name) in ["r1o", "r2o", "r3o", "r4o"].into_iter().enumerate() { + self.recurrent[index] = outputs.remove(name).ok_or_else(|| { + MediaError::Decode(format!("matting_model_missing_output:{name}")) + })?; + } + let foreground = outputs + .remove("fgr") + .ok_or_else(|| MediaError::Decode("matting_model_missing_output:fgr".to_string()))?; + if foreground.shape() != [1, 3, height, width] { + return Err(MediaError::Decode(format!( + "matting_foreground_shape_mismatch:{:?}", + foreground.shape() + ))); + } + let foreground = foreground + .as_slice() + .ok_or_else(|| MediaError::Decode("matting_foreground_not_contiguous".to_string()))?; + let alpha = outputs + .remove("pha") + .ok_or_else(|| MediaError::Decode("matting_model_missing_output:pha".to_string()))?; + if alpha.shape() != [1, 1, height, width] { + return Err(MediaError::Decode(format!( + "matting_alpha_shape_mismatch:{:?}", + alpha.shape() + ))); + } + let alpha = alpha + .iter() + .map(|value| (value.clamp(0.0, 1.0) * 255.0).round() as u8) + .collect(); + let plane = width * height; + let foreground_rgb = (0..plane) + .flat_map(|index| { + [ + foreground[index], + foreground[plane + index], + foreground[2 * plane + index], + ] + .map(|value| (value.clamp(0.0, 1.0) * 255.0).round() as u8) + }) + .collect(); + Ok(AlphaMatteFrame { + width: frame.width, + height: frame.height, + alpha, + foreground_rgb, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_model_is_a_typed_install_error() { + let root = tempfile::tempdir().unwrap(); + let error = verify_rvm_model(root.path()).expect_err("model must be absent"); + assert!(matches!(error, MediaError::ModelInstall(_))); + assert!(error.to_string().contains("matting_model_not_installed")); + } + + #[cfg(feature = "model-download")] + #[test] + fn pre_cancelled_download_never_creates_a_partial_model() { + use futures_util::FutureExt; + + let root = tempfile::tempdir().unwrap(); + let cancel = MediaCancelToken::new(); + cancel.cancel(); + let error = download_rvm_model(root.path(), &cancel, None) + .now_or_never() + .expect("pre-cancelled install completes without polling the network") + .expect_err("pre-cancelled install must fail before network access"); + assert!(matches!(error, MediaError::Cancelled)); + assert!(!matting_model_path(root.path()).exists()); + } + + #[cfg(feature = "ort-backend")] + #[test] + fn official_rvm_model_returns_frame_aligned_alpha() { + let Some(source) = std::env::var_os("OPENTAKE_TEST_RVM_MODEL") else { + return; + }; + let root = tempfile::tempdir().unwrap(); + let destination = matting_model_path(root.path()); + std::fs::create_dir_all(destination.parent().unwrap()).unwrap(); + std::fs::copy(source, destination).unwrap(); + let mut session = RvmMattingSession::load(root.path()).expect("load verified RVM"); + let mut frame = RgbaFrame::black(64, 64); + for y in 8..56 { + for x in 16..48 { + let index = ((y * 64 + x) * 4) as usize; + frame.rgba[index..index + 4].copy_from_slice(&[210, 160, 130, 255]); + } + } + let matte = session + .infer(&frame, &MediaCancelToken::new()) + .expect("infer alpha"); + assert_eq!((matte.width, matte.height), (64, 64)); + assert_eq!(matte.alpha.len(), 64 * 64); + assert_eq!(matte.foreground_rgb.len(), 64 * 64 * 3); + } +} diff --git a/crates/opentake-media/src/analysis/mod.rs b/crates/opentake-media/src/analysis/mod.rs index dce2342b..76d57ae9 100644 --- a/crates/opentake-media/src/analysis/mod.rs +++ b/crates/opentake-media/src/analysis/mod.rs @@ -2,11 +2,38 @@ pub mod autocrop; pub mod beat; +pub mod denoise; +pub mod loudness; +pub mod matting; pub mod silence; +pub mod stabilization; +pub mod stems; pub use autocrop::{ detect_autocrop, AutocropConfig, AutocropPlan, CropRect, CropTransform, FrameBuffer, PixelFormat, }; pub use beat::{detect_beats, BeatDetectionConfig, BeatOnset}; +pub use denoise::{denoise_interleaved, DenoiseError, DenoiseProgressCallback}; +pub use loudness::{ + analyze_loudness, analyze_loudness_with_progress, apply_loudness_gain, LoudnessAnalysis, + LoudnessError, LoudnessNormalizationConfig, LoudnessProgressCallback, +}; +#[cfg(feature = "ort-backend")] +pub use matting::RvmMattingSession; +#[cfg(feature = "model-download")] +pub use matting::{download_rvm_model, MattingDownloadProgress}; +pub use matting::{ + matting_model_path, verify_rvm_model, AlphaMatteFrame, InstalledMattingModel, RVM_MODEL_BYTES, + RVM_MODEL_FILE, RVM_MODEL_ID, RVM_MODEL_SHA256, RVM_MODEL_URL, +}; pub use silence::{detect_silences, SilenceDetectionConfig, SilenceRange}; +pub use stabilization::{ + analyze_stabilization, track_region_motion, track_translation_motion, NormalizedMotionRegion, + RegionMotionTrack, StabilizationConfig, StabilizationMotionSample, +}; +pub use stems::{ + ensure_local_stem_model, separate_stems, verify_local_stem_model, InstalledStemModel, + StemExecution, StemMetrics, StemOutput, StemProgressCallback, StemProvenance, + StemSeparationRequest, StemSeparationResult, +}; diff --git a/crates/opentake-media/src/analysis/stabilization.rs b/crates/opentake-media/src/analysis/stabilization.rs new file mode 100644 index 00000000..3af3059e --- /dev/null +++ b/crates/opentake-media/src/analysis/stabilization.rs @@ -0,0 +1,475 @@ +//! Deterministic camera-motion smoothing for editable stabilization tracks. + +use opentake_domain::{StabilizationKeyframe, StabilizationTrack}; + +use crate::{MediaCancelToken, MediaError, Result, RgbaFrame}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StabilizationMotionSample { + pub frame: i32, + /// Observed camera translation in normalized output-canvas coordinates. + pub translation_x: f64, + pub translation_y: f64, + pub rotation_degrees: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct NormalizedMotionRegion { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RegionMotionTrack { + pub samples: Vec, + pub minimum_confidence: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StabilizationConfig { + /// Half-width of the centered moving-average window. + pub smoothing_radius: usize, +} + +impl Default for StabilizationConfig { + fn default() -> Self { + Self { + smoothing_radius: 2, + } + } +} + +/// Convert tracked camera motion into a non-destructive compensation track. +/// The analyzer never reads or writes the source media; callers own motion +/// extraction and persist the returned track through the edit command layer. +pub fn analyze_stabilization( + samples: &[StabilizationMotionSample], + source_identity: impl Into, + config: StabilizationConfig, + cancel: &MediaCancelToken, +) -> Result { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + if samples.len() < 2 { + return Err(MediaError::Decode( + "stabilization requires at least two motion samples".to_string(), + )); + } + let source_identity = source_identity.into(); + if source_identity.trim().is_empty() { + return Err(MediaError::Decode( + "stabilization source identity is required".to_string(), + )); + } + for (index, sample) in samples.iter().enumerate() { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + if index > 0 && sample.frame <= samples[index - 1].frame { + return Err(MediaError::Decode( + "stabilization motion frames must be strictly increasing".to_string(), + )); + } + if !sample.translation_x.is_finite() + || !sample.translation_y.is_finite() + || !sample.rotation_degrees.is_finite() + { + return Err(MediaError::Decode( + "stabilization motion samples must be finite".to_string(), + )); + } + } + + let radius = config.smoothing_radius.min(samples.len() - 1); + let keyframes = samples + .iter() + .enumerate() + .map(|(index, sample)| { + let start = index.saturating_sub(radius); + let end = (index + radius + 1).min(samples.len()); + let count = (end - start) as f64; + let smoothed_x = samples[start..end] + .iter() + .map(|entry| entry.translation_x) + .sum::() + / count; + let smoothed_y = samples[start..end] + .iter() + .map(|entry| entry.translation_y) + .sum::() + / count; + let smoothed_rotation = samples[start..end] + .iter() + .map(|entry| entry.rotation_degrees) + .sum::() + / count; + StabilizationKeyframe { + frame: sample.frame, + translation_x: smoothed_x - sample.translation_x, + translation_y: smoothed_y - sample.translation_y, + rotation_degrees: smoothed_rotation - sample.rotation_degrees, + } + }) + .collect(); + + Ok(StabilizationTrack { + model: "opentake.motion-smoothing".to_string(), + model_version: 1, + source_identity, + strength: 1.0, + crop_margin: 0.0, + keyframes, + }) +} + +/// Track a dominant translation path from decoded frames using deterministic +/// luma block matching. Each frame is paired with its clip-relative timeline +/// frame so the returned motion can be turned directly into a persisted track. +pub fn track_translation_motion( + frames: &[(i32, RgbaFrame)], + cancel: &MediaCancelToken, +) -> Result> { + if frames.len() < 2 { + return Err(MediaError::Decode( + "stabilization requires at least two decoded frames".to_string(), + )); + } + let width = frames[0].1.width; + let height = frames[0].1.height; + if width < 24 || height < 24 { + return Err(MediaError::Decode( + "stabilization frames are too small for motion tracking".to_string(), + )); + } + if frames + .iter() + .any(|(_, frame)| frame.width != width || frame.height != height) + { + return Err(MediaError::Decode( + "stabilization frames must have consistent dimensions".to_string(), + )); + } + + let mut x = 0.0; + let mut y = 0.0; + let mut samples = vec![StabilizationMotionSample { + frame: frames[0].0, + translation_x: x, + translation_y: y, + rotation_degrees: 0.0, + }]; + for pair in frames.windows(2) { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let (dx, dy) = estimate_translation(&pair[0].1, &pair[1].1, cancel)?; + x += dx as f64 / width as f64; + y += dy as f64 / height as f64; + samples.push(StabilizationMotionSample { + frame: pair[1].0, + translation_x: x, + translation_y: y, + rotation_degrees: 0.0, + }); + } + Ok(samples) +} + +/// Track the selected subject rectangle rather than the dominant full-frame +/// camera motion. The rectangle is normalized to the decoded frame and follows +/// the previous match, so a static background cannot overpower a small moving +/// subject. Translation samples are normalized to the full frame and can be +/// applied directly as position-keyframe deltas. +pub fn track_region_motion( + frames: &[(i32, RgbaFrame)], + region: NormalizedMotionRegion, + cancel: &MediaCancelToken, +) -> Result { + if frames.len() < 2 { + return Err(MediaError::Decode( + "motion tracking requires at least two decoded frames".to_string(), + )); + } + if ![region.x, region.y, region.width, region.height] + .into_iter() + .all(f64::is_finite) + || region.x < 0.0 + || region.y < 0.0 + || region.width <= 0.0 + || region.height <= 0.0 + || region.x + region.width > 1.0 + || region.y + region.height > 1.0 + { + return Err(MediaError::Decode( + "motion tracking region must be a positive normalized rectangle inside the frame" + .to_string(), + )); + } + let width = frames[0].1.width; + let height = frames[0].1.height; + if frames + .iter() + .any(|(_, frame)| frame.width != width || frame.height != height) + { + return Err(MediaError::Decode( + "motion tracking frames must have consistent dimensions".to_string(), + )); + } + let region_width = (region.width * width as f64).round() as i32; + let region_height = (region.height * height as f64).round() as i32; + if region_width < 8 || region_height < 8 { + return Err(MediaError::Decode( + "motion tracking region is too small".to_string(), + )); + } + let mut origin_x = + ((region.x * width as f64).round() as i32).clamp(0, width as i32 - region_width); + let mut origin_y = + ((region.y * height as f64).round() as i32).clamp(0, height as i32 - region_height); + let mut total_x = 0_i32; + let mut total_y = 0_i32; + let mut minimum_confidence = 1.0_f64; + let mut samples = vec![StabilizationMotionSample { + frame: frames[0].0, + translation_x: 0.0, + translation_y: 0.0, + rotation_degrees: 0.0, + }]; + for pair in frames.windows(2) { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let (dx, dy, confidence) = estimate_region_translation( + &pair[0].1, + &pair[1].1, + origin_x, + origin_y, + region_width, + region_height, + cancel, + )?; + origin_x += dx; + origin_y += dy; + total_x += dx; + total_y += dy; + minimum_confidence = minimum_confidence.min(confidence); + samples.push(StabilizationMotionSample { + frame: pair[1].0, + translation_x: total_x as f64 / width as f64, + translation_y: total_y as f64 / height as f64, + rotation_degrees: 0.0, + }); + } + Ok(RegionMotionTrack { + samples, + minimum_confidence, + }) +} + +fn estimate_region_translation( + previous: &RgbaFrame, + current: &RgbaFrame, + origin_x: i32, + origin_y: i32, + region_width: i32, + region_height: i32, + cancel: &MediaCancelToken, +) -> Result<(i32, i32, f64)> { + const SEARCH: i32 = 8; + const STEP: usize = 2; + let mut min_luma = u16::MAX; + let mut max_luma = 0_u16; + for y in (0..region_height).step_by(STEP) { + for x in (0..region_width).step_by(STEP) { + let value = luma(previous, (origin_x + x) as u32, (origin_y + y) as u32); + min_luma = min_luma.min(value); + max_luma = max_luma.max(value); + } + } + let texture = (max_luma.saturating_sub(min_luma) as f64 / 64.0).clamp(0.0, 1.0); + let mut best = (u64::MAX, i32::MAX, i32::MAX, i32::MAX, 0, 0); + for dy in -SEARCH..=SEARCH { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + for dx in -SEARCH..=SEARCH { + let candidate_x = origin_x + dx; + let candidate_y = origin_y + dy; + if candidate_x < 0 + || candidate_y < 0 + || candidate_x + region_width > current.width as i32 + || candidate_y + region_height > current.height as i32 + { + continue; + } + let mut error = 0_u64; + let mut count = 0_u64; + for y in (0..region_height).step_by(STEP) { + for x in (0..region_width).step_by(STEP) { + let a = luma(previous, (origin_x + x) as u32, (origin_y + y) as u32); + let b = luma(current, (candidate_x + x) as u32, (candidate_y + y) as u32); + error += a.abs_diff(b) as u64; + count += 1; + } + } + let normalized = error.checked_div(count).unwrap_or(u64::MAX); + let candidate = (normalized, dx.abs() + dy.abs(), dy.abs(), dx.abs(), dx, dy); + if candidate < best { + best = candidate; + } + } + } + if best.0 == u64::MAX { + return Err(MediaError::Decode( + "motion tracking region left the frame".to_string(), + )); + } + let match_quality = (1.0 - best.0 as f64 / 255.0).clamp(0.0, 1.0); + Ok((best.4, best.5, match_quality * texture)) +} + +fn estimate_translation( + previous: &RgbaFrame, + current: &RgbaFrame, + cancel: &MediaCancelToken, +) -> Result<(i32, i32)> { + const SEARCH: i32 = 8; + const STEP: usize = 8; + let width = current.width as i32; + let height = current.height as i32; + let mut best = (u64::MAX, i32::MAX, i32::MAX, i32::MAX, 0, 0); + for dy in -SEARCH..=SEARCH { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + for dx in -SEARCH..=SEARCH { + let mut error = 0_u64; + let mut count = 0_u64; + for y in ((SEARCH + 1) as usize..(height - SEARCH - 1) as usize).step_by(STEP) { + for x in ((SEARCH + 1) as usize..(width - SEARCH - 1) as usize).step_by(STEP) { + let previous_x = x as i32 - dx; + let previous_y = y as i32 - dy; + let a = luma(previous, previous_x as u32, previous_y as u32); + let b = luma(current, x as u32, y as u32); + error += a.abs_diff(b) as u64; + count += 1; + } + } + let normalized = error.checked_div(count).unwrap_or(u64::MAX); + let candidate = (normalized, dx.abs() + dy.abs(), dy.abs(), dx.abs(), dx, dy); + if candidate < best { + best = candidate; + } + } + } + Ok((best.4, best.5)) +} + +fn luma(frame: &RgbaFrame, x: u32, y: u32) -> u16 { + let offset = ((y * frame.width + x) * 4) as usize; + let r = frame.rgba[offset] as u16; + let g = frame.rgba[offset + 1] as u16; + let b = frame.rgba[offset + 2] as u16; + (54 * r + 183 * g + 19 * b) >> 8 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_pre_cancelled_analysis_before_work() { + let cancel = MediaCancelToken::new(); + cancel.cancel(); + let result = analyze_stabilization( + &[ + StabilizationMotionSample { + frame: 0, + translation_x: 0.0, + translation_y: 0.0, + rotation_degrees: 0.0, + }, + StabilizationMotionSample { + frame: 1, + translation_x: 0.1, + translation_y: 0.0, + rotation_degrees: 0.0, + }, + ], + "asset", + StabilizationConfig::default(), + &cancel, + ); + assert!(matches!(result, Err(MediaError::Cancelled))); + } + + #[test] + fn block_match_tracks_known_translation() { + let make_frame = |offset: u32| { + let mut frame = RgbaFrame::black(64, 48); + for y in 8..40 { + for x in offset..offset + 24 { + let index = ((y * frame.width + x) * 4) as usize; + let value = ((x - offset) * 37 + y * 19 + (x - offset) * y * 3) as u8; + frame.rgba[index..index + 3].copy_from_slice(&[value, value, value]); + } + } + frame + }; + let samples = track_translation_motion( + &[ + (0, make_frame(16)), + (1, make_frame(20)), + (2, make_frame(24)), + ], + &MediaCancelToken::new(), + ) + .expect("track translated fixture"); + assert!(samples[1].translation_x > 0.0); + assert!(samples[2].translation_x > samples[1].translation_x); + assert_eq!(samples[2].translation_y, 0.0); + } + + #[test] + fn region_tracker_keeps_known_subject_center_within_five_pixels() { + let make_frame = |offset_x: u32, offset_y: u32| { + let mut frame = RgbaFrame::black(96, 72); + for y in offset_y..offset_y + 20 { + for x in offset_x..offset_x + 24 { + let index = ((y * frame.width + x) * 4) as usize; + let local_x = x - offset_x; + let local_y = y - offset_y; + frame.rgba[index..index + 3].copy_from_slice(&[ + (local_x * 9 + local_y * 3) as u8, + (local_x * 2 + local_y * 11) as u8, + (local_x * 7 + local_y * 5) as u8, + ]); + } + } + frame + }; + let tracked = track_region_motion( + &[ + (0, make_frame(20, 24)), + (1, make_frame(24, 26)), + (2, make_frame(28, 28)), + ], + NormalizedMotionRegion { + x: 20.0 / 96.0, + y: 24.0 / 72.0, + width: 24.0 / 96.0, + height: 20.0 / 72.0, + }, + &MediaCancelToken::new(), + ) + .expect("track selected subject"); + + assert!(tracked.minimum_confidence >= 0.25); + let final_sample = tracked.samples.last().expect("final sample"); + assert!((final_sample.translation_x * 96.0 - 8.0).abs() <= 5.0); + assert!((final_sample.translation_y * 72.0 - 4.0).abs() <= 5.0); + } +} diff --git a/crates/opentake-media/src/analysis/stems.rs b/crates/opentake-media/src/analysis/stems.rs new file mode 100644 index 00000000..9a7233ee --- /dev/null +++ b/crates/opentake-media/src/analysis/stems.rs @@ -0,0 +1,358 @@ +//! Deterministic two-stem separation shared by the desktop job and tests. +//! +//! The bundled `opentake-center-v1` profile is a tiny, inspectable local model: +//! it extracts the stereo centre (voice/dialogue) and complementary side signal +//! (music/ambience). Both user-facing stems are emitted dual-mono so either one +//! remains audible through OpenTake's current mono export mixdown. It is +//! intentionally local-first and offline. Hosted +//! execution is represented explicitly so a caller cannot upload media without +//! choosing a configured provider; network transport remains in `opentake-gen`. + +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use sha2::{Digest, Sha256}; + +use crate::{ + decode_pcm_interleaved_cancellable, MediaCancelToken, MediaError, PcmFormat, PcmSpec, Result, +}; + +const MODEL_ID: &str = "opentake-center-v1"; +const MODEL_FILE: &str = "opentake-center-v1.json"; +const MODEL_BYTES: &[u8] = b"{\"algorithm\":\"mid-side\",\"id\":\"opentake-center-v1\",\"version\":1,\"vocalCenterGain\":1.0,\"residualGain\":1.0}\n"; +const MODEL_SHA256: &str = "9c72ab220f370000a702fc11c8071905648a56d1102d9519659a6062abb4b376"; +const SAMPLE_RATE: u32 = 48_000; +const CHANNELS: u16 = 2; +const PROGRESS_TOTAL: usize = 1_000; + +pub type StemProgressCallback = Arc; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InstalledStemModel { + pub id: String, + pub path: PathBuf, + pub sha256: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StemExecution<'a> { + Local { model_dir: &'a Path }, + Hosted { provider: String, model: String }, +} + +#[derive(Clone, Debug)] +pub struct StemSeparationRequest<'a> { + pub source: &'a Path, + pub output_dir: &'a Path, + pub execution: StemExecution<'a>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StemOutput { + pub path: PathBuf, + pub name: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StemProvenance { + pub source_sha256: String, + pub execution: String, + pub model_sha256: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StemMetrics { + /// Centre/side cross-talk removed by the local matrix, expressed as an + /// estimated SDR improvement. This is deterministic quality telemetry, not + /// a claim about semantic source labels for arbitrary mixes. + pub vocal_sdr_improvement_db: f64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct StemSeparationResult { + pub vocals: StemOutput, + pub accompaniment: StemOutput, + pub provenance: StemProvenance, + pub metrics: StemMetrics, +} + +fn model_path(model_dir: &Path) -> PathBuf { + model_dir.join("stems").join(MODEL_FILE) +} + +fn digest_bytes(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn digest_file(path: &Path, cancel: Option<&MediaCancelToken>) -> Result { + let mut file = File::open(path)?; + let mut digest = Sha256::new(); + let mut chunk = [0_u8; 64 * 1024]; + loop { + if cancel.is_some_and(MediaCancelToken::checkpoint) { + return Err(MediaError::Cancelled); + } + let read = file.read(&mut chunk)?; + if read == 0 { + break; + } + digest.update(&chunk[..read]); + } + Ok(format!("{:x}", digest.finalize())) +} + +pub fn verify_local_stem_model(model_dir: &Path) -> Result { + let path = model_path(model_dir); + let actual = digest_file(&path, None)?; + if actual != MODEL_SHA256 { + return Err(MediaError::Checksum(format!( + "stem_model_integrity_failed: expected {MODEL_SHA256}, got {actual}" + ))); + } + Ok(InstalledStemModel { + id: MODEL_ID.to_string(), + path, + sha256: actual, + }) +} + +/// Install the bundled, offline model once. Existing files are always verified +/// and never silently replaced, so tampering/corruption produces a typed error. +pub fn ensure_local_stem_model(model_dir: &Path) -> Result { + let path = model_path(model_dir); + if path.exists() { + return verify_local_stem_model(model_dir); + } + let parent = path + .parent() + .ok_or_else(|| MediaError::ModelInstall("stem_model_path_invalid".to_string()))?; + fs::create_dir_all(parent)?; + let partial = parent.join(format!(".{MODEL_FILE}.partial")); + let install = (|| -> Result<()> { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&partial) + .map_err(|error| MediaError::ModelInstall(format!("stem_model_install: {error}")))?; + file.write_all(MODEL_BYTES)?; + file.sync_all()?; + if digest_bytes(MODEL_BYTES) != MODEL_SHA256 { + return Err(MediaError::Checksum( + "bundled stem model checksum does not match manifest".to_string(), + )); + } + fs::rename(&partial, &path)?; + Ok(()) + })(); + if install.is_err() { + let _ = fs::remove_file(&partial); + } + install?; + verify_local_stem_model(model_dir) +} + +fn report(progress: &Option, completed: usize) { + if let Some(report) = progress { + report(completed.min(PROGRESS_TOTAL), PROGRESS_TOTAL); + } +} + +fn safe_source_stem(path: &Path) -> String { + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or("audio"); + let safe = stem + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_') { + character + } else { + '_' + } + }) + .collect::(); + if safe.is_empty() { + "audio".to_string() + } else { + safe + } +} + +fn write_wav_stereo(path: &Path, samples: &[f32], cancel: &MediaCancelToken) -> Result<()> { + let sample_count = u32::try_from(samples.len()) + .map_err(|_| MediaError::Encode("stem_output_too_large".to_string()))?; + let data_len = sample_count + .checked_mul(2) + .ok_or_else(|| MediaError::Encode("stem_output_too_large".to_string()))?; + let mut file = OpenOptions::new().create_new(true).write(true).open(path)?; + file.write_all(b"RIFF")?; + file.write_all(&(36_u32 + data_len).to_le_bytes())?; + file.write_all(b"WAVEfmt ")?; + file.write_all(&16_u32.to_le_bytes())?; + file.write_all(&1_u16.to_le_bytes())?; + file.write_all(&CHANNELS.to_le_bytes())?; + file.write_all(&SAMPLE_RATE.to_le_bytes())?; + file.write_all(&(SAMPLE_RATE * u32::from(CHANNELS) * 2).to_le_bytes())?; + file.write_all(&(CHANNELS * 2).to_le_bytes())?; + file.write_all(&16_u16.to_le_bytes())?; + file.write_all(b"data")?; + file.write_all(&data_len.to_le_bytes())?; + for chunk in samples.chunks(8 * 1024) { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let mut bytes = Vec::with_capacity(chunk.len() * 2); + for sample in chunk { + let quantized = (sample.clamp(-1.0, 1.0) * 32767.0).round() as i16; + bytes.extend_from_slice(&quantized.to_le_bytes()); + } + file.write_all(&bytes)?; + } + file.sync_all()?; + Ok(()) +} + +/// Run the local two-stem owner. Hosted selections are validated here but must +/// be executed by `opentake-gen`, which owns credentials and network transport. +pub fn separate_stems( + request: StemSeparationRequest<'_>, + cancel: &MediaCancelToken, + progress: Option, +) -> Result { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + report(&progress, 0); + let model = match &request.execution { + StemExecution::Local { model_dir } => ensure_local_stem_model(model_dir)?, + StemExecution::Hosted { provider, model } => { + if provider.trim().is_empty() || model.trim().is_empty() { + return Err(MediaError::ModelInstall( + "stem_hosted_provider_and_model_required".to_string(), + )); + } + return Err(MediaError::ModelInstall(format!( + "stem_hosted_execution_requires_configured_provider:{provider}:{model}" + ))); + } + }; + report(&progress, 80); + let source_sha256 = digest_file(request.source, Some(cancel))?; + report(&progress, 160); + let spec = PcmSpec { + sample_rate: SAMPLE_RATE, + channels: CHANNELS, + format: PcmFormat::F32, + }; + let input = decode_pcm_interleaved_cancellable(request.source, &spec, None, cancel)?; + if !input.len().is_multiple_of(usize::from(CHANNELS)) { + return Err(MediaError::Decode( + "stem_input_interleaving_invalid".to_string(), + )); + } + report(&progress, 320); + + let mut vocals = Vec::new(); + let mut accompaniment = Vec::new(); + vocals + .try_reserve_exact(input.len()) + .map_err(|error| MediaError::Decode(format!("stem_audio_allocation_failed: {error}")))?; + accompaniment + .try_reserve_exact(input.len()) + .map_err(|error| MediaError::Decode(format!("stem_audio_allocation_failed: {error}")))?; + let mut side_energy = 0.0_f64; + for (index, frame) in input.chunks_exact(2).enumerate() { + if index.is_multiple_of(8 * 1024) { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + let completed = 320 + index.saturating_mul(360) / (input.len() / 2).max(1); + report(&progress, completed); + } + let centre = (frame[0] + frame[1]) * 0.5; + let side = (frame[0] - frame[1]) * 0.5; + vocals.extend_from_slice(&[centre, centre]); + accompaniment.extend_from_slice(&[side, side]); + side_energy += f64::from(side) * f64::from(side); + } + report(&progress, 700); + + fs::create_dir_all(request.output_dir)?; + let base = safe_source_stem(request.source); + let identity = &source_sha256[..12]; + let vocals_path = request + .output_dir + .join(format!("{base}-{identity}-vocals.wav")); + let accompaniment_path = request + .output_dir + .join(format!("{base}-{identity}-accompaniment.wav")); + let vocals_partial = vocals_path.with_extension("vocals.wav.partial"); + let accompaniment_partial = accompaniment_path.with_extension("accompaniment.wav.partial"); + + for path in [ + &vocals_partial, + &accompaniment_partial, + &vocals_path, + &accompaniment_path, + ] { + if path.exists() { + return Err(MediaError::Encode(format!( + "stem_output_already_exists: {}", + path.display() + ))); + } + } + let publish = (|| -> Result<()> { + write_wav_stereo(&vocals_partial, &vocals, cancel)?; + report(&progress, 820); + write_wav_stereo(&accompaniment_partial, &accompaniment, cancel)?; + report(&progress, 920); + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + fs::rename(&vocals_partial, &vocals_path)?; + fs::rename(&accompaniment_partial, &accompaniment_path)?; + Ok(()) + })(); + if publish.is_err() { + for path in [ + &vocals_partial, + &accompaniment_partial, + &vocals_path, + &accompaniment_path, + ] { + let _ = fs::remove_file(path); + } + } + publish?; + report(&progress, PROGRESS_TOTAL); + + let removed_cross_talk = if side_energy <= f64::EPSILON { + 60.0 + } else { + // The centre output has a mathematically zero side component. Bound the + // metric to a useful telemetry range instead of reporting infinity. + (10.0 * (side_energy / (side_energy * 1.0e-6)).log10()).clamp(0.0, 60.0) + }; + Ok(StemSeparationResult { + vocals: StemOutput { + path: vocals_path, + name: format!("{base} Vocals"), + }, + accompaniment: StemOutput { + path: accompaniment_path, + name: format!("{base} Accompaniment"), + }, + provenance: StemProvenance { + source_sha256, + execution: format!("local:{}", model.id), + model_sha256: Some(model.sha256), + }, + metrics: StemMetrics { + vocal_sdr_improvement_db: removed_cross_talk, + }, + }) +} diff --git a/crates/opentake-media/src/color.rs b/crates/opentake-media/src/color.rs new file mode 100644 index 00000000..f5133c0b --- /dev/null +++ b/crates/opentake-media/src/color.rs @@ -0,0 +1,114 @@ +//! Explicit source-color policy for the current SDR compositor. +//! +//! OpenTake retains the source signalling in the media manifest. PQ/HLG video +//! is converted to BT.709 before it becomes RGBA8 so seek-preview, continuous +//! playback and export all see the same display-referred pixels. This is an SDR +//! delivery policy, not an HDR passthrough claim. + +use opentake_domain::MediaColorMetadata; +use std::process::Command; +use std::sync::OnceLock; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum HdrDecodeBackend { + VideoToolbox, + Zscale, + Unsupported, +} + +fn reports_filter(output: &str, expected: &str) -> bool { + output.lines().any(|line| { + let mut fields = line.split_whitespace(); + let _flags = fields.next(); + fields.next() == Some(expected) + }) +} + +fn backend_from_filter_listing(output: &str) -> HdrDecodeBackend { + if cfg!(target_os = "macos") && reports_filter(output, "scale_vt") { + HdrDecodeBackend::VideoToolbox + } else if reports_filter(output, "zscale") { + HdrDecodeBackend::Zscale + } else { + HdrDecodeBackend::Unsupported + } +} + +fn hdr_decode_backend() -> HdrDecodeBackend { + static BACKEND: OnceLock = OnceLock::new(); + *BACKEND.get_or_init(|| { + let output = Command::new(crate::ff::ffmpeg_path()) + .args(["-hide_banner", "-filters"]) + .output(); + let Ok(output) = output else { + return HdrDecodeBackend::Unsupported; + }; + let mut listing = String::from_utf8_lossy(&output.stdout).into_owned(); + listing.push_str(&String::from_utf8_lossy(&output.stderr)); + backend_from_filter_listing(&listing) + }) +} + +/// FFmpeg filter chain for an HDR source entering the SDR RGBA compositor. +/// Tokens are selected from a fixed allowlist; untrusted probe strings are +/// never interpolated into a filter expression. +pub fn hdr_tonemap_filter(color: &MediaColorMetadata) -> Option { + let transfer = color.transfer.as_deref()?.to_ascii_lowercase(); + let input_transfer = match transfer.as_str() { + "smpte2084" | "pq" => "smpte2084", + "arib-std-b67" | "hlg" => "arib-std-b67", + _ => return None, + }; + match hdr_decode_backend() { + HdrDecodeBackend::VideoToolbox => { + // When the active macOS FFmpeg exposes scale_vt, VideoToolbox + // performs the metadata-driven EDR→SDR conversion in the hardware + // scaler. p010le is the supported hwdownload bridge before the + // ordinary software RGBA pipeline resumes. + Some( + "scale_vt=w=iw:h=ih:color_matrix=bt709:color_primaries=bt709:color_transfer=bt709,hwdownload,format=p010le" + .to_string(), + ) + } + HdrDecodeBackend::Zscale => Some(format!( + "zscale=pin=bt2020:tin={input_transfer}:min=bt2020nc:rin=limited:t=linear:npl=100,format=gbrpf32le,tonemap=mobius:param=0.3:desat=2,zscale=p=bt709:t=bt709:m=bt709:r=limited" + )), + HdrDecodeBackend::Unsupported => None, + } +} + +/// Decoder input arguments required by the platform HDR conversion path. +pub fn hdr_decode_input_args(color: &MediaColorMetadata) -> Vec { + if color.is_hdr() && hdr_decode_backend() == HdrDecodeBackend::VideoToolbox { + vec![ + "-hwaccel".into(), + "videotoolbox".into(), + "-hwaccel_output_format".into(), + "videotoolbox_vld".into(), + ] + } else { + Vec::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn packaged_zscale_listing_never_selects_unavailable_videotoolbox_filter() { + let listing = " .S. tonemap V->V Conversion\n .SC zscale V->V Apply resizing"; + assert_eq!( + backend_from_filter_listing(listing), + HdrDecodeBackend::Zscale + ); + } + + #[test] + fn missing_hdr_filters_are_reported_as_unsupported() { + assert_eq!( + backend_from_filter_listing(" .. scale V->V Scale video"), + HdrDecodeBackend::Unsupported + ); + } +} diff --git a/crates/opentake-media/src/decode/frame.rs b/crates/opentake-media/src/decode/frame.rs index 3057e231..169ece8c 100644 --- a/crates/opentake-media/src/decode/frame.rs +++ b/crates/opentake-media/src/decode/frame.rs @@ -48,6 +48,316 @@ impl Default for FrameRequest { } } +/// Source-frame reconstruction policy used when a timeline requests frames at +/// a different rate than the decoded asset. Optical flow is a deterministic +/// local motion-compensated path; it never implies a cloud/model dependency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameInterpolationMode { + Nearest, + Blend, + OpticalFlow, +} + +/// Explicit recovery behavior when optical flow is unavailable on the current +/// device/runtime. The caller chooses quality, determinism, or fail-closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameInterpolationFallback { + Nearest, + Blend, + Error, +} + +/// One target-rate sample mapped back into the source-frame interval. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FrameRateSample { + pub timestamp_secs: f64, + pub source_frame: u64, + pub next_source_frame: u64, + pub source_alpha: f64, +} + +/// Result of one pair interpolation, including the effective mode after an +/// explicit unsupported-device fallback. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FrameInterpolationResult { + pub frame: RgbaFrame, + pub mode_used: FrameInterpolationMode, +} + +/// Map a finite source sequence onto a target frame rate while preserving both +/// endpoint timestamps exactly. Interior timestamps follow the target-rate +/// grid; the final sample is pinned to the source's final presentation time. +pub fn convert_frame_rate( + source_frame_count: u64, + source_fps: f64, + target_fps: f64, +) -> Result> { + if source_frame_count == 0 { + return Err(MediaError::Decode( + "source_frame_count must be greater than zero".to_string(), + )); + } + if !source_fps.is_finite() || source_fps <= 0.0 { + return Err(MediaError::Decode( + "source_fps must be finite and greater than zero".to_string(), + )); + } + if !target_fps.is_finite() || target_fps <= 0.0 { + return Err(MediaError::Decode( + "target_fps must be finite and greater than zero".to_string(), + )); + } + if source_frame_count == 1 { + return Ok(vec![FrameRateSample { + timestamp_secs: 0.0, + source_frame: 0, + next_source_frame: 0, + source_alpha: 0.0, + }]); + } + + let source_last = source_frame_count - 1; + let duration_secs = source_last as f64 / source_fps; + let target_intervals = (duration_secs * target_fps).round().max(1.0) as u64; + let mut samples = Vec::with_capacity(target_intervals as usize + 1); + for output_frame in 0..=target_intervals { + let timestamp_secs = if output_frame == target_intervals { + duration_secs + } else { + (output_frame as f64 / target_fps).min(duration_secs) + }; + let source_position = (timestamp_secs * source_fps).clamp(0.0, source_last as f64); + let source_frame = source_position.floor() as u64; + let next_source_frame = source_frame.saturating_add(1).min(source_last); + let source_alpha = if source_frame == next_source_frame { + 0.0 + } else { + source_position - source_frame as f64 + }; + samples.push(FrameRateSample { + timestamp_secs, + source_frame, + next_source_frame, + source_alpha, + }); + } + Ok(samples) +} + +/// Interpolate two equal-size RGBA frames at `alpha` in `[0, 1]`. +/// +/// The optical-flow path estimates a deterministic local block-motion field, +/// warps both endpoints toward the requested instant, then blends the aligned +/// pixels. This traditional path is intentionally model-free and provides a +/// stable baseline for preview/export parity. +pub fn interpolate_frame_pair( + first: &RgbaFrame, + last: &RgbaFrame, + alpha: f64, + requested: FrameInterpolationMode, + fallback: FrameInterpolationFallback, + optical_flow_available: bool, +) -> Result { + if first.width != last.width + || first.height != last.height + || first.rgba.len() != last.rgba.len() + { + return Err(MediaError::Decode( + "interpolation frames must have identical dimensions".to_string(), + )); + } + if !alpha.is_finite() { + return Err(MediaError::Decode( + "interpolation alpha must be finite".to_string(), + )); + } + + let mode_used = if requested == FrameInterpolationMode::OpticalFlow && !optical_flow_available { + match fallback { + FrameInterpolationFallback::Nearest => FrameInterpolationMode::Nearest, + FrameInterpolationFallback::Blend => FrameInterpolationMode::Blend, + FrameInterpolationFallback::Error => { + return Err(MediaError::Decode( + "optical-flow interpolation is unavailable and fallback is Error".to_string(), + )); + } + } + } else { + requested + }; + + let alpha = alpha.clamp(0.0, 1.0); + let frame = if alpha == 0.0 { + first.clone() + } else if alpha == 1.0 { + last.clone() + } else { + match mode_used { + FrameInterpolationMode::Nearest => { + if alpha < 0.5 { + first.clone() + } else { + last.clone() + } + } + FrameInterpolationMode::Blend => blend_frames(first, last, alpha), + FrameInterpolationMode::OpticalFlow => optical_flow_frame(first, last, alpha), + } + }; + + Ok(FrameInterpolationResult { frame, mode_used }) +} + +fn blend_frames(first: &RgbaFrame, last: &RgbaFrame, alpha: f64) -> RgbaFrame { + let rgba = first + .rgba + .iter() + .zip(&last.rgba) + .map(|(&a, &b)| lerp_channel(a, b, alpha)) + .collect(); + RgbaFrame::new(first.width, first.height, rgba) +} + +fn optical_flow_frame(first: &RgbaFrame, last: &RgbaFrame, alpha: f64) -> RgbaFrame { + let flow = estimate_block_motion(first, last); + let mut rgba = vec![0; first.rgba.len()]; + for y in 0..first.height { + for x in 0..first.width { + let (motion_x, motion_y) = flow.at(x, y); + let x = x as f64; + let y = y as f64; + let from_first = sample_bilinear(first, x - alpha * motion_x, y - alpha * motion_y); + let from_last = sample_bilinear( + last, + x + (1.0 - alpha) * motion_x, + y + (1.0 - alpha) * motion_y, + ); + let offset = ((y as u32 * first.width + x as u32) * 4) as usize; + for channel in 0..4 { + rgba[offset + channel] = + lerp_channel(from_first[channel], from_last[channel], alpha); + } + } + } + RgbaFrame::new(first.width, first.height, rgba) +} + +struct BlockMotionField { + block_size: u32, + columns: u32, + rows: u32, + vectors: Vec<(f64, f64)>, +} + +impl BlockMotionField { + fn at(&self, x: u32, y: u32) -> (f64, f64) { + let column = (x / self.block_size).min(self.columns.saturating_sub(1)); + let row = (y / self.block_size).min(self.rows.saturating_sub(1)); + self.vectors[(row * self.columns + column) as usize] + } +} + +/// Estimate a deterministic local motion field with block matching. Bounded +/// search and per-block spatial sampling avoid the whole-frame distortion of a +/// single global translation vector without introducing a model dependency. +fn estimate_block_motion(first: &RgbaFrame, last: &RgbaFrame) -> BlockMotionField { + let shortest = first.width.min(first.height).max(1); + let block_size = shortest.min(32); + let columns = first.width.div_ceil(block_size); + let rows = first.height.div_ceil(block_size); + let search_radius = (block_size / 2).clamp(1, 12) as i32; + let sample_step = (block_size / 8).max(1); + let mut vectors = Vec::with_capacity((columns * rows) as usize); + + for row in 0..rows { + for column in 0..columns { + let start_x = column * block_size; + let start_y = row * block_size; + let end_x = (start_x + block_size).min(first.width); + let end_y = (start_y + block_size).min(first.height); + let mut best = (f64::INFINITY, i32::MAX, 0, 0); + for dy in -search_radius..=search_radius { + for dx in -search_radius..=search_radius { + let mut error = 0.0; + let mut samples = 0u32; + for y in (start_y..end_y).step_by(sample_step as usize) { + for x in (start_x..end_x).step_by(sample_step as usize) { + let target_x = x as i32 + dx; + let target_y = y as i32 + dy; + let target = if target_x < 0 + || target_y < 0 + || target_x >= last.width as i32 + || target_y >= last.height as i32 + { + 255.0 + } else { + luma_at(last, target_x as u32, target_y as u32) + }; + error += (luma_at(first, x, y) - target).abs(); + samples += 1; + } + } + let mean_error = error / samples.max(1) as f64; + let distance = dx * dx + dy * dy; + let candidate = (mean_error, distance, dy, dx); + if candidate < best { + best = candidate; + } + } + } + vectors.push((best.3 as f64, best.2 as f64)); + } + } + + BlockMotionField { + block_size, + columns, + rows, + vectors, + } +} + +fn luma_at(frame: &RgbaFrame, x: u32, y: u32) -> f64 { + let offset = ((y * frame.width + x) * 4) as usize; + let r = frame.rgba[offset] as f64; + let g = frame.rgba[offset + 1] as f64; + let b = frame.rgba[offset + 2] as f64; + let a = frame.rgba[offset + 3] as f64 / 255.0; + (0.2126 * r + 0.7152 * g + 0.0722 * b) * a +} + +fn sample_bilinear(frame: &RgbaFrame, x: f64, y: f64) -> [u8; 4] { + let x0 = x.floor() as i64; + let y0 = y.floor() as i64; + let fx = x - x0 as f64; + let fy = y - y0 as f64; + let mut out = [0; 4]; + for (channel, value) in out.iter_mut().enumerate() { + let p00 = sample_channel(frame, x0, y0, channel); + let p10 = sample_channel(frame, x0 + 1, y0, channel); + let p01 = sample_channel(frame, x0, y0 + 1, channel); + let p11 = sample_channel(frame, x0 + 1, y0 + 1, channel); + let top = p00 + (p10 - p00) * fx; + let bottom = p01 + (p11 - p01) * fx; + *value = (top + (bottom - top) * fy).round().clamp(0.0, 255.0) as u8; + } + out +} + +fn sample_channel(frame: &RgbaFrame, x: i64, y: i64, channel: usize) -> f64 { + if x < 0 || y < 0 || x >= frame.width as i64 || y >= frame.height as i64 { + return if channel == 3 { 255.0 } else { 0.0 }; + } + let offset = ((y as u32 * frame.width + x as u32) * 4) as usize; + frame.rgba[offset + channel] as f64 +} + +fn lerp_channel(first: u8, last: u8, alpha: f64) -> u8 { + (first as f64 + (last as f64 - first as f64) * alpha) + .round() + .clamp(0.0, 255.0) as u8 +} + /// Scale `(w, h)` down to fit within `max` while preserving aspect ratio. Never /// enlarges. A zero in either `max` dimension disables that bound. Mirrors /// `AVAssetImageGenerator.maximumSize` semantics ("not larger than this box, @@ -74,9 +384,21 @@ pub fn fit_within(w: u32, h: u32, max: (u32, u32)) -> (u32, u32) { /// Build the ffmpeg arg list for decoding one frame to rawvideo RGBA on stdout. /// Pure so the exact CLI contract is testable. +#[cfg(test)] fn frame_args(path: &Path, req: &FrameRequest) -> Vec { + frame_args_with_color(path, req, None) +} + +fn frame_args_with_color( + path: &Path, + req: &FrameRequest, + color: Option<&opentake_domain::MediaColorMetadata>, +) -> Vec { let seek = (req.time_secs - req.tolerance_secs).max(0.0); let mut args: Vec = Vec::new(); + if let Some(color) = color { + args.extend(crate::color::hdr_decode_input_args(color)); + } // Fast input seek to just before the target keyframe window. args.push("-ss".into()); args.push(format!("{seek:.6}")); @@ -87,6 +409,9 @@ fn frame_args(path: &Path, req: &FrameRequest) -> Vec { args.push("1".into()); let mut filters: Vec = Vec::new(); + if let Some(filter) = color.and_then(crate::color::hdr_tonemap_filter) { + filters.push(filter); + } if req.apply_rotation { // Honor the display matrix when transposing (ffmpeg applies it via the // autorotate behavior; the scale filter runs after rotation). @@ -134,8 +459,17 @@ pub fn decode_frame_at_cancellable( if cancel.is_cancelled() { return Err(MediaError::Cancelled); } + // Probe color only for ordinary files. FIFOs/device inputs are valid FFmpeg + // sources too; opening them once for ffprobe would consume or block the + // stream before the actual cancellable decoder child is spawned. + let color = path + .metadata() + .ok() + .filter(|metadata| metadata.is_file()) + .and_then(|_| crate::probe::probe(path).ok()) + .and_then(|probe| probe.color); let mut child = ff::ffmpeg() - .args(frame_args(path, req)) + .args(frame_args_with_color(path, req, color.as_ref())) .spawn() .map_err(|e| MediaError::Ffmpeg(format!("spawn: {e}")))?; cancel.child_spawned(); diff --git a/crates/opentake-media/src/decode/mod.rs b/crates/opentake-media/src/decode/mod.rs index 34cbdec5..d0fe7653 100644 --- a/crates/opentake-media/src/decode/mod.rs +++ b/crates/opentake-media/src/decode/mod.rs @@ -8,8 +8,9 @@ pub mod stream; pub use audio_stream::{decode_pcm_interleaved, decode_pcm_interleaved_cancellable}; pub use frame::{ - decode_frame_at, decode_frame_at_cancellable, decode_frames_at, decode_frames_at_cancellable, - fit_within, FrameRequest, + convert_frame_rate, decode_frame_at, decode_frame_at_cancellable, decode_frames_at, + decode_frames_at_cancellable, fit_within, interpolate_frame_pair, FrameInterpolationFallback, + FrameInterpolationMode, FrameInterpolationResult, FrameRateSample, FrameRequest, }; pub use pcm::{ extract_pcm, extract_pcm_cancellable, extract_pcm_cancellable_with_progress, PcmBuffer, diff --git a/crates/opentake-media/src/decode/stream.rs b/crates/opentake-media/src/decode/stream.rs index 1a03bd80..b20b3fb6 100644 --- a/crates/opentake-media/src/decode/stream.rs +++ b/crates/opentake-media/src/decode/stream.rs @@ -182,7 +182,10 @@ fn run_video_stream( tx: SyncSender>, control: StreamDecodeControl, ) { - let args = video_stream_args(&req); + let color = crate::probe::probe(&req.path) + .ok() + .and_then(|probe| probe.color); + let args = video_stream_args_with_color(&req, color.as_ref()); let mut child = match ff::ffmpeg().args(args).spawn() { Ok(child) => child, Err(e) => { @@ -275,10 +278,21 @@ fn frame_to_secs(frame: i64, fps: i32) -> f64 { frame.max(0) as f64 / fps.max(1) as f64 } +#[cfg(test)] fn video_stream_args(req: &VideoStreamRequest) -> Vec { + video_stream_args_with_color(req, None) +} + +fn video_stream_args_with_color( + req: &VideoStreamRequest, + color: Option<&opentake_domain::MediaColorMetadata>, +) -> Vec { let mut args = Vec::new(); args.push("-ss".to_string()); args.push(format!("{:.6}", req.start_secs())); + if let Some(color) = color { + args.extend(crate::color::hdr_decode_input_args(color)); + } if !req.apply_rotation { args.push("-noautorotate".to_string()); } @@ -294,7 +308,11 @@ fn video_stream_args(req: &VideoStreamRequest) -> Vec { args.push(frame_limit.to_string()); } - let mut filters = vec![format!("fps=fps={}", req.timeline_fps)]; + let mut filters = Vec::new(); + if let Some(filter) = color.and_then(crate::color::hdr_tonemap_filter) { + filters.push(filter); + } + filters.push(format!("fps=fps={}", req.timeline_fps)); if req.max_size.0 > 0 || req.max_size.1 > 0 { let mw = if req.max_size.0 > 0 { req.max_size.0.to_string() diff --git a/crates/opentake-media/src/encode/mix.rs b/crates/opentake-media/src/encode/mix.rs index d06bc132..beb8b01f 100644 --- a/crates/opentake-media/src/encode/mix.rs +++ b/crates/opentake-media/src/encode/mix.rs @@ -20,6 +20,25 @@ /// The canonical mixdown sample rate. 48 kHz is the export-audio standard and /// what the encoder requests from ffmpeg for the muxed AAC/LPCM track. pub const MIX_SAMPLE_RATE: u32 = 48_000; +/// Headroom reserved below a requested true-peak ceiling for reconstruction +/// overshoot introduced by lossy codecs such as AAC. Two dB is intentional: +/// the release export's AAC encoder reconstructed the speech acceptance fixture +/// 1.77 dB above the clamped PCM sample peak, so a one-dB margin did not keep +/// the encoded deliverable below the user-selected ceiling. +pub const TRUE_PEAK_CODEC_SAFETY_DB: f64 = 2.0; + +/// Apply the shared preview/export ceiling stage in-place. `None` preserves the +/// legacy rail clamp performed by the surrounding mixer. +pub fn apply_true_peak_ceiling(samples: &mut [f32], ceiling_dbtp: Option) { + let Some(ceiling_dbtp) = ceiling_dbtp.filter(|value| value.is_finite()) else { + return; + }; + let ceiling = 10.0_f32 + .powf(((ceiling_dbtp - TRUE_PEAK_CODEC_SAFETY_DB).clamp(-120.0, 0.0) / 20.0) as f32); + for sample in samples { + *sample = sample.clamp(-ceiling, ceiling); + } +} /// One audio clip's contribution to the mix: a mono f32 source window plus the /// per-sample gain to apply, laid down starting at `start_sample` on the shared @@ -192,6 +211,16 @@ mod tests { assert_eq!(mix_clips(&[c]).unwrap(), vec![0.0, 0.5, 1.0]); } + #[test] + fn true_peak_ceiling_reserves_codec_reconstruction_margin() { + let mut samples = vec![1.0, -1.0, 0.25]; + apply_true_peak_ceiling(&mut samples, Some(-1.0)); + let expected = 10.0_f32.powf(-3.0 / 20.0); + assert!((samples[0] - expected).abs() < 1e-6); + assert!((samples[1] + expected).abs() < 1e-6); + assert_eq!(samples[2], 0.25); + } + #[test] fn static_gain_helper_skips_envelope_at_unity() { let c = ClipAudio::with_static_gain(0, vec![0.4, 0.4], 1.0); diff --git a/crates/opentake-media/src/encode/mod.rs b/crates/opentake-media/src/encode/mod.rs index 9b871006..8ee4ef72 100644 --- a/crates/opentake-media/src/encode/mod.rs +++ b/crates/opentake-media/src/encode/mod.rs @@ -21,7 +21,7 @@ use std::thread::{self, JoinHandle}; use std::time::Duration; use crate::cancel::MediaCancelToken; -use crate::decode::pcm::PcmBuffer; +use crate::decode::pcm::{PcmBuffer, PcmFormat, PcmSpec}; use crate::error::{MediaError, Result}; use crate::frame::RgbaFrame; @@ -50,6 +50,10 @@ fn encode_args(out: &Path, w: u32, h: u32, fps: i32, preset: &ExportPreset) -> V args.push(preset.vcodec_arg().into()); args.push("-pix_fmt".into()); args.push(preset.pix_fmt_arg().into()); + if preset.codec == VideoCodec::ProRes4444 { + args.push("-profile:v".into()); + args.push("4444".into()); + } args.extend(preset.color_args()); args.push(out.to_string_lossy().into_owned()); @@ -117,10 +121,16 @@ pub struct VideoEncoder { first_pass: PathBuf, output: File, acodec: &'static str, - pending_audio: Option, + pending_audio: Option, child_reaped: bool, } +struct PendingAudio { + path: PathBuf, + spec: PcmSpec, + sample_count: u64, +} + impl VideoEncoder { /// Start an encoder writing to `out`. `w`/`h` must already be even. pub fn new(out: &Path, w: u32, h: u32, fps: i32, preset: &ExportPreset) -> Result { @@ -149,7 +159,7 @@ impl VideoEncoder { ::from_mode(0o700), ) .map_err(MediaError::Io)?; - let extension = if preset.codec == VideoCodec::ProRes422 { + let extension = if matches!(preset.codec, VideoCodec::ProRes422 | VideoCodec::ProRes4444) { "mov" } else { "mp4" @@ -229,16 +239,81 @@ impl VideoEncoder { Ok(()) } - /// Record the mixed-down mono audio buffer to mux on `finish`. The buffer's - /// `spec.sample_rate` is the rate ffmpeg is told to read the muxed PCM at - /// (the orchestrator decodes/mixes at [`MIX_SAMPLE_RATE`]). An empty buffer - /// is ignored — `finish` then keeps the video-only output. - pub fn push_audio(&mut self, pcm: PcmBuffer) { + /// Record one complete mixed-down mono audio buffer. Internally this uses + /// the same file-backed chunk sink as long-timeline export, so the encoder + /// never retains the caller's `Vec` until `finish`. + pub fn push_audio(&mut self, pcm: PcmBuffer) -> Result<()> { + if let Some(pending) = self.pending_audio.take() { + let _ = std::fs::remove_file(pending.path); + } if pcm.samples_f32.is_empty() { - self.pending_audio = None; + return Ok(()); + } + self.push_audio_chunk(pcm.spec, &pcm.samples_f32, &MediaCancelToken::new()) + } + + /// Append one bounded mono f32 chunk to the private PCM spool used by the + /// final mux. The spool is file-backed, cancellable, and requires every + /// chunk to keep the same PCM contract. + pub fn push_audio_chunk( + &mut self, + spec: PcmSpec, + samples: &[f32], + cancel: &MediaCancelToken, + ) -> Result<()> { + if samples.is_empty() { + return Ok(()); + } + if spec.channels != 1 || spec.format != PcmFormat::F32 || spec.sample_rate == 0 { + return Err(MediaError::Encode( + "streamed audio must be mono f32 at a positive sample rate".to_string(), + )); + } + let path = self.workspace.path().join("audio.pcm"); + let mut output = if let Some(pending) = &self.pending_audio { + if pending.spec != spec { + return Err(MediaError::Encode( + "streamed audio format changed between chunks".to_string(), + )); + } + OpenOptions::new() + .append(true) + .open(&pending.path) + .map_err(MediaError::Io)? } else { - self.pending_audio = Some(pcm); + OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(MediaError::Io)? + }; + for chunk in samples.chunks(OUTPUT_COPY_CHUNK / 2) { + if cancel.checkpoint() { + return Err(MediaError::Cancelled); + } + output + .write_all(&mix::mono_f32_to_s16le(chunk)) + .map_err(MediaError::Io)?; } + output.flush().map_err(MediaError::Io)?; + let appended = u64::try_from(samples.len()) + .map_err(|_| MediaError::Encode("streamed audio sample count overflow".to_string()))?; + match &mut self.pending_audio { + Some(pending) => { + pending.sample_count = + pending.sample_count.checked_add(appended).ok_or_else(|| { + MediaError::Encode("streamed audio sample count overflow".to_string()) + })?; + } + None => { + self.pending_audio = Some(PendingAudio { + path, + spec, + sample_count: appended, + }); + } + } + Ok(()) } /// Abort a mid-stream encode (e.g. a user cancel): kill the ffmpeg child and @@ -276,7 +351,7 @@ impl VideoEncoder { report_progress(progress, FIRST_PASS_END); match self.pending_audio.take() { - Some(pcm) => self.mux_audio(&pcm, cancel, progress, mux_wait_hook)?, + Some(audio) => self.mux_audio(&audio, cancel, progress, mux_wait_hook)?, None => self.copy_video_only(cancel, progress)?, }; if cancel.checkpoint() { @@ -397,20 +472,11 @@ impl VideoEncoder { fn mux_audio( &mut self, - pcm: &PcmBuffer, + audio: &PendingAudio, cancel: &MediaCancelToken, progress: Option<&EncodeProgressCallback>, mux_wait_hook: Option<&dyn Fn()>, ) -> Result<()> { - let pcm_path = self.workspace.path().join("audio.pcm"); - let mut pcm_tmp = OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&pcm_path) - .map_err(MediaError::Io)?; - write_pcm_s16le_cancellable(&pcm.samples_f32, &mut pcm_tmp, cancel, progress, None)?; - pcm_tmp.flush().map_err(MediaError::Io)?; report_progress(progress, PCM_WRITE_END); let mux_path = self.workspace.path().join( @@ -427,9 +493,9 @@ impl VideoEncoder { ); let args = mux_args( &self.first_pass, - &pcm_path, + &audio.path, &mux_path, - pcm.spec.sample_rate, + audio.spec.sample_rate, self.acodec, ); let mut child = crate::ff::ffmpeg() @@ -572,6 +638,7 @@ fn drain_stderr(mut stderr: ChildStderr) -> Result<()> { } } +#[cfg(test)] fn write_pcm_s16le_cancellable( samples: &[f32], destination: &mut File, @@ -679,6 +746,36 @@ mod tests { .is_ok_and(|status| status.success()) } + #[test] + fn audio_chunks_spool_incrementally_without_retaining_the_timeline_mix() { + assert!(crate::ff::ffmpeg_available(), "test requires FFmpeg"); + let temp = tempfile::tempdir().unwrap(); + let output = temp.path().join("chunked.mp4"); + let preset = ExportPreset::new(VideoCodec::H264, ExportResolution::P720); + let mut encoder = VideoEncoder::new(&output, 2, 2, 1, &preset).unwrap(); + encoder + .push_frame(&RgbaFrame::new(2, 2, vec![0; 2 * 2 * 4])) + .unwrap(); + let spec = PcmSpec { + sample_rate: 48_000, + channels: 1, + format: PcmFormat::F32, + }; + let cancel = MediaCancelToken::new(); + encoder + .push_audio_chunk(spec, &[0.25, -0.25], &cancel) + .unwrap(); + encoder + .push_audio_chunk(spec, &[0.5, -0.5], &cancel) + .unwrap(); + let pending = encoder.pending_audio.as_ref().unwrap(); + assert_eq!(pending.sample_count, 4); + assert_eq!(std::fs::metadata(&pending.path).unwrap().len(), 8); + + encoder.finish().unwrap(); + assert!(output.is_file()); + } + #[cfg(unix)] #[test] fn dropping_live_encoder_reaps_child_and_releases_output() { @@ -747,13 +844,24 @@ mod tests { } #[test] - fn encode_args_prores_pixfmt_and_no_color_tag() { + fn encode_args_prores_pixfmt_and_bt709_delivery_tags() { let preset = ExportPreset::new(VideoCodec::ProRes422, ExportResolution::P2160); let args = encode_args(Path::new("/o.mov"), 3840, 2160, 30, &preset); assert!(args.windows(2).any(|w| w == ["-c:v", "prores_ks"])); assert!(args.windows(2).any(|w| w == ["-pix_fmt", "yuv422p10le"])); - // ProRes path does not add BT.709 color tags here. - assert!(!args.windows(2).any(|w| w == ["-colorspace", "bt709"])); + assert!(args.windows(2).any(|w| w == ["-colorspace", "bt709"])); + assert!(args + .iter() + .any(|arg| arg.contains("setparams=color_primaries=bt709"))); + } + + #[test] + fn encode_args_prores_4444_preserve_alpha() { + let preset = ExportPreset::new(VideoCodec::ProRes4444, ExportResolution::P1080); + let args = encode_args(Path::new("/matte.mov"), 1920, 1080, 30, &preset); + assert!(args.windows(2).any(|w| w == ["-c:v", "prores_ks"])); + assert!(args.windows(2).any(|w| w == ["-pix_fmt", "yuva444p10le"])); + assert!(args.windows(2).any(|w| w == ["-profile:v", "4444"])); } #[test] diff --git a/crates/opentake-media/src/encode/preset.rs b/crates/opentake-media/src/encode/preset.rs index 8c669cd5..3ef0e75a 100644 --- a/crates/opentake-media/src/encode/preset.rs +++ b/crates/opentake-media/src/encode/preset.rs @@ -9,6 +9,8 @@ pub enum VideoCodec { H264, H265, ProRes422, + /// ProRes 4444 with an alpha plane for local generated derivatives. + ProRes4444, } /// Short-edge target resolution. @@ -48,6 +50,7 @@ impl ExportPreset { VideoCodec::H264 => "libx264", VideoCodec::H265 => "libx265", VideoCodec::ProRes422 => "prores_ks", + VideoCodec::ProRes4444 => "prores_ks", } } @@ -55,7 +58,7 @@ impl ExportPreset { /// AAC (upstream presets). pub fn acodec_arg(&self) -> &'static str { match self.codec { - VideoCodec::ProRes422 => "pcm_s16le", + VideoCodec::ProRes422 | VideoCodec::ProRes4444 => "pcm_s16le", _ => "aac", } } @@ -65,24 +68,27 @@ impl ExportPreset { pub fn pix_fmt_arg(&self) -> &'static str { match self.codec { VideoCodec::ProRes422 => "yuv422p10le", + VideoCodec::ProRes4444 => "yuva444p10le", _ => "yuv420p", } } - /// BT.709 color-tagging args (primaries/transfer/matrix), applied for the - /// H.26x lossy codecs to match upstream's locked BT.709 pipeline. + /// BT.709 delivery tagging. `setparams` writes all three properties onto + /// every frame before the encoder sees it; the stream flags are retained as + /// an explicit container/codec request. This combination is required by + /// current FFmpeg/libx264, where stream flags alone leave primaries and + /// transfer as `unknown` in the produced bitstream. pub fn color_args(&self) -> Vec { - match self.codec { - VideoCodec::ProRes422 => vec![], - _ => vec![ - "-colorspace".into(), - "bt709".into(), - "-color_primaries".into(), - "bt709".into(), - "-color_trc".into(), - "bt709".into(), - ], - } + vec![ + "-vf".into(), + "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709".into(), + "-colorspace".into(), + "bt709".into(), + "-color_primaries".into(), + "bt709".into(), + "-color_trc".into(), + "bt709".into(), + ] } } @@ -116,10 +122,15 @@ mod tests { assert_eq!(prores.vcodec_arg(), "prores_ks"); assert_eq!(prores.acodec_arg(), "pcm_s16le"); // LPCM assert_eq!(prores.pix_fmt_arg(), "yuv422p10le"); + + let alpha = ExportPreset::new(VideoCodec::ProRes4444, ExportResolution::P1080); + assert_eq!(alpha.vcodec_arg(), "prores_ks"); + assert_eq!(alpha.acodec_arg(), "pcm_s16le"); + assert_eq!(alpha.pix_fmt_arg(), "yuva444p10le"); } #[test] - fn h26x_get_bt709_color_args_prores_does_not() { + fn every_delivery_codec_gets_bt709_frame_and_stream_tags() { let h265 = ExportPreset::new(VideoCodec::H265, ExportResolution::P720); let args = h265.color_args(); assert!(args.windows(2).any(|w| w == ["-colorspace", "bt709"])); @@ -127,7 +138,10 @@ mod tests { assert!(args.windows(2).any(|w| w == ["-color_trc", "bt709"])); let prores = ExportPreset::new(VideoCodec::ProRes422, ExportResolution::P720); - assert!(prores.color_args().is_empty()); + assert!(prores + .color_args() + .iter() + .any(|arg| arg.contains("setparams=color_primaries=bt709"))); } #[test] diff --git a/crates/opentake-media/src/ff.rs b/crates/opentake-media/src/ff.rs index 4bfcb91b..b6229ba1 100644 --- a/crates/opentake-media/src/ff.rs +++ b/crates/opentake-media/src/ff.rs @@ -1,4 +1,5 @@ -//! Thin internal helpers for driving the system `ffmpeg`/`ffprobe` binaries. +//! Thin internal helpers for driving bundled or development `ffmpeg`/`ffprobe` +//! binaries. //! //! We deliberately do **not** link libav*: the local toolchain is ffmpeg 8.1 //! (libavcodec 62) which the C-binding crates do not support, and pkg-config is @@ -6,23 +7,86 @@ //! binary discovery and one-shot ffprobe JSON queries so the higher-level decode //! modules stay readable. //! -//! Environment overrides `OPENTAKE_FFMPEG` / `OPENTAKE_FFPROBE` let callers (and -//! packaged builds) point at a bundled binary. +//! Packaged binaries live beside the OpenTake executable. Environment overrides +//! `OPENTAKE_FFMPEG` / `OPENTAKE_FFPROBE` remain available to tests and +//! development tools; the desktop shell pins them to the bundled sidecars before +//! media initialization. use std::ffi::OsString; use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use ffmpeg_sidecar::command::FfmpegCommand; -/// Path to the `ffmpeg` binary: `$OPENTAKE_FFMPEG`, else `ffmpeg` on `PATH`. +fn sidecar_filename(binary: &str) -> String { + if cfg!(windows) { + format!("{binary}.exe") + } else { + binary.to_string() + } +} + +/// Return a regular, non-symlink sidecar next to `executable`. +/// +/// Keeping this pure helper separate makes the packaged-path security boundary +/// deterministic to test without mutating the process executable or PATH. +pub fn packaged_sidecar_beside(executable: &Path, binary: &str) -> Option { + let parent = executable.parent()?; + let candidate = parent.join(sidecar_filename(binary)); + let metadata = std::fs::symlink_metadata(&candidate).ok()?; + if metadata.file_type().is_file() && !metadata.file_type().is_symlink() { + Some(candidate) + } else { + None + } +} + +/// Find a verified-by-the-package-manager sidecar beside the current executable. +/// Runtime code still checks that the path is a regular file; the build/package +/// pipeline owns its pinned SHA-256 and version verification. +pub fn packaged_sidecar_path(binary: &str) -> Option { + let executable = std::env::current_exe().ok()?; + packaged_sidecar_beside(&executable, binary) +} + +/// Resolve one CLI tool without mutating global process state in tests. +/// +/// An explicit override always wins, then a regular non-symlink packaged +/// sidecar beside the executable, and finally the platform command name on +/// `PATH`. +fn resolve_cli_path( + override_path: Option, + executable: Option<&Path>, + binary: &str, +) -> OsString { + override_path + .or_else(|| { + executable + .and_then(|path| packaged_sidecar_beside(path, binary)) + .map(PathBuf::into_os_string) + }) + .unwrap_or_else(|| OsString::from(binary)) +} + +/// Path to `ffmpeg`: explicit development override, packaged sidecar, then PATH. pub fn ffmpeg_path() -> OsString { - std::env::var_os("OPENTAKE_FFMPEG").unwrap_or_else(|| OsString::from("ffmpeg")) + let executable = std::env::current_exe().ok(); + resolve_cli_path( + std::env::var_os("OPENTAKE_FFMPEG"), + executable.as_deref(), + "ffmpeg", + ) } -/// Path to the `ffprobe` binary: `$OPENTAKE_FFPROBE`, else `ffprobe` on `PATH`. +/// Path to `ffprobe`: explicit development override, packaged sidecar, then PATH. pub fn ffprobe_path() -> OsString { - std::env::var_os("OPENTAKE_FFPROBE").unwrap_or_else(|| OsString::from("ffprobe")) + let executable = std::env::current_exe().ok(); + resolve_cli_path( + std::env::var_os("OPENTAKE_FFPROBE"), + executable.as_deref(), + "ffprobe", + ) } /// A fresh `FfmpegCommand` bound to [`ffmpeg_path`]. @@ -114,17 +178,69 @@ mod tests { #[test] fn env_override_is_respected_for_ffmpeg() { - // We can't safely mutate process env in parallel tests for the *default*, - // but we can assert the default value when the var is unset in this proc. - if std::env::var_os("OPENTAKE_FFMPEG").is_none() { - assert_eq!(ffmpeg_path(), OsString::from("ffmpeg")); - } + let temp = tempfile::tempdir().unwrap(); + let executable = temp.path().join(if cfg!(windows) { + "opentake.exe" + } else { + "opentake" + }); + std::fs::write(&executable, b"app").unwrap(); + std::fs::write(temp.path().join(sidecar_filename("ffmpeg")), b"sidecar").unwrap(); + + assert_eq!( + resolve_cli_path( + Some(OsString::from("/opt/opentake/custom-ffmpeg")), + Some(&executable), + "ffmpeg", + ), + OsString::from("/opt/opentake/custom-ffmpeg"), + ); } #[test] fn default_ffprobe_is_ffprobe() { - if std::env::var_os("OPENTAKE_FFPROBE").is_none() { - assert_eq!(ffprobe_path(), OsString::from("ffprobe")); - } + assert_eq!( + resolve_cli_path(None, None, "ffprobe"), + OsString::from("ffprobe") + ); + } + + #[test] + fn packaged_sidecar_must_be_regular_and_beside_executable() { + let temp = tempfile::tempdir().unwrap(); + let executable = temp.path().join(if cfg!(windows) { + "opentake.exe" + } else { + "opentake" + }); + std::fs::write(&executable, b"app").unwrap(); + let sidecar = temp.path().join(sidecar_filename("ffmpeg")); + std::fs::write(&sidecar, b"sidecar").unwrap(); + + assert_eq!( + packaged_sidecar_beside(&executable, "ffmpeg"), + Some(sidecar.clone()) + ); + assert_eq!( + resolve_cli_path(None, Some(&executable), "ffmpeg"), + sidecar.into_os_string() + ); + assert_eq!(packaged_sidecar_beside(&executable, "ffprobe"), None); + } + + #[cfg(unix)] + #[test] + fn packaged_sidecar_rejects_symlink() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let executable = temp.path().join("opentake"); + let outside = temp.path().join("outside"); + let sidecar = temp.path().join("ffmpeg"); + std::fs::write(&executable, b"app").unwrap(); + std::fs::write(&outside, b"untrusted").unwrap(); + symlink(&outside, &sidecar).unwrap(); + + assert_eq!(packaged_sidecar_beside(&executable, "ffmpeg"), None); } } diff --git a/crates/opentake-media/src/index_coordinator.rs b/crates/opentake-media/src/index_coordinator.rs index bb49dee0..15ca5e60 100644 --- a/crates/opentake-media/src/index_coordinator.rs +++ b/crates/opentake-media/src/index_coordinator.rs @@ -17,7 +17,8 @@ use std::path::Path; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; use opentake_domain::media::MediaAsset; use opentake_domain::ClipType; @@ -29,8 +30,21 @@ use crate::transcribe::cache::has_cached_on_disk; /// Cross-window reference-counted export-active flag. Background work yields /// while the count is non-zero. Port of `ExportPauseCounter` /// (`SearchIndexCoordinator.swift:37-47`). +#[derive(Default)] +struct ExportPauseInner { + active: AtomicUsize, + changed: Condvar, + gate: Mutex<()>, +} + #[derive(Clone, Default)] -pub struct ExportPause(Arc); +pub struct ExportPause(Arc); + +/// Balanced playback/export pressure holder. Dropping the final guard wakes +/// every blocked background worker, including early-return and unwind paths. +pub struct ExportPauseGuard { + pause: ExportPause, +} impl ExportPause { pub fn new() -> Self { @@ -38,19 +52,56 @@ impl ExportPause { } /// Mark an export as begun (increment). pub fn begin(&self) { - self.0.fetch_add(1, Ordering::SeqCst); + self.0.active.fetch_add(1, Ordering::SeqCst); } /// Mark an export as ended (decrement; saturating at 0). pub fn end(&self) { - let _ = self + let previous = self .0 + .active .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |v| { Some(v.saturating_sub(1)) }); + if matches!(previous, Ok(1)) { + self.0.changed.notify_all(); + } } /// True while any export is active. pub fn is_active(&self) -> bool { - self.0.load(Ordering::SeqCst) > 0 + self.0.active.load(Ordering::SeqCst) > 0 + } + + /// Begin pressure and return an unwind-safe, automatically balanced guard. + pub fn guard(&self) -> ExportPauseGuard { + self.begin(); + ExportPauseGuard { + pause: self.clone(), + } + } + + /// Block until playback/export pressure clears. The cancellation predicate + /// is checked at short intervals so shutdown and job cancellation cannot be + /// stranded behind a missing `end`. Returns `false` when cancelled. + pub fn wait_while_active(&self, cancelled: impl Fn() -> bool) -> bool { + let mut gate = self.0.gate.lock().unwrap_or_else(|e| e.into_inner()); + while self.is_active() { + if cancelled() { + return false; + } + let (next, _) = self + .0 + .changed + .wait_timeout(gate, Duration::from_millis(20)) + .unwrap_or_else(|e| e.into_inner()); + gate = next; + } + !cancelled() + } +} + +impl Drop for ExportPauseGuard { + fn drop(&mut self) { + self.pause.end(); } } @@ -135,6 +186,30 @@ mod tests { // saturating: extra end stays at 0. p.end(); assert!(!p.is_active()); + + // A guard makes every early-return path balanced, and a blocked worker + // wakes promptly once the final nested holder leaves. + let outer = p.guard(); + let inner = p.guard(); + let waiter_pause = p.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let waiter = std::thread::spawn(move || { + tx.send(waiter_pause.wait_while_active(|| false)).unwrap(); + }); + assert!(rx + .recv_timeout(std::time::Duration::from_millis(25)) + .is_err()); + drop(inner); + assert!(rx + .recv_timeout(std::time::Duration::from_millis(25)) + .is_err()); + drop(outer); + assert!(rx.recv_timeout(std::time::Duration::from_secs(1)).unwrap()); + waiter.join().unwrap(); + + p.begin(); + assert!(!p.wait_while_active(|| true)); + p.end(); } #[test] diff --git a/crates/opentake-media/src/lib.rs b/crates/opentake-media/src/lib.rs index 3d42d1b5..e9ac24b6 100644 --- a/crates/opentake-media/src/lib.rs +++ b/crates/opentake-media/src/lib.rs @@ -2,7 +2,7 @@ //! //! Ports PalmierPro's AVFoundation / DSWaveformImage / macOS-Speech / CoreML //! media stack to cross-platform Rust: -//! - **probe / decode / encode**: system ffmpeg CLI via [`ff`] (no libav* link). +//! - **probe / decode / encode**: packaged/development ffmpeg CLI via [`ff`] (no libav* link). //! - **thumbnails**: seek-decode + JPEG sprite-grid disk cache. //! - **waveform**: Symphonia PCM decode → RMS downsample → normalized buckets. //! - **transcribe**: `Transcriber` trait (+ data model, locale, cache, search); @@ -20,15 +20,40 @@ //! ## Why ffmpeg over the CLI //! The local toolchain is ffmpeg 8.1 (libavcodec 62), which the C-binding crates //! (`ffmpeg-next` / `ffmpeg-the-third`) do not support, and `pkg-config` is -//! absent. `ffmpeg-sidecar` drives the binaries on `PATH` — zero native linkage -//! and a clean cross-platform build — so it is the chosen backend (SPEC §1.2, +//! absent. `ffmpeg-sidecar` drives checksum-pinned packaged binaries (or PATH in +//! development only) — zero native linkage and a clean cross-platform build — +//! so it is the chosen backend (SPEC §1.2, //! "若 ffmpeg-next 不支持 8.x … 改用 ffmpeg-sidecar"). mod ff; +#[cfg(all(feature = "ort-backend", target_os = "windows"))] +pub(crate) fn initialize_ort_backend() { + static INITIALIZE: std::sync::Once = std::sync::Once::new(); + INITIALIZE.call_once(|| { + assert!( + ort::set_api(ort_tract::api()), + "ort API was initialized before the Windows tract backend" + ); + }); +} + +#[cfg(all(feature = "ort-backend", not(target_os = "windows")))] +pub(crate) fn initialize_ort_backend() {} + +#[cfg(all(test, feature = "ort-backend", target_os = "windows"))] +mod windows_ort_backend_tests { + #[test] + fn tract_backend_initializes_before_ort_session_use() { + crate::initialize_ort_backend(); + ort::session::Session::builder().expect("tract must provide the ort session API"); + } +} + pub mod analysis; pub mod cache_key; pub mod cancel; +pub mod color; pub mod decode; pub mod encode; pub mod error; @@ -37,6 +62,7 @@ pub mod index_coordinator; pub mod library; pub mod ort_worker; pub mod probe; +pub mod proxy; pub mod search; pub mod thumbnail; pub mod timecode; @@ -123,13 +149,17 @@ pub use cancel::MediaCancelToken; pub use error::{MediaError, Result}; pub use frame::RgbaFrame; -pub use probe::{probe, MediaProbe}; +pub use color::{hdr_decode_input_args, hdr_tonemap_filter}; +pub use probe::{parse_probe, probe, MediaProbe}; +pub use proxy::{create_proxy, file_sha256, ProxyProgressCallback, ProxyRequest, ProxyResult}; pub use decode::{ - decode_frame_at, decode_frame_at_cancellable, decode_frames_at, decode_frames_at_cancellable, - decode_pcm_interleaved, decode_pcm_interleaved_cancellable, extract_pcm, - extract_pcm_cancellable, extract_pcm_cancellable_with_progress, FrameRequest, PcmBuffer, - PcmFormat, PcmProgressCallback, PcmSpec, StreamDecodeControl, StreamVideoFrame, VideoStream, + convert_frame_rate, decode_frame_at, decode_frame_at_cancellable, decode_frames_at, + decode_frames_at_cancellable, decode_pcm_interleaved, decode_pcm_interleaved_cancellable, + extract_pcm, extract_pcm_cancellable, extract_pcm_cancellable_with_progress, + interpolate_frame_pair, FrameInterpolationFallback, FrameInterpolationMode, + FrameInterpolationResult, FrameRateSample, FrameRequest, PcmBuffer, PcmFormat, + PcmProgressCallback, PcmSpec, StreamDecodeControl, StreamVideoFrame, VideoStream, VideoStreamRequest, DEFAULT_VIDEO_STREAM_QUEUE_CAPACITY, }; @@ -168,8 +198,8 @@ pub use transcribe::{ pub use transcribe::whisper::WhisperTranscriber; pub use search::{ - rank as search_visual_ranked, AssetIndex, CancelToken, Embedder, EmbedderSpec, Hit, - SamplerOptions, + rank as search_visual_ranked, AssetIndex, CancelToken, Embedder, EmbedderSpec, Header, Hit, + Row, SamplerOptions, }; pub use index_coordinator::{work_needed, ExportPause, IndexProgress, WorkNeeded}; @@ -179,7 +209,10 @@ pub use ort_worker::ExecutionProvider; /// ffmpeg/ffprobe availability probes (re-exported for integration tests and /// host-capability checks). pub mod ffmpeg_status { - pub use crate::ff::{ffmpeg_available, ffprobe_available}; + pub use crate::ff::{ + ffmpeg_available, ffmpeg_path, ffprobe_available, packaged_sidecar_beside, + packaged_sidecar_path, + }; } /// Facade bundling the media engine's roots for `opentake-core` (SPEC §8.4). @@ -221,6 +254,33 @@ impl MediaEngine { probe::probe_file(file) } + /// Decode the nearest source frame for preview/render materialization. + pub fn decode_frame(&self, path: &Path, request: &FrameRequest) -> Result<(f64, RgbaFrame)> { + decode::decode_frame_at(path, request) + } + + /// Decode the first audio track into the requested PCM contract. + pub fn extract_pcm( + &self, + path: &Path, + spec: &PcmSpec, + range: Option<(f64, f64)>, + ) -> Result { + decode::extract_pcm(path, spec, range) + } + + /// Start the streaming encoder used by the render/export adapter. + pub fn video_encoder( + &self, + output: &Path, + width: u32, + height: u32, + fps: i32, + preset: &ExportPreset, + ) -> Result { + encode::VideoEncoder::new(output, width, height, fps, preset) + } + /// Generate (and cache) a video thumbnail sequence. pub fn video_thumbnails( &self, @@ -263,6 +323,20 @@ impl MediaEngine { transcribe::search::search(&self.cache_root, query, assets, limit) } + /// Rank one encoded visual query against caller-owned current index + /// snapshots. Model loading/text encoding stay in the bounded worker; this + /// facade owns the deterministic index/ranking boundary. + pub fn search_visual( + &self, + query_vector: &[f32], + indexes: &[(String, AssetIndex)], + limit: usize, + relative_cutoff: f32, + min_score: Option, + ) -> Vec { + search::rank(query_vector, indexes, limit, relative_cutoff, min_score) + } + /// The shared export-pause signal; `opentake-render` calls `begin`/`end` /// around exports so background indexing yields. pub fn export_pause(&self) -> ExportPause { @@ -379,6 +453,13 @@ mod tests { let _: Option = None; let _ = PcmFormat::F32; let _ = VideoCodec::H264; + + // The high-level facade owns every service family as methods; callers + // do not need to assemble the flat modules themselves. + let _ = MediaEngine::decode_frame; + let _ = MediaEngine::extract_pcm; + let _ = MediaEngine::video_encoder; + let _ = MediaEngine::search_visual; } // --- extract_audio codec selection (Issue #39 review #3) --- diff --git a/crates/opentake-media/src/ort_worker/mod.rs b/crates/opentake-media/src/ort_worker/mod.rs index 47868623..c0d1a766 100644 --- a/crates/opentake-media/src/ort_worker/mod.rs +++ b/crates/opentake-media/src/ort_worker/mod.rs @@ -13,7 +13,7 @@ pub mod tensor; pub use tensor::{frame_to_hwc, hwc_to_nchw_normalized, mean_pool}; /// Execution provider preference; the loader falls back to CPU when an -/// accelerator is unavailable. +/// accelerator is unavailable. Windows ships the pure-Rust tract CPU backend. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExecutionProvider { Cpu, @@ -24,8 +24,8 @@ pub enum ExecutionProvider { } impl ExecutionProvider { - /// The platform-preferred provider (CoreML on macOS, DirectML on Windows, - /// CUDA on Linux), used as the first choice before CPU fallback. + /// The platform-preferred provider (CoreML on macOS, CPU tract on Windows, + /// CPU on Linux), used as the first choice before CPU fallback. pub fn platform_default() -> Self { #[cfg(target_os = "macos")] { @@ -33,7 +33,7 @@ impl ExecutionProvider { } #[cfg(target_os = "windows")] { - ExecutionProvider::DirectMl + ExecutionProvider::Cpu } #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] { @@ -66,6 +66,532 @@ pub struct IoSpec { pub outputs: Vec, } +use std::any::Any; +use std::collections::HashMap; +use std::marker::PhantomData; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::{Arc, Condvar, Mutex, Weak}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::index_coordinator::ExportPause; +use crate::search::CancelToken; + +/// The production operation class carried with every queued job. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JobKind { + Index, + Transcribe, + Search, +} + +/// Scheduling priority. Playback/export does not enter this queue: its shared +/// [`ExportPause`] gate prevents new jobs from starting at batch boundaries. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum JobPriority { + Background, + Interactive, +} + +/// Stable identity used for observability, prioritisation, and deduplication. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobRequest { + pub kind: JobKind, + pub model_identity: String, + pub dedupe_key: String, + pub priority: JobPriority, +} + +impl JobRequest { + pub fn new( + kind: JobKind, + model_identity: impl Into, + dedupe_key: impl Into, + priority: JobPriority, + ) -> Self { + Self { + kind, + model_identity: model_identity.into(), + dedupe_key: dedupe_key.into(), + priority, + } + } +} + +/// Consumer-visible lifecycle. Every accepted job reaches exactly one terminal +/// state even when its task returns an error or panics. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JobState { + Queued, + Running, + Cancelled, + Completed, + Failed, +} + +/// Typed queue/result failures. Model and job errors remain recoverable: the +/// single worker continues serving the next request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WorkerError { + QueueFull, + Cancelled, + Shutdown, + Panicked, + Model(String), + Job(String), + ResultType, +} + +impl std::fmt::Display for WorkerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::QueueFull => f.write_str("inference queue is full"), + Self::Cancelled => f.write_str("inference job was cancelled"), + Self::Shutdown => f.write_str("inference worker is shut down"), + Self::Panicked => f.write_str("inference job panicked"), + Self::Model(message) => write!(f, "model error: {message}"), + Self::Job(message) => write!(f, "job error: {message}"), + Self::ResultType => f.write_str("inference result type mismatch"), + } + } +} + +impl std::error::Error for WorkerError {} + +type ErasedResult = Arc; +type JobTask = Box< + dyn FnOnce(&OrtModelRegistry, &CancelToken) -> Result + + Send + + 'static, +>; + +struct JobStatus { + state: JobState, + result: Option>, +} + +struct SharedJob { + request: JobRequest, + cancel: CancelToken, + status: Mutex, + changed: Condvar, +} + +impl SharedJob { + fn new(request: JobRequest) -> Self { + Self { + request, + cancel: CancelToken::new(), + status: Mutex::new(JobStatus { + state: JobState::Queued, + result: None, + }), + changed: Condvar::new(), + } + } + + fn set_running(&self) -> bool { + let mut status = self.status.lock().unwrap_or_else(|e| e.into_inner()); + if self.cancel.is_cancelled() { + false + } else { + status.state = JobState::Running; + self.changed.notify_all(); + true + } + } + + fn finish(&self, result: Result) { + let mut status = self.status.lock().unwrap_or_else(|e| e.into_inner()); + status.state = match &result { + Ok(_) => JobState::Completed, + Err(WorkerError::Cancelled) => JobState::Cancelled, + Err(_) => JobState::Failed, + }; + status.result = Some(result); + self.changed.notify_all(); + } +} + +struct QueuedJob { + sequence: u64, + shared: Arc, + task: Option, +} + +enum WorkerMessage { + Job(QueuedJob), + Shutdown, +} + +struct WorkerInner { + sender: SyncSender, + dedupe: Mutex>>, + capacity: usize, + queued: AtomicUsize, + sequence: AtomicU64, + active: AtomicUsize, + shutdown: AtomicBool, + thread: Mutex>>, +} + +/// A bounded, single-thread heavy-inference executor. Its queue is shared by +/// indexing, transcription, and semantic-search callers; jobs carry model and +/// source identities, deduplicate while live, and yield to playback/export. +#[derive(Clone)] +pub struct OrtWorker { + inner: Arc, +} + +/// Typed view of one accepted (or deduplicated) job. +pub struct JobHandle { + shared: Arc, + marker: PhantomData, +} + +impl Clone for JobHandle { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + marker: PhantomData, + } + } +} + +impl JobHandle +where + T: Clone + Send + Sync + 'static, +{ + pub fn state(&self) -> JobState { + self.shared + .status + .lock() + .unwrap_or_else(|e| e.into_inner()) + .state + } + + pub fn cancel(&self) { + self.shared.cancel.cancel(); + } + + pub fn wait(&self) -> Result { + let mut status = self.shared.status.lock().unwrap_or_else(|e| e.into_inner()); + while status.result.is_none() { + status = self + .shared + .changed + .wait(status) + .unwrap_or_else(|e| e.into_inner()); + } + match status.result.as_ref().expect("result checked above") { + Ok(value) => value + .downcast_ref::() + .cloned() + .ok_or(WorkerError::ResultType), + Err(error) => Err(error.clone()), + } + } + + pub fn wait_until_running(&self, timeout: Duration) -> Result<(), WorkerError> { + let deadline = Instant::now() + timeout; + let mut status = self.shared.status.lock().unwrap_or_else(|e| e.into_inner()); + loop { + match status.state { + JobState::Running => return Ok(()), + JobState::Cancelled => return Err(WorkerError::Cancelled), + JobState::Failed | JobState::Completed => { + return Err(WorkerError::Job( + "job finished before it was observed running".into(), + )) + } + JobState::Queued => {} + } + let now = Instant::now(); + if now >= deadline { + return Err(WorkerError::Job( + "timed out waiting for running state".into(), + )); + } + let (next, timed) = self + .shared + .changed + .wait_timeout(status, deadline - now) + .unwrap_or_else(|e| e.into_inner()); + status = next; + if timed.timed_out() && status.state != JobState::Running { + return Err(WorkerError::Job( + "timed out waiting for running state".into(), + )); + } + } + } +} + +/// Single-worker model cache. A worker task can lazily install a typed model by +/// stable identity; subsequent jobs reuse the exact `Arc` without a second load. +#[derive(Default)] +pub struct OrtModelRegistry { + models: Mutex>, +} + +impl OrtModelRegistry { + pub fn get_or_try_init(&self, key: &str, load: F) -> Result, WorkerError> + where + T: Send + Sync + 'static, + F: FnOnce() -> Result, + { + let mut models = self.models.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(existing) = models.get(key) { + return existing + .clone() + .downcast::() + .map_err(|_| WorkerError::ResultType); + } + let model = Arc::new(load()?); + models.insert(key.to_string(), model.clone()); + Ok(model) + } +} + +impl OrtWorker { + /// Spawn one worker with a hard bounded admission queue. + pub fn spawn(export_pause: ExportPause, capacity: usize) -> Self { + let (sender, receiver) = mpsc::sync_channel(capacity.max(1)); + let inner = Arc::new(WorkerInner { + sender, + dedupe: Mutex::new(HashMap::new()), + capacity: capacity.max(1), + queued: AtomicUsize::new(0), + sequence: AtomicU64::new(0), + active: AtomicUsize::new(0), + shutdown: AtomicBool::new(false), + thread: Mutex::new(None), + }); + let thread_inner = inner.clone(); + let thread = std::thread::Builder::new() + .name("opentake-ort-worker".into()) + .spawn(move || worker_loop(thread_inner, receiver, export_pause)) + .expect("spawn bounded inference worker"); + *inner.thread.lock().unwrap_or_else(|e| e.into_inner()) = Some(thread); + Self { inner } + } + + /// Submit a typed job. A live duplicate key reuses the same result and does + /// not consume queue capacity or execute a second task. + pub fn submit(&self, request: JobRequest, task: F) -> Result, WorkerError> + where + T: Clone + Send + Sync + 'static, + F: FnOnce(&OrtModelRegistry, &CancelToken) -> Result + Send + 'static, + { + if self.inner.shutdown.load(Ordering::SeqCst) { + return Err(WorkerError::Shutdown); + } + + let mut dedupe = self.inner.dedupe.lock().unwrap_or_else(|e| e.into_inner()); + dedupe.retain(|_, weak| weak.strong_count() > 0); + if let Some(shared) = dedupe.get(&request.dedupe_key).and_then(Weak::upgrade) { + return Ok(JobHandle { + shared, + marker: PhantomData, + }); + } + + if self + .inner + .queued + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |queued| { + (queued < self.inner.capacity).then_some(queued + 1) + }) + .is_err() + { + return Err(WorkerError::QueueFull); + } + + let key = request.dedupe_key.clone(); + let shared = Arc::new(SharedJob::new(request)); + dedupe.insert(key.clone(), Arc::downgrade(&shared)); + let sequence = self.inner.sequence.fetch_add(1, Ordering::SeqCst); + let erased: JobTask = Box::new(move |models, cancel| { + task(models, cancel).map(|value| Arc::new(value) as ErasedResult) + }); + let message = WorkerMessage::Job(QueuedJob { + sequence, + shared: shared.clone(), + task: Some(erased), + }); + match self.inner.sender.try_send(message) { + Ok(()) => Ok(JobHandle { + shared, + marker: PhantomData, + }), + Err(TrySendError::Full(_)) => { + self.inner.queued.fetch_sub(1, Ordering::SeqCst); + dedupe.remove(&key); + Err(WorkerError::QueueFull) + } + Err(TrySendError::Disconnected(_)) => { + self.inner.queued.fetch_sub(1, Ordering::SeqCst); + dedupe.remove(&key); + Err(WorkerError::Shutdown) + } + } + } + + pub fn active_jobs(&self) -> usize { + self.inner.active.load(Ordering::SeqCst) + } + + pub fn queued_jobs(&self) -> usize { + self.inner.queued.load(Ordering::SeqCst) + } + + /// Cancel queued work, wait for the cooperative running job, and join the + /// sole worker thread. Idempotent. + pub fn shutdown(&self) -> Result<(), WorkerError> { + if !self.inner.shutdown.swap(true, Ordering::SeqCst) { + // A full channel is not an error: the worker observes the atomic + // shutdown flag at its next cancellation/dispatch boundary. + let _ = self.inner.sender.try_send(WorkerMessage::Shutdown); + } + if let Some(thread) = self + .inner + .thread + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take() + { + thread.join().map_err(|_| WorkerError::Panicked)?; + } + Ok(()) + } +} + +fn worker_loop(inner: Arc, receiver: Receiver, pause: ExportPause) { + let registry = OrtModelRegistry::default(); + let mut pending = Vec::::new(); + let mut high_streak = 0usize; + + loop { + if pending.is_empty() { + match receiver.recv() { + Ok(WorkerMessage::Job(job)) => pending.push(job), + Ok(WorkerMessage::Shutdown) | Err(_) => break, + } + } + + let mut shutdown = false; + loop { + match receiver.try_recv() { + Ok(WorkerMessage::Job(job)) => pending.push(job), + Ok(WorkerMessage::Shutdown) => { + shutdown = true; + break; + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + shutdown = true; + break; + } + } + } + if shutdown || inner.shutdown.load(Ordering::SeqCst) { + cancel_pending(&pending); + break; + } + + if !pause.wait_while_active(|| inner.shutdown.load(Ordering::SeqCst)) { + cancel_pending(&pending); + break; + } + + // Requests may arrive while the worker is pressure-gated. Re-drain at + // the scheduling boundary so their priority participates immediately. + loop { + match receiver.try_recv() { + Ok(WorkerMessage::Job(job)) => pending.push(job), + Ok(WorkerMessage::Shutdown) => { + cancel_pending(&pending); + inner.active.store(0, Ordering::SeqCst); + return; + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + cancel_pending(&pending); + inner.active.store(0, Ordering::SeqCst); + return; + } + } + } + + // Four high-priority jobs is the starvation bound. Otherwise choose the + // oldest job at the highest available priority (FIFO within priority). + let has_background = pending + .iter() + .any(|job| job.shared.request.priority == JobPriority::Background); + let force_background = has_background && high_streak >= 4; + let selected_priority = if force_background { + JobPriority::Background + } else { + pending + .iter() + .map(|job| job.shared.request.priority) + .max() + .unwrap_or(JobPriority::Background) + }; + let index = pending + .iter() + .enumerate() + .filter(|(_, job)| job.shared.request.priority == selected_priority) + .min_by_key(|(_, job)| job.sequence) + .map(|(index, _)| index) + .expect("pending queue is not empty"); + let mut job = pending.swap_remove(index); + inner.queued.fetch_sub(1, Ordering::SeqCst); + if selected_priority == JobPriority::Interactive { + high_streak += 1; + } else { + high_streak = 0; + } + + if !job.shared.set_running() { + remove_dedupe(&inner, &job.shared); + job.shared.finish(Err(WorkerError::Cancelled)); + continue; + } + inner.active.fetch_add(1, Ordering::SeqCst); + let task = job.task.take().expect("queued job owns one task"); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + task(®istry, &job.shared.cancel) + })) + .unwrap_or(Err(WorkerError::Panicked)); + inner.active.fetch_sub(1, Ordering::SeqCst); + remove_dedupe(&inner, &job.shared); + job.shared.finish(outcome); + } + + inner.active.store(0, Ordering::SeqCst); + inner.queued.store(0, Ordering::SeqCst); +} + +fn cancel_pending(pending: &[QueuedJob]) { + for job in pending { + job.shared.cancel.cancel(); + job.shared.finish(Err(WorkerError::Cancelled)); + } +} + +fn remove_dedupe(inner: &WorkerInner, shared: &Arc) { + let mut dedupe = inner.dedupe.lock().unwrap_or_else(|e| e.into_inner()); + if dedupe + .get(&shared.request.dedupe_key) + .and_then(Weak::upgrade) + .is_some_and(|current| Arc::ptr_eq(¤t, shared)) + { + dedupe.remove(&shared.request.dedupe_key); + } +} + #[cfg(feature = "ort-backend")] mod model { use std::collections::HashMap; @@ -79,6 +605,8 @@ mod model { use super::ExecutionProvider; use crate::error::{MediaError, Result}; + pub type OrtIoContract = (Vec<(String, String)>, Vec<(String, String)>); + /// A loaded ONNX model + a CPU-fallback-friendly session. `Session` is not /// `Sync`; wrap in a `Mutex` so the worker can share it. pub struct OrtModel { @@ -88,6 +616,7 @@ mod model { impl OrtModel { /// Load `path` with the given EP preference, falling back to CPU. pub fn load(path: &Path, _ep: ExecutionProvider) -> Result { + crate::initialize_ort_backend(); let builder = Session::builder().map_err(|e| MediaError::ModelInstall(format!("ort: {e}")))?; let builder = builder @@ -137,11 +666,29 @@ mod model { } Ok(out) } + + /// Names and debug-formatted tensor contracts declared by the model. + /// Used to fail closed when a downloaded advanced model does not match + /// the pinned architecture before any user media reaches inference. + pub fn io_contract(&self) -> OrtIoContract { + let session = self.session.lock().unwrap(); + let inputs = session + .inputs + .iter() + .map(|input| (input.name.clone(), format!("{:?}", input.input_type))) + .collect(); + let outputs = session + .outputs + .iter() + .map(|output| (output.name.clone(), format!("{:?}", output.output_type))) + .collect(); + (inputs, outputs) + } } } #[cfg(feature = "ort-backend")] -pub use model::OrtModel; +pub use model::{OrtIoContract, OrtModel}; #[cfg(test)] mod tests { @@ -172,4 +719,21 @@ mod tests { assert_eq!(t.shape[0], -1); assert_eq!(t.dtype, TensorDType::F32); } + + #[cfg(feature = "ort-backend")] + #[test] + fn installed_advanced_model_contract_can_be_inspected_before_inference() { + let Some(path) = std::env::var_os("OPENTAKE_TEST_ONNX_MODEL") else { + return; + }; + let model = OrtModel::load( + std::path::Path::new(&path), + ExecutionProvider::platform_default(), + ) + .expect("load supplied ONNX model"); + let contract = model.io_contract(); + assert!(!contract.0.is_empty()); + assert!(!contract.1.is_empty()); + eprintln!("ONNX_IO_CONTRACT={contract:?}"); + } } diff --git a/crates/opentake-media/src/probe.rs b/crates/opentake-media/src/probe.rs index 8829ee9b..4794ba14 100644 --- a/crates/opentake-media/src/probe.rs +++ b/crates/opentake-media/src/probe.rs @@ -8,6 +8,8 @@ use std::path::Path; +use opentake_domain::MediaColorMetadata; + use crate::error::{MediaError, Result}; use crate::ff; @@ -28,6 +30,9 @@ pub struct MediaProbe { /// `mov,mp4,m4a,3gp,3g2,mj2`). Security-sensitive import boundaries use /// this to verify that downloaded bytes match their declared container. pub format_name: Option, + /// Source video color signalling retained for HDR-aware decode and durable + /// project metadata. Absent only when the stream reports no color fields. + pub color: Option, } /// Open the container and read the first video stream + audio presence. @@ -125,6 +130,7 @@ pub fn parse_probe(json: &serde_json::Value) -> MediaProbe { let mut height = None; let mut fps = None; let mut video_duration = None; + let mut color = None; if let Some(v) = video { let w = v.get("width").and_then(|x| x.as_u64()).map(|x| x as u32); @@ -152,6 +158,28 @@ pub fn parse_probe(json: &serde_json::Value) -> MediaProbe { .get("duration") .and_then(|x| x.as_str()) .and_then(|s| s.parse::().ok()); + + let metadata = MediaColorMetadata { + primaries: v + .get("color_primaries") + .and_then(|value| value.as_str()) + .map(str::to_owned), + transfer: v + .get("color_transfer") + .and_then(|value| value.as_str()) + .map(str::to_owned), + matrix: v + .get("color_space") + .and_then(|value| value.as_str()) + .map(str::to_owned), + range: v + .get("color_range") + .and_then(|value| value.as_str()) + .map(str::to_owned), + }; + if !metadata.is_empty() { + color = Some(metadata); + } } let container_duration = json @@ -175,6 +203,7 @@ pub fn parse_probe(json: &serde_json::Value) -> MediaProbe { has_audio, has_video, format_name, + color, } } diff --git a/crates/opentake-media/src/proxy.rs b/crates/opentake-media/src/proxy.rs new file mode 100644 index 00000000..c4f71970 --- /dev/null +++ b/crates/opentake-media/src/proxy.rs @@ -0,0 +1,222 @@ +//! Project-local low-resolution proxy creation. +//! +//! The source is read-only and hashed before and after transcoding. FFmpeg +//! writes to a sibling partial file which is renamed only after a successful +//! probe, so cancellation and failures never expose a truncated proxy. + +use std::fs::{self, File}; +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::time::Duration; + +use sha2::{Digest, Sha256}; + +use crate::cancel::MediaCancelToken; +use crate::error::{MediaError, Result}; +use crate::{ff, probe}; + +pub type ProxyProgressCallback = Arc; + +#[derive(Clone, Copy, Debug)] +pub struct ProxyRequest<'a> { + pub source: &'a Path, + pub output: &'a Path, + pub max_size: (u32, u32), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProxyResult { + pub path: PathBuf, + pub source_sha256: String, + pub width: u32, + pub height: u32, +} + +fn report(progress: &Option, done: usize) { + if let Some(callback) = progress { + callback(done, 1000); + } +} + +pub fn file_sha256(path: &Path) -> Result { + let mut reader = BufReader::new(File::open(path)?); + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn partial_path(output: &Path) -> PathBuf { + match output.extension().and_then(|extension| extension.to_str()) { + Some(extension) => output.with_extension(format!("{extension}.partial")), + None => output.with_extension("partial"), + } +} + +fn cleanup_partial(path: &Path) { + if path.is_file() { + let _ = fs::remove_file(path); + } +} + +pub fn create_proxy( + request: ProxyRequest<'_>, + cancel: &MediaCancelToken, + progress: Option, +) -> Result { + report(&progress, 0); + if cancel.is_cancelled() { + return Err(MediaError::Cancelled); + } + if request.max_size.0 == 0 || request.max_size.1 == 0 { + return Err(MediaError::Ffmpeg( + "proxy dimensions must be positive".to_string(), + )); + } + if !request.source.is_file() { + return Err(MediaError::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + request.source.display().to_string(), + ))); + } + if request.output.exists() { + return Err(MediaError::Ffmpeg( + "proxy destination already exists".to_string(), + )); + } + + let partial = partial_path(request.output); + if partial.exists() { + return Err(MediaError::Ffmpeg( + "proxy partial destination already exists".to_string(), + )); + } + if let Some(parent) = request.output.parent() { + fs::create_dir_all(parent)?; + } + + let source_sha256 = file_sha256(request.source)?; + report(&progress, 100); + if cancel.is_cancelled() { + return Err(MediaError::Cancelled); + } + + let scale = format!( + "scale=w={}:h={}:force_original_aspect_ratio=decrease:force_divisible_by=2", + request.max_size.0, request.max_size.1 + ); + let mut child = Command::new(ff::ffmpeg_path()) + .args(["-hide_banner", "-loglevel", "error", "-nostdin", "-y"]) + .arg("-i") + .arg(request.source) + .args(["-map", "0:v:0", "-map", "0:a?", "-vf"]) + .arg(scale) + .args([ + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-movflags", + "+faststart", + "-f", + "mp4", + ]) + .arg(&partial) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| MediaError::Ffmpeg(format!("proxy spawn: {error}")))?; + cancel.child_spawned(); + + loop { + if cancel.is_cancelled() { + let _ = child.kill(); + let _ = child.wait(); + cleanup_partial(&partial); + return Err(MediaError::Cancelled); + } + match child.try_wait() { + Ok(Some(status)) => { + if !status.success() || !partial.is_file() { + cleanup_partial(&partial); + return Err(MediaError::Ffmpeg( + "proxy transcode did not complete".to_string(), + )); + } + break; + } + Ok(None) => { + report(&progress, 500); + std::thread::sleep(Duration::from_millis(20)); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + cleanup_partial(&partial); + return Err(error.into()); + } + } + } + + report(&progress, 900); + if cancel.is_cancelled() { + cleanup_partial(&partial); + return Err(MediaError::Cancelled); + } + if file_sha256(request.source)? != source_sha256 { + cleanup_partial(&partial); + return Err(MediaError::Checksum( + "source changed while proxy was being created".to_string(), + )); + } + + let metadata = match probe(&partial) { + Ok(metadata) if metadata.has_video => metadata, + Ok(_) => { + cleanup_partial(&partial); + return Err(MediaError::no_track("video", &partial)); + } + Err(error) => { + cleanup_partial(&partial); + return Err(error); + } + }; + let (width, height) = match (metadata.width, metadata.height) { + (Some(width), Some(height)) if width > 0 && height > 0 => (width, height), + _ => { + cleanup_partial(&partial); + return Err(MediaError::Decode( + "proxy has no usable dimensions".to_string(), + )); + } + }; + if request.output.exists() { + cleanup_partial(&partial); + return Err(MediaError::Ffmpeg( + "proxy destination appeared during transcode".to_string(), + )); + } + fs::rename(&partial, request.output)?; + report(&progress, 1000); + + Ok(ProxyResult { + path: request.output.to_path_buf(), + source_sha256, + width, + height, + }) +} diff --git a/crates/opentake-media/src/search/ort_embedder.rs b/crates/opentake-media/src/search/ort_embedder.rs index d87acc00..6ae1859e 100644 --- a/crates/opentake-media/src/search/ort_embedder.rs +++ b/crates/opentake-media/src/search/ort_embedder.rs @@ -109,6 +109,7 @@ impl OrtEmbedder { } fn build_session(path: &Path) -> Result { + crate::initialize_ort_backend(); let builder = Session::builder().map_err(|e| MediaError::ModelInstall(format!("ort: {e}")))?; // Default EP set; ort falls back to CPU when an accelerator is unavailable. let builder = builder diff --git a/crates/opentake-media/src/thumbnail/project.rs b/crates/opentake-media/src/thumbnail/project.rs index d3ac14cf..aaa3fd14 100644 --- a/crates/opentake-media/src/thumbnail/project.rs +++ b/crates/opentake-media/src/thumbnail/project.rs @@ -223,6 +223,8 @@ mod tests { source_height: Some(4), source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-media/src/transcribe/captions.rs b/crates/opentake-media/src/transcribe/captions.rs index ebeb88ee..bae57e64 100644 --- a/crates/opentake-media/src/transcribe/captions.rs +++ b/crates/opentake-media/src/transcribe/captions.rs @@ -41,7 +41,7 @@ use opentake_domain::Clip; -use super::{TranscriptionResult, TranscriptionSegment}; +use super::{TranscriptionResult, TranscriptionSegment, TranscriptionWord}; /// Per-phrase floor display duration, in **seconds**. 1:1 with upstream /// `AppTheme.Caption.minDisplayDuration = 0.7` (`AppTheme.swift:249`), the @@ -395,8 +395,7 @@ pub fn caption_specs bool>( if clips.is_empty() { continue; } - let seg_phrases: Vec = result - .segments + let seg_phrases: Vec = visible_caption_segments(result, &clips, fps) .iter() .flat_map(|seg| phrases(seg, fits, MIN_DISPLAY_DURATION_SECS)) .collect(); @@ -434,6 +433,151 @@ pub fn caption_specs bool>( out } +/// Rebuild source segments from the words that are still visible through the +/// current clip fragments. A cached source segment keeps its original text even +/// after a ripple cut; using it verbatim would resurrect deleted fillers in +/// newly generated captions. Uncut segments retain their original punctuation, +/// while cut segments are sliced from that original text using the surviving +/// Whisper token sequence (including subword tokens such as `synchron` + +/// `ized`). +fn visible_caption_segments( + result: &TranscriptionResult, + clips: &[&CaptionTarget<'_>], + fps: i32, +) -> Vec { + let fps_d = fps as f64; + let mut rebuilt = Vec::new(); + for segment in &result.segments { + let segment_words = result + .words + .iter() + .filter(|word| match (word.start, word.end) { + (Some(start), Some(end)) => { + let midpoint = (start + end) / 2.0; + midpoint >= segment.start && midpoint < segment.end + } + _ => false, + }) + .collect::>(); + if segment_words.is_empty() { + rebuilt.push(segment.clone()); + continue; + } + + let owners = segment_words + .iter() + .map(|word| { + let (start, end) = (word.start?, word.end?); + let midpoint_frame = (start + end) / 2.0 * fps_d; + clips.iter().position(|target| { + let (visible_start, visible_end) = visible_source_span(target.clip); + visible_start <= midpoint_frame && midpoint_frame < visible_end + }) + }) + .collect::>(); + + if owners.iter().all(Option::is_some) && owners.windows(2).all(|pair| pair[0] == pair[1]) { + rebuilt.push(segment.clone()); + continue; + } + + let token_ranges = token_byte_ranges(&segment.text, &segment_words); + let mut index = 0; + while index < segment_words.len() { + let Some(owner) = owners[index] else { + index += 1; + continue; + }; + let run_start = index; + index += 1; + while index < segment_words.len() && owners[index] == Some(owner) { + index += 1; + } + let run_end = index; + let text = token_ranges + .as_ref() + .and_then(|ranges| slice_token_run(&segment.text, ranges, run_start, run_end)) + .unwrap_or_else(|| detokenize(&segment_words[run_start..run_end])); + let start = segment_words[run_start].start.unwrap_or(segment.start); + let end = segment_words[run_end - 1].end.unwrap_or(segment.end); + if !text.trim().is_empty() && end > start { + rebuilt.push(TranscriptionSegment { text, start, end }); + } + } + } + rebuilt +} + +fn token_byte_ranges(text: &str, words: &[&TranscriptionWord]) -> Option> { + let mut normalized = Vec::new(); + for (byte_start, ch) in text.char_indices() { + if !ch.is_alphanumeric() { + continue; + } + let byte_end = byte_start + ch.len_utf8(); + for folded in ch.to_lowercase() { + normalized.push((folded, byte_start, byte_end)); + } + } + + let mut cursor = 0; + let mut ranges = Vec::with_capacity(words.len()); + for word in words { + let needle = word + .text + .chars() + .filter(|ch| ch.is_alphanumeric()) + .flat_map(char::to_lowercase) + .collect::>(); + if needle.is_empty() { + return None; + } + if cursor > normalized.len() || needle.len() > normalized.len() - cursor { + return None; + } + let position = (cursor..=normalized.len().saturating_sub(needle.len())).find(|start| { + normalized[*start..*start + needle.len()] + .iter() + .map(|entry| entry.0) + .eq(needle.iter().copied()) + })?; + let end_position = position + needle.len() - 1; + ranges.push((normalized[position].1, normalized[end_position].2)); + cursor = position + needle.len(); + } + Some(ranges) +} + +fn slice_token_run( + text: &str, + ranges: &[(usize, usize)], + start: usize, + end: usize, +) -> Option { + let byte_start = ranges.get(start)?.0; + let mut byte_end = ranges.get(end.checked_sub(1)?)?.1; + let next_start = ranges.get(end).map(|range| range.0).unwrap_or(text.len()); + let punctuation_start = byte_end; + // Preserve punctuation attached to the final surviving token, but not the + // whitespace leading into a removed token. + for (offset, ch) in text[punctuation_start..next_start].char_indices() { + if ch.is_whitespace() || ch.is_alphanumeric() { + break; + } + byte_end = punctuation_start + offset + ch.len_utf8(); + } + Some(text.get(byte_start..byte_end)?.trim().to_string()) +} + +fn detokenize(words: &[&TranscriptionWord]) -> String { + words + .iter() + .map(|word| word.text.trim()) + .filter(|text| !text.is_empty()) + .collect::>() + .join(" ") +} + /// The clip whose visible source window overlaps phrase `p` the most, but only /// when the overlap is real (`> 0`) and covers at least half the phrase. 1:1 port /// of `bestClip(for:among:)` (`EditorViewModel+Captions.swift:186-195`). @@ -786,6 +930,45 @@ mod tests { assert_eq!(out[0].start_frame, 0); } + #[test] + fn caption_specs_rebuild_cut_segment_from_visible_words() { + // A three-frame ripple cut removes only "Um" between two fragments of + // the same source. A cached segment still contains the old full text; + // regenerated captions must use the surviving words and preserve a + // split Whisper token as the original "synchronized" spelling. + let before = clip("before", 0, 151, 0, 1.0); + let after = clip("after", 151, 746, 154, 1.0); + let transcript = result( + vec![ + word("Um", 5.0, 5.1), + word("today", 5.2, 5.4), + word("synchron", 5.4, 5.7), + word("ized", 5.7, 5.9), + ], + vec![seg("Um, today synchronized", 5.0, 5.9)], + ); + let targets = vec![ + CaptionTarget { + clip_id: "before".into(), + track_id: "voice".into(), + clip: &before, + transcript: Some(&transcript), + }, + CaptionTarget { + clip_id: "after".into(), + track_id: "voice".into(), + clip: &after, + transcript: Some(&transcript), + }, + ]; + + let out = caption_specs(&targets, 30, CaptionCase::Auto, "g", &fits_words(5)); + + assert_eq!(out.len(), 1); + assert_eq!(out[0].content, "today synchronized"); + assert!(!out[0].content.contains("Um")); + } + #[test] fn caption_specs_empty_transcript_yields_nothing() { let c = clip("c1", 0, 300, 0, 1.0); diff --git a/crates/opentake-media/src/transcribe/whisper.rs b/crates/opentake-media/src/transcribe/whisper.rs index f0cb067c..2100aa94 100644 --- a/crates/opentake-media/src/transcribe/whisper.rs +++ b/crates/opentake-media/src/transcribe/whisper.rs @@ -50,6 +50,203 @@ fn cs_to_secs(cs: i64) -> f64 { cs as f64 / 100.0 } +// whisper.cpp's experimental token timestamps can spread the first words of +// an utterance backwards across a long leading pause (the segment timestamp +// token commonly starts at the pause itself). That is dangerous for edit +// tools: deleting a filler word would then delete silence, or an earlier word, +// instead of the spoken filler. Tighten only *silent segment edges* using the +// same PCM that Whisper consumed, then preserve token order by mapping the +// original token positions into the audible interval. Internal pauses are +// intentionally untouched. +fn align_segment_to_speech( + pcm: &PcmBuffer, + segment: &mut TranscriptionSegment, + words: &mut [TranscriptionWord], +) { + const WINDOW_SECS: f64 = 0.020; + const HOP_SECS: f64 = 0.010; + const ABSOLUTE_RMS_FLOOR: f64 = 0.001; // -60 dBFS + const RELATIVE_TO_PEAK: f64 = 0.02; // -34 dB from this segment's peak + const EDGE_PADDING_SECS: f64 = 0.040; + const MIN_EDGE_TRIM_SECS: f64 = 0.080; + + let sample_rate = pcm.spec.sample_rate as usize; + let old_start = segment.start.max(0.0); + let old_end = segment.end.min(pcm.duration_secs()); + let old_duration = old_end - old_start; + if sample_rate == 0 || old_duration <= 0.0 || words.is_empty() { + return; + } + + let range_start = (old_start * sample_rate as f64).floor() as usize; + let range_end = ((old_end * sample_rate as f64).ceil() as usize).min(pcm.samples_f32.len()); + if range_end <= range_start { + return; + } + + let window = ((WINDOW_SECS * sample_rate as f64).round() as usize).max(1); + let hop = ((HOP_SECS * sample_rate as f64).round() as usize).max(1); + let mut energy_windows = Vec::new(); + let mut cursor = range_start; + while cursor < range_end { + let end = (cursor + window).min(range_end); + let samples = &pcm.samples_f32[cursor..end]; + let square_sum = samples + .iter() + .map(|sample| f64::from(*sample) * f64::from(*sample)) + .sum::(); + let rms = (square_sum / samples.len() as f64).sqrt(); + energy_windows.push((cursor, end, rms)); + cursor = cursor.saturating_add(hop); + } + + let peak_rms = energy_windows + .iter() + .map(|(_, _, rms)| *rms) + .fold(0.0_f64, f64::max); + if peak_rms <= ABSOLUTE_RMS_FLOOR { + return; + } + let threshold = ABSOLUTE_RMS_FLOOR.max(peak_rms * RELATIVE_TO_PEAK); + let Some(first_audible) = energy_windows + .iter() + .position(|(_, _, rms)| *rms > threshold) + else { + return; + }; + let last_audible = energy_windows + .iter() + .rposition(|(_, _, rms)| *rms > threshold) + .unwrap_or(first_audible); + + let detected_start = energy_windows[first_audible].0 as f64 / sample_rate as f64; + let detected_end = energy_windows[last_audible].1 as f64 / sample_rate as f64; + let mut new_start = (detected_start - EDGE_PADDING_SECS).max(old_start); + let mut new_end = (detected_end + EDGE_PADDING_SECS).min(old_end); + if new_start - old_start < MIN_EDGE_TRIM_SECS { + new_start = old_start; + } + if old_end - new_end < MIN_EDGE_TRIM_SECS { + new_end = old_end; + } + if new_end <= new_start || (new_start == old_start && new_end == old_end) { + return; + } + + let new_duration = new_end - new_start; + let remap = |time: f64| { + let progress = ((time - old_start) / old_duration).clamp(0.0, 1.0); + new_start + progress * new_duration + }; + for word in words { + if let Some(start) = word.start { + word.start = Some(remap(start)); + } + if let Some(end) = word.end { + word.end = Some(remap(end)); + } + if let (Some(start), Some(end)) = (word.start, word.end) { + if end < start { + word.end = Some(start); + } + } + } + segment.start = new_start; + segment.end = new_end; +} + +/// Keep edit-facing word rows lexical and give Whisper's zero-duration lexical +/// tokens a real, non-overlapping interval. Whisper commonly emits `You` +/// `[t,t]` followed by `know` `[t,t+n]`; dropping the first span makes the +/// multi-word filler impossible to review or remove. Punctuation-only tokens +/// are not independently editable words, so their time is available to the +/// neighboring lexical token. +fn normalize_word_timings(segment: &TranscriptionSegment, words: &mut Vec) { + const EPSILON: f64 = 1e-9; + + words.retain(|word| word.text.chars().any(char::is_alphanumeric)); + let mut index = 0; + while index < words.len() { + let (Some(start), Some(end)) = (words[index].start, words[index].end) else { + index += 1; + continue; + }; + if end > start + EPSILON { + index += 1; + continue; + } + + // A same-start run with a later positive interval (for example + // `You [t,t]`, `know [t,t+n]`) represents a single Whisper interval + // shared by multiple lexical tokens. Split it by character weight. + let mut run_end = index + 1; + let mut shared_end = start; + while run_end < words.len() { + let (Some(next_start), Some(next_end)) = (words[run_end].start, words[run_end].end) + else { + break; + }; + if (next_start - start).abs() > EPSILON { + break; + } + shared_end = shared_end.max(next_end); + run_end += 1; + } + if shared_end > start + EPSILON { + let total_weight = words[index..run_end] + .iter() + .map(|word| { + word.text + .chars() + .filter(|c| c.is_alphanumeric()) + .count() + .max(1) + }) + .sum::() as f64; + let mut cursor = start; + for word in &mut words[index..run_end] { + let weight = word + .text + .chars() + .filter(|c| c.is_alphanumeric()) + .count() + .max(1) as f64; + let next = if (cursor - shared_end).abs() <= EPSILON { + shared_end + } else { + (cursor + (shared_end - start) * weight / total_weight).min(shared_end) + }; + word.start = Some(cursor); + word.end = Some(next); + cursor = next; + } + if let Some(last) = words.get_mut(run_end - 1) { + last.end = Some(shared_end); + } + index = run_end; + continue; + } + + // A lone zero-duration token owns the gap to the next lexical token. + let next_start = words[index + 1..] + .iter() + .filter_map(|word| word.start) + .find(|next| *next > start + EPSILON) + .unwrap_or(segment.end); + if next_start > start + EPSILON { + words[index].end = Some(next_start.min(segment.end)); + } else if index > 0 { + let previous_end = words[index - 1].end.unwrap_or(segment.start); + let fallback_start = (segment.end - 0.08).max(previous_end); + if segment.end > fallback_start + EPSILON { + words[index].start = Some(fallback_start); + words[index].end = Some(segment.end); + } + } + index += 1; + } +} + impl Transcriber for WhisperTranscriber { fn transcribe_pcm( &self, @@ -100,16 +297,13 @@ impl Transcriber for WhisperTranscriber { // they only ever show up reconstructed at the segment level. // Excluded from `full_text` too, so the plain-text summary stays // consistent with `segments`. - if !trimmed.is_empty() && !super::is_non_speech_marker(trimmed) { + let keep_segment = !trimmed.is_empty() && !super::is_non_speech_marker(trimmed); + if keep_segment { full_text.push_str(&seg_text); - segments.push(TranscriptionSegment { - text: trimmed.to_string(), - start: cs_to_secs(t0), - end: cs_to_secs(t1), - }); } let n_tokens = state.full_n_tokens(i).unwrap_or(0); + let mut segment_words = Vec::new(); for j in 0..n_tokens { let tok_text = match state.full_get_token_text(i, j) { Ok(t) => t, @@ -131,12 +325,23 @@ impl Transcriber for WhisperTranscriber { Some(d) => (Some(cs_to_secs(d.t0)), Some(cs_to_secs(d.t1))), None => (None, None), }; - words.push(TranscriptionWord { + segment_words.push(TranscriptionWord { text: trimmed_tok.to_string(), start, end, }); } + if keep_segment { + let mut segment = TranscriptionSegment { + text: trimmed.to_string(), + start: cs_to_secs(t0), + end: cs_to_secs(t1), + }; + align_segment_to_speech(pcm, &mut segment, &mut segment_words); + normalize_word_timings(&segment, &mut segment_words); + segments.push(segment); + words.extend(segment_words); + } } let language = opts @@ -156,10 +361,116 @@ impl Transcriber for WhisperTranscriber { #[cfg(test)] mod tests { use super::*; + use crate::decode::pcm::{PcmFormat, PcmSpec}; #[test] fn centiseconds_convert_to_seconds() { assert!((cs_to_secs(150) - 1.5).abs() < 1e-9); assert_eq!(cs_to_secs(0), 0.0); } + + fn pcm(samples: Vec, sample_rate: u32) -> PcmBuffer { + PcmBuffer { + spec: PcmSpec { + sample_rate, + channels: 1, + format: PcmFormat::F32, + }, + samples_f32: samples, + } + } + + fn word(text: &str, start: f64, end: f64) -> TranscriptionWord { + TranscriptionWord { + text: text.into(), + start: Some(start), + end: Some(end), + } + } + + #[test] + fn silent_segment_edges_are_trimmed_and_words_remapped() { + let sample_rate = 1_000; + let mut samples = vec![0.0; 2 * sample_rate as usize]; + samples.extend(vec![0.5; 6 * sample_rate as usize]); + samples.extend(vec![0.0; 2 * sample_rate as usize]); + let pcm = pcm(samples, sample_rate); + let mut segment = TranscriptionSegment { + text: "one two three".into(), + start: 0.0, + end: 10.0, + }; + let mut words = vec![ + word("one", 1.0, 2.0), + word("two", 4.0, 5.0), + word("three", 8.0, 9.0), + ]; + + align_segment_to_speech(&pcm, &mut segment, &mut words); + + assert!((segment.start - 1.96).abs() < 0.02, "{:?}", segment); + assert!((segment.end - 8.05).abs() < 0.02, "{:?}", segment); + assert!(words[0].start.unwrap() >= segment.start); + assert!(words[2].end.unwrap() <= segment.end); + assert!(words.windows(2).all(|pair| { + pair[0].start.unwrap() <= pair[1].start.unwrap() + && pair[0].end.unwrap() <= pair[1].end.unwrap() + })); + } + + #[test] + fn fully_audible_segment_keeps_original_timestamps() { + let pcm = pcm(vec![0.5; 2_000], 1_000); + let mut segment = TranscriptionSegment { + text: "hello".into(), + start: 0.0, + end: 2.0, + }; + let mut words = vec![word("hello", 0.2, 1.8)]; + + align_segment_to_speech(&pcm, &mut segment, &mut words); + + assert_eq!(segment.start, 0.0); + assert_eq!(segment.end, 2.0); + assert_eq!(words[0].start, Some(0.2)); + assert_eq!(words[0].end, Some(1.8)); + } + + #[test] + fn zero_duration_words_receive_reviewable_non_overlapping_spans() { + let segment = TranscriptionSegment { + text: "Um, today. You know.".into(), + start: 4.8, + end: 12.5, + }; + let mut words = vec![ + word("Um", 5.0, 5.0), + word(",", 5.0, 5.2), + word("today", 5.2, 5.7), + word("You", 11.8, 11.8), + word("know", 11.8, 12.3), + word(".", 12.3, 12.4), + ]; + + normalize_word_timings(&segment, &mut words); + + assert_eq!( + words + .iter() + .map(|word| word.text.as_str()) + .collect::>(), + vec!["Um", "today", "You", "know"] + ); + assert_eq!(words[0].start, Some(5.0)); + assert_eq!(words[0].end, Some(5.2)); + assert_eq!(words[2].start, Some(11.8)); + assert!(words[2].end.unwrap() > 11.8); + assert_eq!(words[2].end, words[3].start); + assert_eq!(words[3].end, Some(12.3)); + assert!(words.windows(2).all(|pair| { + pair[0].end.unwrap() <= pair[1].start.unwrap() + && pair[0].end.unwrap() > pair[0].start.unwrap() + })); + assert!(words.last().unwrap().end.unwrap() > words.last().unwrap().start.unwrap()); + } } diff --git a/crates/opentake-media/tests/denoise.rs b/crates/opentake-media/tests/denoise.rs new file mode 100644 index 00000000..42426ebe --- /dev/null +++ b/crates/opentake-media/tests/denoise.rs @@ -0,0 +1,117 @@ +use opentake_domain::{AudioDenoise, DenoiseMode}; +use opentake_media::analysis::{denoise_interleaved, DenoiseError}; +use opentake_media::MediaCancelToken; + +const SAMPLE_RATE: u32 = 48_000; + +fn speech_fixture(seconds: usize) -> Vec { + (0..SAMPLE_RATE as usize * seconds) + .map(|index| { + let time = index as f32 / SAMPLE_RATE as f32; + let phrase = if (time * 2.5).fract() < 0.68 { + 1.0 + } else { + 0.0 + }; + let attack = ((time * 2.5).fract() * 12.0).min(1.0); + phrase + * attack + * ((std::f32::consts::TAU * 173.0 * time).sin() * 0.24 + + (std::f32::consts::TAU * 346.0 * time).sin() * 0.08 + + (std::f32::consts::TAU * 691.0 * time).sin() * 0.035) + }) + .collect() +} + +fn noisy_fixture(clean: &[f32]) -> Vec { + let mut state = 0x5eed_1234_u32; + clean + .iter() + .map(|sample| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let white = (state as f64 / u32::MAX as f64 * 2.0 - 1.0) as f32; + sample + white * 0.075 + }) + .collect() +} + +fn snr_db(clean: &[f32], candidate: &[f32]) -> f64 { + let signal = clean + .iter() + .map(|value| f64::from(*value).powi(2)) + .sum::(); + let error = clean + .iter() + .zip(candidate) + .map(|(expected, actual)| f64::from(*actual - *expected).powi(2)) + .sum::(); + 10.0 * (signal / error.max(1.0e-20)).log10() +} + +#[test] +fn deterministic_noise_fixture_and_bypass() { + let clean = speech_fixture(5); + let noisy = noisy_fixture(&clean); + let source_before = noisy.clone(); + let config = AudioDenoise { + mode: DenoiseMode::Adaptive, + strength: 0.9, + preview_enabled: true, + }; + let processed = denoise_interleaved( + &noisy, + 1, + SAMPLE_RATE, + config, + &MediaCancelToken::new(), + None, + ) + .expect("denoise deterministic fixture"); + + let input_snr = snr_db(&clean, &noisy); + let output_snr = snr_db(&clean, &processed); + assert!( + output_snr >= input_snr + 3.0, + "input SNR={input_snr:.2} dB output SNR={output_snr:.2} dB" + ); + assert!(processed.iter().all(|sample| sample.abs() <= 1.0)); + let input_peak = noisy + .iter() + .map(|sample| sample.abs()) + .fold(0.0_f32, f32::max); + let output_peak = processed + .iter() + .map(|sample| sample.abs()) + .fold(0.0_f32, f32::max); + assert!( + output_peak <= input_peak + 1.0e-6, + "denoise must not introduce a new peak: input={input_peak:.6} output={output_peak:.6}" + ); + assert_eq!( + noisy, source_before, + "processing must not mutate source PCM" + ); + + let bypass = denoise_interleaved( + &noisy, + 1, + SAMPLE_RATE, + AudioDenoise { + strength: 0.0, + ..config + }, + &MediaCancelToken::new(), + None, + ) + .expect("bypass"); + assert_eq!(bypass, noisy, "zero strength is a bit-exact bypass"); + + let cancelled = MediaCancelToken::new(); + cancelled.cancel(); + assert!(matches!( + denoise_interleaved(&noisy, 1, SAMPLE_RATE, config, &cancelled, None), + Err(DenoiseError::Cancelled) + )); +} diff --git a/crates/opentake-media/tests/facade_contract.rs b/crates/opentake-media/tests/facade_contract.rs new file mode 100644 index 00000000..7fd1876f --- /dev/null +++ b/crates/opentake-media/tests/facade_contract.rs @@ -0,0 +1,161 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use opentake_media::{ + AssetIndex, ExportPreset, ExportResolution, FrameRequest, Header, MediaEngine, PcmBuffer, + PcmFormat, PcmSpec, Row, TranscribeOptions, Transcriber, TranscriptionResult, VideoCodec, +}; + +struct FixtureTranscriber; + +impl Transcriber for FixtureTranscriber { + fn transcribe_pcm( + &self, + pcm: &PcmBuffer, + _opts: &TranscribeOptions, + ) -> opentake_media::Result { + Ok(TranscriptionResult { + text: format!("{} samples", pcm.samples_f32.len()), + language: Some("en".into()), + words: vec![], + segments: vec![], + }) + } +} + +fn manifest(crate_name: &str) -> String { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("media crate belongs to the workspace"); + std::fs::read_to_string(workspace.join("crates").join(crate_name).join("Cargo.toml")) + .expect("read workspace crate manifest") +} + +fn make_av_fixture(path: &Path) -> bool { + Command::new("ffmpeg") + .args([ + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=0x336699:s=32x18:r=4", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=16000", + "-t", + "1", + "-c:v", + "mpeg4", + "-c:a", + "aac", + "-y", + ]) + .arg(path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) +} + +#[test] +fn all_services_are_reachable_only_through_facade_and_dependencies_stay_acyclic() { + // The production dependency direction is domain <- media. Neither the + // zero-IO domain leaf nor media imports core/render back upward; render may + // consume media at its adapter/test boundary without creating a cycle. + let media_manifest = manifest("opentake-media"); + let domain_manifest = manifest("opentake-domain"); + let render_manifest = manifest("opentake-render"); + let core_manifest = manifest("opentake-core"); + assert!(media_manifest.contains("opentake-domain = { workspace = true }")); + assert!(!media_manifest.contains("opentake-core")); + assert!(!media_manifest.contains("opentake-render")); + assert!(!domain_manifest.contains("opentake-media")); + assert!(render_manifest.contains("opentake-media = { workspace = true }")); + assert!(!core_manifest.contains("opentake-render")); + + let temp = tempfile::tempdir().unwrap(); + let engine = MediaEngine::new(temp.path().join("cache"), temp.path().join("models")); + + // Search is a real facade operation over the persisted index value model. + let indexes = vec![( + "asset-a".to_string(), + AssetIndex { + header: Header { + model: "fixture".into(), + model_version: 1, + sampler_version: 1, + dim: 2, + count: 1, + }, + rows: vec![Row { + time: 0.25, + shot_start: 0.0, + shot_end: 1.0, + }], + vectors: vec![1.0, 0.0], + }, + )]; + let hits = engine.search_visual(&[1.0, 0.0], &indexes, 20, 0.85, Some(0.05)); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].asset_id, "asset-a"); + + // The exact IO methods must compile on every platform. Environments without + // ffmpeg stop after the pure dependency/search assertions. + let source = temp.path().join("facade-source.mp4"); + if !make_av_fixture(&source) { + return; + } + + let probe = engine.probe(&source).unwrap(); + assert_eq!((probe.width, probe.height), (Some(32), Some(18))); + assert!(probe.has_audio && probe.has_video); + + let frame = engine + .decode_frame( + &source, + &FrameRequest { + time_secs: 0.25, + max_size: (32, 18), + tolerance_secs: 0.25, + apply_rotation: true, + }, + ) + .unwrap() + .1; + assert_eq!((frame.width, frame.height), (32, 18)); + + let pcm_spec = PcmSpec { + sample_rate: 16_000, + channels: 1, + format: PcmFormat::F32, + }; + let pcm = engine.extract_pcm(&source, &pcm_spec, None).unwrap(); + assert_eq!(pcm.spec, pcm_spec); + assert!((15_000..=16_500).contains(&pcm.samples_f32.len())); + + let transcript_cache = opentake_media::TranscriptCache::new(temp.path().join("cache")); + let transcript = engine + .transcribe(&source, true, None, &FixtureTranscriber, &transcript_cache) + .unwrap(); + assert!(transcript.text.ends_with(" samples")); + + let encoded = temp.path().join("facade-encoded.mp4"); + let mut encoder = engine + .video_encoder( + &encoded, + 32, + 18, + 4, + &ExportPreset::new(VideoCodec::H264, ExportResolution::P720), + ) + .unwrap(); + encoder.push_frame(&frame).unwrap(); + encoder.finish().unwrap(); + assert!(encoded.is_file()); + assert!(engine.probe(&encoded).unwrap().has_video); + + let _: PathBuf = engine.cache_root().to_path_buf(); +} diff --git a/crates/opentake-media/tests/ffmpeg_integration.rs b/crates/opentake-media/tests/ffmpeg_integration.rs index 6fbb169a..72ea035b 100644 --- a/crates/opentake-media/tests/ffmpeg_integration.rs +++ b/crates/opentake-media/tests/ffmpeg_integration.rs @@ -14,7 +14,7 @@ use opentake_media::decode::spawn_video_stream; use opentake_media::ffmpeg_status::{ffmpeg_available, ffprobe_available}; use opentake_media::{ decode_frame_at, encode, extract_pcm, probe, video_thumbnails, waveform, ExportPreset, - ExportResolution, FrameRequest, PcmFormat, PcmSpec, VideoCodec, VideoEncoder, + ExportResolution, FrameRequest, PcmFormat, PcmSpec, RgbaFrame, VideoCodec, VideoEncoder, VideoStreamRequest, }; @@ -426,6 +426,46 @@ fn encode_prores_roundtrip_produces_prores_mov() { encode_codec_roundtrip(VideoCodec::ProRes422, "mov", "prores"); } +#[test] +fn prores_4444_roundtrip_preserves_alpha_plane() { + if !ffmpeg_available() || !ffprobe_available() { + eprintln!("SKIP: ffmpeg/ffprobe unavailable"); + return; + } + let root = tempfile::tempdir().unwrap(); + let output = root.path().join("alpha.mov"); + let preset = ExportPreset::new(VideoCodec::ProRes4444, ExportResolution::P720); + let mut encoder = VideoEncoder::new(&output, 2, 2, 1, &preset).unwrap(); + encoder + .push_frame(&RgbaFrame { + width: 2, + height: 2, + rgba: vec![ + 255, 0, 0, 0, 0, 255, 0, 85, 0, 0, 255, 170, 255, 255, 255, 255, + ], + }) + .unwrap(); + encoder.finish().unwrap(); + let decoded = decode_frame_at( + &output, + &FrameRequest { + max_size: (2, 2), + tolerance_secs: 0.0, + ..FrameRequest::default() + }, + ) + .unwrap() + .1; + let alpha = decoded + .rgba + .chunks_exact(4) + .map(|pixel| pixel[3]) + .collect::>(); + for (actual, expected) in alpha.iter().zip([0_u8, 85, 170, 255]) { + assert!(actual.abs_diff(expected) <= 3, "{alpha:?}"); + } +} + #[test] #[ignore = "requires OPENTAKE_MAIN10_FIXTURE pointing at a real HEVC Main10 clip"] fn continuous_decode_scales_real_main10_frames_without_corruption() { diff --git a/crates/opentake-media/tests/hdr.rs b/crates/opentake-media/tests/hdr.rs new file mode 100644 index 00000000..8b40a438 --- /dev/null +++ b/crates/opentake-media/tests/hdr.rs @@ -0,0 +1,173 @@ +use opentake_domain::MediaColorMetadata; +use opentake_media::{ + decode_frame_at, hdr_tonemap_filter, parse_probe, probe, ExportPreset, ExportResolution, + FrameRequest, VideoCodec, VideoEncoder, +}; +use serde_json::json; + +#[test] +fn hdr_probe_and_sdr_delivery_policy_preserve_source_metadata() { + for (transfer, expected_token) in [("smpte2084", "smpte2084"), ("arib-std-b67", "arib-std-b67")] + { + let probe = parse_probe(&json!({ + "streams": [{ + "codec_type": "video", + "width": 3840, + "height": 2160, + "avg_frame_rate": "30/1", + "color_primaries": "bt2020", + "color_transfer": transfer, + "color_space": "bt2020nc", + "color_range": "tv" + }], + "format": {"duration": "5.0"} + })); + + let color = probe.color.expect("HDR metadata must survive probing"); + assert_eq!( + color, + MediaColorMetadata { + primaries: Some("bt2020".into()), + transfer: Some(transfer.into()), + matrix: Some("bt2020nc".into()), + range: Some("tv".into()), + } + ); + assert!(color.is_hdr()); + let filter = hdr_tonemap_filter(&color).expect("PQ/HLG must choose an explicit tonemap"); + if cfg!(target_os = "macos") { + assert!(filter.contains("scale_vt=")); + assert!(filter.contains("color_transfer=bt709")); + assert!(filter.contains("hwdownload,format=p010le")); + } else { + assert!(filter.contains(expected_token)); + assert!(filter.contains("tonemap=")); + assert!(filter.contains("p=bt709:t=bt709:m=bt709")); + } + } + + let preset = ExportPreset::new(VideoCodec::H265, ExportResolution::P1080); + let args = preset.color_args(); + assert!(args + .windows(2) + .any(|pair| pair == ["-color_primaries", "bt709"])); + assert!(args.windows(2).any(|pair| pair == ["-color_trc", "bt709"])); + assert!(args.windows(2).any(|pair| pair == ["-colorspace", "bt709"])); +} + +#[test] +fn sdr_or_unknown_transfer_does_not_apply_hdr_tonemapping() { + let sdr = MediaColorMetadata { + primaries: Some("bt709".into()), + transfer: Some("bt709".into()), + matrix: Some("bt709".into()), + range: Some("tv".into()), + }; + assert!(!sdr.is_hdr()); + assert_eq!(hdr_tonemap_filter(&sdr), None); + assert_eq!(hdr_tonemap_filter(&MediaColorMetadata::default()), None); +} + +#[test] +fn packaged_hdr_decode_path_materializes_bt709_rgba_pixels() { + if !opentake_media::ffmpeg_status::ffmpeg_available() + || !opentake_media::ffmpeg_status::ffprobe_available() + { + return; + } + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("pq.mp4"); + let generated = std::process::Command::new("ffmpeg") + .args([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x90:rate=24", + "-frames:v", + "1", + "-vf", + "format=yuv420p10le", + "-c:v", + "libx265", + "-preset", + "ultrafast", + "-x265-params", + "log-level=error:hdr-opt=1:repeat-headers=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc", + "-color_primaries", + "bt2020", + "-color_trc", + "smpte2084", + "-colorspace", + "bt2020nc", + ]) + .arg(&source) + .output() + .unwrap(); + assert!( + generated.status.success(), + "generate HDR fixture: {}", + String::from_utf8_lossy(&generated.stderr) + ); + let metadata = probe(&source).unwrap(); + assert!(metadata.color.as_ref().is_some_and(|color| color.is_hdr())); + + let (_, frame) = decode_frame_at( + &source, + &FrameRequest { + time_secs: 0.0, + max_size: (160, 90), + ..FrameRequest::default() + }, + ) + .expect("platform HDR conversion must decode to RGBA"); + assert_eq!((frame.width, frame.height), (160, 90)); + let (min, max) = frame + .rgba + .chunks_exact(4) + .flat_map(|pixel| pixel[..3].iter().copied()) + .fold((u8::MAX, u8::MIN), |(min, max), value| { + (min.min(value), max.max(value)) + }); + assert!( + max.saturating_sub(min) > 32, + "tone-mapped frame must retain contrast" + ); + + let delivered = temp.path().join("delivery.mp4"); + let preset = ExportPreset::new(VideoCodec::H264, ExportResolution::P720); + let mut encoder = VideoEncoder::new(&delivered, frame.width, frame.height, 1, &preset).unwrap(); + encoder.push_frame(&frame).unwrap(); + encoder.finish().unwrap(); + let tags = std::process::Command::new("ffprobe") + .args([ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=color_primaries,color_transfer,color_space", + "-of", + "json", + ]) + .arg(&delivered) + .output() + .unwrap(); + assert!(tags.status.success()); + let tags: serde_json::Value = serde_json::from_slice(&tags.stdout).unwrap(); + assert_eq!( + tags.pointer("/streams/0/color_primaries"), + Some(&json!("bt709")) + ); + assert_eq!( + tags.pointer("/streams/0/color_transfer"), + Some(&json!("bt709")) + ); + assert_eq!( + tags.pointer("/streams/0/color_space"), + Some(&json!("bt709")) + ); +} diff --git a/crates/opentake-media/tests/loudness.rs b/crates/opentake-media/tests/loudness.rs new file mode 100644 index 00000000..c436e955 --- /dev/null +++ b/crates/opentake-media/tests/loudness.rs @@ -0,0 +1,107 @@ +use opentake_media::analysis::{ + analyze_loudness, apply_loudness_gain, LoudnessNormalizationConfig, +}; +use opentake_media::encode::mix::apply_true_peak_ceiling; + +const SAMPLE_RATE: u32 = 48_000; + +fn sine_fixture(amplitude: f32, frequency_hz: f32, duration_seconds: usize) -> Vec { + let sample_count = SAMPLE_RATE as usize * duration_seconds; + (0..sample_count) + .map(|index| { + let phase = index as f32 * frequency_hz * std::f32::consts::TAU / SAMPLE_RATE as f32; + phase.sin() * amplitude + }) + .collect() +} + +#[test] +fn normalization_reaches_configured_lufs_within_tolerance() { + let samples = sine_fixture(0.08, 997.0, 4); + let config = LoudnessNormalizationConfig { + target_lufs: -16.0, + true_peak_ceiling_dbtp: -1.0, + }; + + let analysis = analyze_loudness(&samples, SAMPLE_RATE, config).expect("analyze fixture"); + // Cross-checked against FFmpeg 8.1 loudnorm for the same 997 Hz / 0.08 + // amplitude / 48 kHz fixture (`input_i=-24.95`, `input_tp=-21.94`). + assert!((analysis.input_integrated_lufs - -24.95).abs() <= 0.2); + assert!((analysis.input_true_peak_dbtp - -21.94).abs() <= 0.1); + let normalized = apply_loudness_gain(&samples, analysis.gain_db); + let measured = analyze_loudness( + &normalized, + SAMPLE_RATE, + LoudnessNormalizationConfig { + target_lufs: analysis.output_integrated_lufs, + true_peak_ceiling_dbtp: 0.0, + }, + ) + .expect("measure normalized fixture"); + + assert!( + (measured.input_integrated_lufs - config.target_lufs).abs() <= 1.0, + "measured={} target={} gain={} input={} peak={}", + measured.input_integrated_lufs, + config.target_lufs, + analysis.gain_db, + analysis.input_integrated_lufs, + measured.input_true_peak_dbtp, + ); + assert!(measured.input_true_peak_dbtp <= config.true_peak_ceiling_dbtp + 0.05); +} + +fn verify_program_fixture(mut samples: Vec) { + let config = LoudnessNormalizationConfig::default(); + let analysis = analyze_loudness(&samples, SAMPLE_RATE, config).expect("analyze fixture"); + samples = apply_loudness_gain(&samples, analysis.gain_db); + apply_true_peak_ceiling(&mut samples, Some(config.true_peak_ceiling_dbtp)); + let measured = analyze_loudness(&samples, SAMPLE_RATE, config).expect("measure output"); + assert!( + (measured.input_integrated_lufs - config.target_lufs).abs() <= 1.0, + "measured={} target={} gain={} input={} peak={}", + measured.input_integrated_lufs, + config.target_lufs, + analysis.gain_db, + analysis.input_integrated_lufs, + measured.input_true_peak_dbtp, + ); + assert!(measured.input_true_peak_dbtp <= config.true_peak_ceiling_dbtp + 0.05); +} + +#[test] +fn speech_and_music_fixtures_reach_target_without_exceeding_true_peak() { + let sample_count = SAMPLE_RATE as usize * 5; + let speech = (0..sample_count) + .map(|index| { + let time = index as f32 / SAMPLE_RATE as f32; + let syllable = if (time * 3.2).fract() < 0.62 { + 1.0 + } else { + 0.08 + }; + let voiced = (std::f32::consts::TAU * 173.0 * time).sin() * 0.035 + + (std::f32::consts::TAU * 346.0 * time).sin() * 0.018; + let plosive = if index % 31_337 < 12 { 0.72 } else { 0.0 }; + voiced * syllable + plosive + }) + .collect(); + verify_program_fixture(speech); + + let music = (0..sample_count) + .map(|index| { + let time = index as f32 / SAMPLE_RATE as f32; + let tonal = (std::f32::consts::TAU * 220.0 * time).sin() * 0.045 + + (std::f32::consts::TAU * 329.63 * time).sin() * 0.035 + + (std::f32::consts::TAU * 440.0 * time).sin() * 0.025; + let beat_phase = index % (SAMPLE_RATE as usize / 2); + let beat = if beat_phase < 240 { + 0.35 * (1.0 - beat_phase as f32 / 240.0) + } else { + 0.0 + }; + tonal + beat + }) + .collect(); + verify_program_fixture(music); +} diff --git a/crates/opentake-media/tests/proxy.rs b/crates/opentake-media/tests/proxy.rs new file mode 100644 index 00000000..c1aa9f8c --- /dev/null +++ b/crates/opentake-media/tests/proxy.rs @@ -0,0 +1,104 @@ +use std::fs; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use opentake_media::{ + create_proxy, probe, MediaCancelToken, MediaError, ProxyProgressCallback, ProxyRequest, +}; + +fn make_video(path: &Path) { + let status = std::process::Command::new("ffmpeg") + .args([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=640x360:rate=30", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000", + "-t", + "1", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + ]) + .arg(path) + .status() + .expect("spawn ffmpeg fixture"); + assert!(status.success()); +} + +#[test] +fn proxy_creation_is_cancellable_atomic_persistent_and_source_preserving() { + if !opentake_media::ffmpeg_status::ffmpeg_available() + || !opentake_media::ffmpeg_status::ffprobe_available() + { + return; + } + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("source.mp4"); + let output = temp.path().join("media/proxies/source-proxy.mp4"); + make_video(&source); + let source_before = fs::read(&source).unwrap(); + let progress_values = Arc::new(Mutex::new(Vec::new())); + let capture = progress_values.clone(); + let progress: ProxyProgressCallback = Arc::new(move |done, total| { + capture.lock().unwrap().push((done, total)); + }); + + let result = create_proxy( + ProxyRequest { + source: &source, + output: &output, + max_size: (320, 180), + }, + &MediaCancelToken::new(), + Some(progress), + ) + .expect("create bounded proxy"); + + assert_eq!(result.path, output); + assert_eq!(result.width, 320); + assert_eq!(result.height, 180); + assert_eq!(result.source_sha256.len(), 64); + assert_eq!(fs::read(&source).unwrap(), source_before); + let proxy_probe = probe(&output).unwrap(); + assert_eq!( + (proxy_probe.width, proxy_probe.height), + (Some(320), Some(180)) + ); + assert!(proxy_probe.has_video && proxy_probe.has_audio); + let values = progress_values.lock().unwrap(); + assert_eq!(values.first().copied(), Some((0, 1000))); + assert_eq!(values.last().copied(), Some((1000, 1000))); + assert!(!temp + .path() + .join("media/proxies/source-proxy.mp4.partial") + .exists()); + + let cancelled = temp.path().join("media/proxies/cancelled.mp4"); + let cancel = MediaCancelToken::new(); + cancel.cancel(); + assert!(matches!( + create_proxy( + ProxyRequest { + source: &source, + output: &cancelled, + max_size: (320, 180), + }, + &cancel, + None, + ), + Err(MediaError::Cancelled) + )); + assert!(!cancelled.exists()); + assert!(!cancelled.with_extension("mp4.partial").exists()); +} diff --git a/crates/opentake-media/tests/stems.rs b/crates/opentake-media/tests/stems.rs new file mode 100644 index 00000000..51e0af6c --- /dev/null +++ b/crates/opentake-media/tests/stems.rs @@ -0,0 +1,219 @@ +use std::fs; +use std::path::Path; +use std::sync::Arc; + +use opentake_media::analysis::stems::{ + ensure_local_stem_model, separate_stems, verify_local_stem_model, StemExecution, + StemProgressCallback, StemSeparationRequest, +}; +use opentake_media::{MediaCancelToken, MediaError}; +use tempfile::TempDir; + +const SAMPLE_RATE: u32 = 48_000; +const FRAMES: usize = 48_000; + +fn write_stereo_fixture(path: &Path) { + let data_len = (FRAMES * 2 * 2) as u32; + let mut wav = Vec::with_capacity(44 + data_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_len).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&SAMPLE_RATE.to_le_bytes()); + wav.extend_from_slice(&(SAMPLE_RATE * 4).to_le_bytes()); + wav.extend_from_slice(&4_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + for frame in 0..FRAMES { + let t = frame as f32 / SAMPLE_RATE as f32; + let vocal = 0.28 * (std::f32::consts::TAU * 440.0 * t).sin(); + let music = 0.22 * (std::f32::consts::TAU * 997.0 * t).sin(); + for sample in [vocal + music, vocal - music] { + wav.extend_from_slice(&((sample.clamp(-1.0, 1.0) * 32767.0) as i16).to_le_bytes()); + } + } + fs::write(path, wav).expect("write deterministic stereo fixture"); +} + +fn assert_clean_output_dir(path: &Path) { + let entries = fs::read_dir(path) + .expect("read output directory") + .collect::, _>>() + .expect("collect output directory"); + assert!( + entries.is_empty(), + "cancelled job must clean partial outputs" + ); +} + +#[test] +fn local_or_explicit_provider_selection_cancellation_provenance_and_cleanup() { + let temp = TempDir::new().expect("temp root"); + let source = temp.path().join("center-vocal.wav"); + let models = temp.path().join("models"); + let outputs = temp.path().join("outputs"); + write_stereo_fixture(&source); + fs::create_dir_all(&outputs).expect("create outputs"); + + let installed = ensure_local_stem_model(&models).expect("install bundled local model"); + assert!(installed.path.is_file()); + assert_eq!(verify_local_stem_model(&models).unwrap(), installed); + + let progress_values = Arc::new(std::sync::Mutex::new(Vec::new())); + let progress_capture = progress_values.clone(); + let progress: StemProgressCallback = Arc::new(move |done, total| { + progress_capture.lock().unwrap().push((done, total)); + }); + let result = separate_stems( + StemSeparationRequest { + source: &source, + output_dir: &outputs, + execution: StemExecution::Local { model_dir: &models }, + }, + &MediaCancelToken::new(), + Some(progress), + ) + .expect("separate local stems"); + + assert!(result.vocals.path.is_file()); + assert!(result.accompaniment.path.is_file()); + assert_eq!(result.provenance.execution, "local:opentake-center-v1"); + assert_eq!(result.provenance.source_sha256.len(), 64); + assert_eq!( + result.provenance.model_sha256, + Some(installed.sha256.clone()) + ); + assert!(result.metrics.vocal_sdr_improvement_db >= 12.0); + let spec = opentake_media::PcmSpec { + sample_rate: SAMPLE_RATE, + channels: 2, + format: opentake_media::PcmFormat::F32, + }; + let mixture = opentake_media::decode_pcm_interleaved(&source, &spec, None).unwrap(); + let separated_vocals = + opentake_media::decode_pcm_interleaved(&result.vocals.path, &spec, None).unwrap(); + let separated_accompaniment = + opentake_media::decode_pcm_interleaved(&result.accompaniment.path, &spec, None).unwrap(); + let mut reference = Vec::with_capacity(FRAMES * 2); + let mut accompaniment_reference = Vec::with_capacity(FRAMES * 2); + for frame in 0..FRAMES { + let t = frame as f32 / SAMPLE_RATE as f32; + let vocal = 0.28 * (std::f32::consts::TAU * 440.0 * t).sin(); + let music = 0.22 * (std::f32::consts::TAU * 997.0 * t).sin(); + reference.extend_from_slice(&[vocal, vocal]); + // A user-facing accompaniment stem must remain audible after a mono + // export/downmix, so the isolated side signal is emitted dual-mono. + accompaniment_reference.extend_from_slice(&[music, music]); + } + let sdr = |candidate: &[f32]| { + let signal = reference + .iter() + .map(|sample| f64::from(*sample) * f64::from(*sample)) + .sum::(); + let error = reference + .iter() + .zip(candidate) + .map(|(expected, actual)| { + let delta = f64::from(*expected - *actual); + delta * delta + }) + .sum::() + .max(1.0e-12); + 10.0 * (signal / error).log10() + }; + let measured_improvement = sdr(&separated_vocals) - sdr(&mixture); + assert!( + measured_improvement >= 12.0, + "decoded vocals must improve SDR by >= 12 dB, got {measured_improvement:.3} dB" + ); + let accompaniment_signal = accompaniment_reference + .iter() + .map(|sample| f64::from(*sample) * f64::from(*sample)) + .sum::(); + let accompaniment_error = accompaniment_reference + .iter() + .zip(&separated_accompaniment) + .map(|(expected, actual)| { + let delta = f64::from(*expected - *actual); + delta * delta + }) + .sum::() + .max(1.0e-12); + let accompaniment_sdr = 10.0 * (accompaniment_signal / accompaniment_error).log10(); + assert!( + accompaniment_sdr >= 60.0, + "decoded accompaniment must be mono-compatible, got {accompaniment_sdr:.3} dB SDR" + ); + let mut mono_compatible_mix = Vec::with_capacity(FRAMES * 2); + for frame in 0..FRAMES { + let t = frame as f32 / SAMPLE_RATE as f32; + let vocal = 0.28 * (std::f32::consts::TAU * 440.0 * t).sin(); + let music = 0.22 * (std::f32::consts::TAU * 997.0 * t).sin(); + mono_compatible_mix.extend_from_slice(&[vocal + music, vocal + music]); + } + let reconstruction_signal = mono_compatible_mix + .iter() + .map(|sample| f64::from(*sample) * f64::from(*sample)) + .sum::(); + let reconstruction_error = mono_compatible_mix + .iter() + .zip(separated_vocals.iter().zip(&separated_accompaniment)) + .map(|(expected, (vocals, accompaniment))| { + let delta = f64::from(*expected - (*vocals + *accompaniment)); + delta * delta + }) + .sum::() + .max(1.0e-12); + let reconstruction_sdr = 10.0 * (reconstruction_signal / reconstruction_error).log10(); + assert!( + reconstruction_sdr >= 60.0, + "stem sum must reconstruct the documented mono-compatible mixture at >= 60 dB SDR, got {reconstruction_sdr:.3} dB" + ); + let progress_values = progress_values.lock().unwrap(); + assert_eq!(progress_values.first().copied(), Some((0, 1000))); + assert_eq!(progress_values.last().copied(), Some((1000, 1000))); + + let corrupt = fs::read(&installed.path).expect("read installed model"); + fs::write(&installed.path, [corrupt, b"corrupt".to_vec()].concat()) + .expect("corrupt installed model"); + assert!(matches!( + verify_local_stem_model(&models), + Err(MediaError::Checksum(_)) + )); + + let cancelled_outputs = temp.path().join("cancelled"); + fs::create_dir_all(&cancelled_outputs).unwrap(); + let cancel = MediaCancelToken::new(); + cancel.cancel(); + let cancelled = separate_stems( + StemSeparationRequest { + source: &source, + output_dir: &cancelled_outputs, + execution: StemExecution::Local { model_dir: &models }, + }, + &cancel, + None, + ); + assert!(matches!(cancelled, Err(MediaError::Cancelled))); + assert_clean_output_dir(&cancelled_outputs); + + let hosted_outputs = temp.path().join("hosted"); + fs::create_dir_all(&hosted_outputs).unwrap(); + let hosted = separate_stems( + StemSeparationRequest { + source: &source, + output_dir: &hosted_outputs, + execution: StemExecution::Hosted { + provider: "".to_string(), + model: "vendor/stems-v1".to_string(), + }, + }, + &MediaCancelToken::new(), + None, + ); + assert!(matches!(hosted, Err(MediaError::ModelInstall(_)))); + assert_clean_output_dir(&hosted_outputs); +} diff --git a/crates/opentake-motion/Cargo.toml b/crates/opentake-motion/Cargo.toml index 3d206684..77dbaba4 100644 --- a/crates/opentake-motion/Cargo.toml +++ b/crates/opentake-motion/Cargo.toml @@ -21,6 +21,14 @@ sha2 = "0.10" hex = "0.4" thiserror = "1" +# The live backend speaks Chrome DevTools Protocol directly. Both dependencies +# are feature-gated, so the default/offline crate keeps its existing surface. +base64 = { version = "0.22", optional = true } +tungstenite = { version = "0.29", optional = true, default-features = false, features = ["handshake"] } + +[target.'cfg(unix)'.dependencies] +libc = { version = "0.2", optional = true } + [dev-dependencies] # Offline PNG round-trip for the stub renderer's frame-file checks (mirrors the # render crate's dev-dep; no network, no assets). @@ -30,6 +38,6 @@ tempfile = "3" [features] default = [] # Gates the real headless-Chromium (CDP) backend behind a feature so neither the -# default build nor CI tests require a Chromium binary. The skeleton compiles -# unconditionally; only the live CDP wiring is feature-gated (see renderer.rs). -chromium = [] +# default build nor CI tests require a Chromium binary. Browser discovery and +# the fail-closed API compile unconditionally; live CDP wiring is feature-gated. +chromium = ["dep:base64", "dep:libc", "dep:tungstenite"] diff --git a/crates/opentake-motion/src/error.rs b/crates/opentake-motion/src/error.rs index 4f107f99..d7001e68 100644 --- a/crates/opentake-motion/src/error.rs +++ b/crates/opentake-motion/src/error.rs @@ -35,6 +35,11 @@ pub enum MotionError { #[error("render timed out after {0:?}")] Timeout(std::time::Duration), + /// The caller cancelled an in-flight render. Browser/profile/output cleanup + /// is complete before this error is returned. + #[error("render cancelled")] + Cancelled, + /// A sandbox policy was violated (e.g. a disallowed network origin). #[error("sandbox violation: {0}")] Sandbox(String), diff --git a/crates/opentake-motion/src/integration.rs b/crates/opentake-motion/src/integration.rs index 29bd1d1e..6e21919d 100644 --- a/crates/opentake-motion/src/integration.rs +++ b/crates/opentake-motion/src/integration.rs @@ -11,13 +11,15 @@ //! //! Decoding a frame file back to RGBA is deliberately *not* hard-wired to a PNG //! library here. Frames may be produced by the [`StubRenderer`](crate::renderer) -//! (our tiny stored-block PNG), by the later native headless-Chromium fallback +//! (our tiny stored-block PNG), by the native headless-Chromium fallback //! (standard PNG), by Motion Canvas image-sequence output, or by a future //! raw-RGBA fast path. So [`MotionClipSource`] takes a //! `FrameDecoder` — a `Fn(&Path) -> Option` — supplied by the //! integrating layer (which already owns an image/codec stack). Tests inject the -//! stub's own decoder; the app injects `image`/ffmpeg. This keeps this crate's -//! default dependency surface free of a decoder while still being fully testable. +//! stub's own decoder, and the feature-gated Chromium acceptance decodes a live +//! browser PNG through this same boundary; the app injects `image`/ffmpeg. This +//! keeps this crate's default dependency surface free of a decoder while still +//! being fully testable. use std::path::Path; @@ -59,6 +61,8 @@ impl<'a> MotionClipSource<'a> { /// Decode the frame at a 0-based index, clamping past-the-end to the last /// frame (freeze-frame hold, consistent with [`RenderedClip::frame_path`]). + /// Missing/corrupt input remains an absent frame (`None`); this adapter does + /// not repair, replace, or otherwise mutate the frame cache. pub fn frame(&self, frame: i64) -> Option { let idx = if frame < 0 { 0usize } else { frame as usize }; let path = self.clip.frame_path(idx)?; @@ -167,9 +171,29 @@ mod tests { #[test] fn missing_decoder_result_is_none() { let (clip, _tmp) = render_clip(true); - // A decoder that always fails surfaces None (compositor treats as absent). - let src = MotionClipSource::new(clip, |_p: &Path| None); - assert!(src.decoded_frame("ref", 0).is_none()); + let valid_path = clip.frames[0].clone(); + let corrupt_path = clip.frames[1].clone(); + let missing_path = clip.frames[2].clone(); + let cache_dir = valid_path.parent().unwrap().to_path_buf(); + let src = MotionClipSource::new(clip, image_decoder); + + let valid = src + .decoded_frame("ref", 0) + .expect("valid frame remains decodable"); + assert_eq!((valid.width, valid.height), (6, 4)); + + std::fs::write(&corrupt_path, b"not a png").unwrap(); + std::fs::remove_file(&missing_path).unwrap(); + let entries_before = std::fs::read_dir(&cache_dir).unwrap().count(); + + assert!(src.decoded_frame("ref", 1).is_none()); + assert_eq!(std::fs::read(&corrupt_path).unwrap(), b"not a png"); + assert!(src.decoded_frame("ref", 2).is_none()); + assert!(!missing_path.exists()); + assert_eq!( + std::fs::read_dir(&cache_dir).unwrap().count(), + entries_before + ); } #[test] diff --git a/crates/opentake-motion/src/lib.rs b/crates/opentake-motion/src/lib.rs index 82b31d37..a9271649 100644 --- a/crates/opentake-motion/src/lib.rs +++ b/crates/opentake-motion/src/lib.rs @@ -58,7 +58,8 @@ pub use manifest::{ DurationMode, DurationSpec, FpsPolicy, MotionPlugin, MotionPluginAuthor, ParamSpec, }; pub use renderer::{ - deterministic_clock_script, HeadlessChromiumRenderer, MotionRenderer, StubRenderer, + deterministic_clock_script, HeadlessChromiumRenderer, MotionCancellationToken, MotionRenderer, + StubRenderer, }; pub use sandbox::{AllowedOrigin, SandboxPolicy}; pub use source::{limits, MotionRenderRequest, MotionSource, ParamValue, RenderedClip}; diff --git a/crates/opentake-motion/src/renderer.rs b/crates/opentake-motion/src/renderer.rs index d7c92d5b..591b746d 100644 --- a/crates/opentake-motion/src/renderer.rs +++ b/crates/opentake-motion/src/renderer.rs @@ -8,18 +8,22 @@ //! each frame a solid color derived from `(frame, content-hash)`. It exists so //! the whole pipeline (validation → cache → frame files → compositor ingest) //! is unit-testable offline with **no browser**. -//! - [`HeadlessChromiumRenderer`] — the real backend skeleton. It documents and -//! sequences the deterministic CDP flow (virtual time + per-frame screenshot -//! with alpha, docs §3) but the live Chromium calls are gated behind the -//! `chromium` cargo feature. Without that feature (the default, and in CI) it -//! returns a clear [`MotionError::RendererUnavailable`] instead of pretending -//! to render. +//! - [`HeadlessChromiumRenderer`] — the live CDP backend (virtual time + +//! per-frame screenshot with alpha), gated behind the `chromium` cargo +//! feature. Without that feature it returns a clear +//! [`MotionError::RendererUnavailable`]. //! //! Both share [`deterministic_clock_script`] — the injected JS that freezes the //! page clock and exposes `OpenTake.seek(seconds)`, the render contract authors //! animate against. +#[cfg(feature = "chromium")] +use std::path::Path; use std::path::PathBuf; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use crate::cache::{content_hash, MotionCache}; use crate::error::{MotionError, MotionResult}; @@ -39,6 +43,25 @@ pub trait MotionRenderer { fn render(&self, req: &MotionRenderRequest) -> MotionResult; } +/// Cooperative cancellation shared between the caller and a live browser +/// render. Cancelling is idempotent and may happen from any thread. +#[derive(Clone, Debug, Default)] +pub struct MotionCancellationToken(Arc); + +impl MotionCancellationToken { + pub fn new() -> Self { + Self::default() + } + + pub fn cancel(&self) { + self.0.store(true, Ordering::Release); + } + + pub fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + /// The deterministic clock contract injected into every native fallback /// rendered document. It: /// 1. Pauses CSS/Web animations by pinning `document.timeline.currentTime`. @@ -54,13 +77,30 @@ pub fn deterministic_clock_script() -> &'static str { if (window.OpenTake && window.OpenTake.__installed) return; var current = 0; var listeners = []; + var randomState = 0x6d2b79f5; + try { Date.now = function () { return Math.round(current * 1000); }; } catch (e) {} + try { + Object.defineProperty(performance, 'now', { + configurable: true, + value: function () { return current * 1000; } + }); + } catch (e) {} + try { + Math.random = function () { + randomState = (randomState + 0x6d2b79f5) | 0; + var n = Math.imul(randomState ^ (randomState >>> 15), 1 | randomState); + n = (n + Math.imul(n ^ (n >>> 7), 61 | n)) ^ n; + return ((n ^ (n >>> 14)) >>> 0) / 4294967296; + }; + } catch (e) {} window.OpenTake = { __installed: true, // Current virtual time in seconds. currentTime: function () { return current; }, // Host calls this once per frame with t = frameIndex / fps. - seek: function (seconds) { + seek: async function (seconds) { current = seconds; + randomState = (0x6d2b79f5 ^ Math.round(seconds * 1000000)) | 0; try { if (document.timeline) { // Freeze the document timeline to the virtual clock (ms). @@ -70,9 +110,11 @@ pub fn deterministic_clock_script() -> &'static str { }); } } catch (e) { /* timeline may be read-only; listeners still fire */ } + var pending = []; for (var i = 0; i < listeners.length; i++) { - try { listeners[i](seconds); } catch (e) {} + try { pending.push(Promise.resolve(listeners[i](seconds))); } catch (e) {} } + await Promise.all(pending); }, // Authors register frame callbacks: OpenTake.onSeek(t => { ... }). onSeek: function (fn) { if (typeof fn === 'function') listeners.push(fn); } @@ -272,9 +314,9 @@ impl Crc32 { } } -/// The real headless-Chromium backend (skeleton). +/// The real headless-Chromium backend. /// -/// The deterministic fallback flow this skeleton documents, step by step, is: +/// Its deterministic fallback flow is: /// 1. Launch an offscreen Chromium with no network, an empty profile, and no /// filesystem access beyond the served document — applying [`SandboxPolicy`]. /// 2. `Emulation.setDeviceMetricsOverride` to the requested `width`×`height`. @@ -290,26 +332,102 @@ impl Crc32 { /// when `transparent`), writing the PNG to `cache_dir/frame_iiiii.png`. /// 7. Return the [`RenderedClip`]. /// -/// The live CDP wiring is gated behind the `chromium` cargo feature so neither -/// the default build nor CI needs a browser. Without the feature, [`render`] +/// The CDP wiring is gated behind the `chromium` cargo feature so the default +/// build does not require a browser or websocket dependency. The live path +/// locates Chrome/Chromium/Edge, uses a fresh disposable profile, injects a +/// strict CSP, intercepts every request with `Fetch`, and kills the browser on +/// cancellation, timeout, or protocol failure. Without the feature, [`render`] /// returns [`MotionError::RendererUnavailable`]. -/// -/// TODO(#34, native fallback): implement the steps above against a CDP -/// client (e.g. `chromiumoxide`) under `#[cfg(feature = "chromium")]`, including: -/// - locating/launching the browser binary and surfacing a clear error if absent, -/// - enforcing the network allowlist via `Fetch.enable` + request interception, -/// - applying the CSP and timeout fuse, -/// - mapping CDP failures to `MotionError::RenderFailed` / `::Timeout`. #[derive(Clone, Debug)] pub struct HeadlessChromiumRenderer { cache: MotionCache, policy: SandboxPolicy, + browser_path: Option, + cancellation: MotionCancellationToken, } impl HeadlessChromiumRenderer { /// Build the renderer with a cache and sandbox policy. pub fn new(cache: MotionCache, policy: SandboxPolicy) -> Self { - HeadlessChromiumRenderer { cache, policy } + HeadlessChromiumRenderer { + cache, + policy, + browser_path: None, + cancellation: MotionCancellationToken::new(), + } + } + + /// Override browser discovery. Useful for portable app bundles and for + /// deterministic crash-path tests. + pub fn with_browser_path(mut self, path: impl Into) -> Self { + self.browser_path = Some(path.into()); + self + } + + /// Attach a cooperative cancellation token. + pub fn with_cancellation_token(mut self, token: MotionCancellationToken) -> Self { + self.cancellation = token; + self + } + + /// Locate Chrome, Chromium, or Edge without launching it. An explicit + /// `OPENTAKE_CHROMIUM_PATH` wins, followed by platform install locations and + /// finally PATH. + pub fn find_browser() -> Option { + if let Some(path) = std::env::var_os("OPENTAKE_CHROMIUM_PATH").map(PathBuf::from) { + if path.is_file() { + return Some(path); + } + } + + const COMMON: &[&str] = &[ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/usr/bin/google-chrome-stable", + "/usr/bin/google-chrome", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/usr/bin/microsoft-edge", + ]; + if let Some(path) = COMMON.iter().map(PathBuf::from).find(|path| path.is_file()) { + return Some(path); + } + + for base in ["PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA"] { + let Some(base) = std::env::var_os(base) else { + continue; + }; + for relative in [ + "Google/Chrome/Application/chrome.exe", + "Microsoft/Edge/Application/msedge.exe", + "Chromium/Application/chrome.exe", + ] { + let path = PathBuf::from(&base).join(relative); + if path.is_file() { + return Some(path); + } + } + } + + let path = std::env::var_os("PATH")?; + for dir in std::env::split_paths(&path) { + for name in [ + "google-chrome-stable", + "google-chrome", + "chromium", + "chromium-browser", + "microsoft-edge", + "chrome.exe", + "msedge.exe", + ] { + let candidate = dir.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + } + None } /// The sandbox policy in effect. @@ -355,13 +473,7 @@ impl MotionRenderer for HeadlessChromiumRenderer { #[cfg(feature = "chromium")] { - // TODO(#34): real CDP render. Until implemented, fail loudly rather - // than silently — a half-done browser path must never masquerade as - // a successful render. - let _ = (&self.cache, Self::frame_time_grid(req)); - Err(MotionError::renderer_unavailable( - "headless-Chromium backend is enabled but not yet implemented (Issue #34 native fallback TODO)", - )) + chromium_backend::render(self, req) } #[cfg(not(feature = "chromium"))] { @@ -374,6 +486,716 @@ impl MotionRenderer for HeadlessChromiumRenderer { } } +#[cfg(feature = "chromium")] +mod chromium_backend { + use std::io::{BufRead, BufReader}; + use std::net::TcpStream; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::mpsc; + use std::thread; + use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + + use base64::Engine as _; + use serde_json::{json, Value}; + use tungstenite::stream::MaybeTlsStream; + use tungstenite::{Message, WebSocket}; + + use super::*; + + static PROFILE_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn trace(message: impl AsRef) { + if std::env::var_os("OPENTAKE_MOTION_TRACE").is_some() { + eprintln!("[opentake-motion] {}", message.as_ref()); + } + } + + pub(super) fn render( + renderer: &HeadlessChromiumRenderer, + req: &MotionRenderRequest, + ) -> MotionResult { + if renderer.cancellation.is_cancelled() { + return Err(MotionError::Cancelled); + } + + let browser_path = renderer + .browser_path + .clone() + .or_else(HeadlessChromiumRenderer::find_browser) + .ok_or_else(|| { + MotionError::renderer_unavailable( + "no supported Chrome, Chromium, or Edge executable was found; set OPENTAKE_CHROMIUM_PATH", + ) + })?; + trace(format!("browser={}", browser_path.display())); + + let document = match &req.source { + MotionSource::Code { html_css_js } => sandboxed_document(html_css_js, &renderer.policy), + MotionSource::Template { id, .. } => { + return Err(MotionError::unknown_template(format!( + "{id} (HeadlessChromiumRenderer requires the caller to resolve templates to inline Code)" + ))); + } + }; + + let hash = content_hash(req); + if renderer.cache.is_cached(req) { + return Ok(clip_from_cache(req, hash, renderer.cache.dir_for(req))); + } + + let dir = renderer.cache.ensure_dir(req)?; + remove_partial_frames(&dir)?; + let mut partial = PartialFrames::new(dir.clone()); + let deadline = Instant::now() + .checked_add(renderer.policy.timeout) + .unwrap_or_else(Instant::now); + check_abort(renderer, deadline)?; + + let (mut browser, websocket_url) = BrowserProcess::launch( + &browser_path, + deadline, + renderer.policy.timeout, + &renderer.cancellation, + )?; + trace("browser launched and CDP endpoint is ready"); + let (socket, _) = tungstenite::connect(websocket_url.as_str()).map_err(|error| { + MotionError::render_failed(format!("failed to connect to Chromium CDP: {error}")) + })?; + trace("connected to browser CDP"); + set_socket_poll_timeout(&socket)?; + let mut cdp = Cdp::new( + socket, + renderer.policy.clone(), + renderer.cancellation.clone(), + deadline, + ); + + let target = cdp.command( + "Target.createTarget", + json!({"url": "about:blank", "background": false}), + None, + )?; + let target_id = required_string(&target, "targetId")?; + let attached = cdp.command( + "Target.attachToTarget", + json!({"targetId": target_id, "flatten": true}), + None, + )?; + let session = required_string(&attached, "sessionId")?; + trace("created and attached render target"); + + cdp.command("Page.enable", json!({}), Some(&session))?; + cdp.command("Runtime.enable", json!({}), Some(&session))?; + cdp.command("Log.enable", json!({}), Some(&session))?; + // A background target can have its compositor throttled on Windows, + // leaving a later surface screenshot pending indefinitely. + cdp.command("Page.bringToFront", json!({}), Some(&session))?; + cdp.command( + "Fetch.enable", + json!({"patterns": [{"urlPattern": "*", "requestStage": "Request"}]}), + Some(&session), + )?; + cdp.command( + "Emulation.setDeviceMetricsOverride", + json!({ + "width": req.width, + "height": req.height, + "deviceScaleFactor": 1, + "mobile": false, + "screenWidth": req.width, + "screenHeight": req.height + }), + Some(&session), + )?; + let alpha = if req.transparent { 0.0 } else { 1.0 }; + cdp.command( + "Emulation.setDefaultBackgroundColorOverride", + json!({"color": {"r": 255, "g": 255, "b": 255, "a": alpha}}), + Some(&session), + )?; + cdp.command( + "Page.addScriptToEvaluateOnNewDocument", + json!({"source": deterministic_clock_script()}), + Some(&session), + )?; + + let url = HeadlessChromiumRenderer::data_url_for_code(&document); + cdp.command("Page.navigate", json!({"url": url}), Some(&session))?; + cdp.wait_for_event("Page.loadEventFired", Some(&session))?; + trace("inline motion document loaded"); + // Pausing before navigation also pauses the load lifecycle itself in + // recent Chromium. The deterministic clock is already installed before + // author code; freeze the browser's own timeline immediately after the + // synchronous inline document has loaded and before any frame capture. + cdp.command( + "Emulation.setVirtualTimePolicy", + json!({"policy": "pause"}), + Some(&session), + )?; + if let Some(blocked) = cdp.take_blocked_url() { + return Err(MotionError::sandbox(format!( + "network access to {blocked:?} is not in the allowlist" + ))); + } + + let mut frames = Vec::with_capacity(req.duration_frames as usize); + for (index, seconds) in HeadlessChromiumRenderer::frame_time_grid(req) + .into_iter() + .enumerate() + { + check_abort(renderer, deadline)?; + trace(format!("frame {index}: seek start at {seconds:.17}s")); + let expression = format!( + "(async () => {{ if (!window.OpenTake) throw new Error('OpenTake clock missing'); await window.OpenTake.seek({seconds:.17}); return window.OpenTake.currentTime(); }})()" + ); + let evaluated = cdp.command( + "Runtime.evaluate", + json!({ + "expression": expression, + "returnByValue": true, + "awaitPromise": true + }), + Some(&session), + )?; + trace(format!("frame {index}: seek complete")); + if evaluated + .get("exceptionDetails") + .and_then(Value::as_object) + .is_some() + { + return Err(MotionError::render_failed(format!( + "author document failed while seeking frame {index}: {evaluated}" + ))); + } + if let Some(blocked) = cdp.take_blocked_url() { + return Err(MotionError::sandbox(format!( + "network access to {blocked:?} is not in the allowlist" + ))); + } + + let captured = cdp.command( + "Page.captureScreenshot", + json!({ + "format": "png", + "fromSurface": true, + "captureBeyondViewport": false, + "optimizeForSpeed": false + }), + Some(&session), + )?; + trace(format!("frame {index}: screenshot captured")); + let encoded = required_string(&captured, "data")?; + let png = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|error| { + MotionError::render_failed(format!( + "Chromium returned malformed screenshot data for frame {index}: {error}" + )) + })?; + if !png.starts_with(b"\x89PNG\r\n\x1a\n") { + return Err(MotionError::render_failed(format!( + "Chromium returned a non-PNG screenshot for frame {index}" + ))); + } + let path = MotionCache::frame_file(&dir, index); + std::fs::write(&path, png)?; + frames.push(path); + } + + cdp.command("Target.closeTarget", json!({"targetId": target_id}), None)?; + partial.commit(); + browser.shutdown(); + + Ok(RenderedClip { + content_hash: hash, + frames, + fps: req.fps, + width: req.width, + height: req.height, + transparent: req.transparent, + }) + } + + fn clip_from_cache( + req: &MotionRenderRequest, + content_hash: String, + dir: PathBuf, + ) -> RenderedClip { + RenderedClip { + content_hash, + frames: (0..req.duration_frames as usize) + .map(|index| MotionCache::frame_file(&dir, index)) + .collect(), + fps: req.fps, + width: req.width, + height: req.height, + transparent: req.transparent, + } + } + + fn sandboxed_document(document: &str, policy: &SandboxPolicy) -> String { + let origins = policy + .allowed_origins + .iter() + .map(|origin| origin.as_str()) + .collect::>() + .join(" "); + let sources = if origins.is_empty() { + "'none'".to_owned() + } else { + origins + }; + let csp = format!( + "default-src 'none'; script-src 'unsafe-inline' data: {sources}; style-src 'unsafe-inline' data: {sources}; img-src data: {sources}; media-src data: {sources}; font-src data: {sources}; connect-src {sources}; object-src 'none'; base-uri 'none'; form-action 'none'; frame-src 'none'; worker-src 'none'" + ); + format!( + "{document}" + ) + } + + fn check_abort(renderer: &HeadlessChromiumRenderer, deadline: Instant) -> MotionResult<()> { + if renderer.cancellation.is_cancelled() { + return Err(MotionError::Cancelled); + } + if Instant::now() >= deadline { + return Err(MotionError::Timeout(renderer.policy.timeout)); + } + Ok(()) + } + + fn required_string(value: &Value, key: &str) -> MotionResult { + value + .get(key) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + MotionError::render_failed(format!( + "Chromium CDP response is missing string field {key:?}: {value}" + )) + }) + } + + fn remove_partial_frames(dir: &Path) -> MotionResult<()> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("frame_") && name.ends_with(".png") { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) + } + + struct PartialFrames { + dir: PathBuf, + committed: bool, + } + + impl PartialFrames { + fn new(dir: PathBuf) -> Self { + Self { + dir, + committed: false, + } + } + + fn commit(&mut self) { + self.committed = true; + } + } + + impl Drop for PartialFrames { + fn drop(&mut self) { + if !self.committed { + let _ = remove_partial_frames(&self.dir); + } + } + } + + struct BrowserProcess { + child: Child, + profile: PathBuf, + } + + impl BrowserProcess { + fn launch( + executable: &Path, + deadline: Instant, + timeout: Duration, + cancellation: &MotionCancellationToken, + ) -> MotionResult<(Self, String)> { + let profile = unique_profile_dir(); + std::fs::create_dir_all(&profile)?; + let mut command = Command::new(executable); + command + .args([ + "--headless=new", + "--remote-debugging-port=0", + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-component-update", + "--disable-client-side-phishing-detection", + "--disable-domain-reliability", + "--disable-sync", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--run-all-compositor-stages-before-draw", + "--metrics-recording-only", + "--disable-breakpad", + "--disable-extensions", + "--disable-dev-shm-usage", + "--disable-features=FileSystemAccessAPI,InterestFeedContentSuggestions,OptimizationHints,MediaRouter", + "--password-store=basic", + "--use-mock-keychain", + "about:blank", + ]) + .arg(format!("--user-data-dir={}", profile.display())) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command.spawn().map_err(|error| { + let _ = std::fs::remove_dir_all(&profile); + if error.kind() == std::io::ErrorKind::NotFound { + MotionError::renderer_unavailable(format!( + "Chromium executable does not exist at {}", + executable.display() + )) + } else { + MotionError::render_failed(format!( + "failed to launch Chromium at {}: {error}", + executable.display() + )) + } + })?; + let stderr = child + .stderr + .take() + .ok_or_else(|| MotionError::render_failed("Chromium stderr was not captured"))?; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + for line in BufReader::new(stderr).lines() { + match line { + Ok(line) => { + if sender.send(line).is_err() { + break; + } + } + Err(_) => break, + } + } + }); + + let mut process = BrowserProcess { child, profile }; + loop { + if cancellation.is_cancelled() { + return Err(MotionError::Cancelled); + } + if Instant::now() >= deadline { + return Err(MotionError::Timeout(timeout)); + } + if let Some(status) = process.child.try_wait()? { + return Err(MotionError::render_failed(format!( + "Chromium exited before CDP was ready: {status}" + ))); + } + match receiver.recv_timeout(Duration::from_millis(20)) { + Ok(line) => { + if let Some((_, url)) = line.split_once("DevTools listening on ") { + return Ok((process, url.trim().to_owned())); + } + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(MotionError::render_failed( + "Chromium closed stderr before publishing its CDP endpoint", + )); + } + } + } + } + + fn shutdown(&mut self) { + #[cfg(unix)] + { + // Chrome launches renderer and utility helpers. Terminate the + // isolated process group so a wedged author script cannot + // outlive the browser root and keep mutating its profile. + let process_group = -(self.child.id() as i32); + // SAFETY: launch places this child in a process group whose id + // is the child's pid; a negative pid targets that group only. + unsafe { + libc::kill(process_group, libc::SIGKILL); + } + } + let _ = self.child.kill(); + let _ = self.child.wait(); + } + + fn remove_profile(&self) { + // Chromium can keep helper processes alive for a few milliseconds + // after its root process exits. Those helpers may race a one-shot + // remove_dir_all by creating a final state file, leaving profiles + // behind on Linux timeout and cancellation paths. + const CLEANUP_ATTEMPTS: usize = 100; + for attempt in 0..CLEANUP_ATTEMPTS { + match std::fs::remove_dir_all(&self.profile) { + Ok(()) => return, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, + Err(_) if attempt + 1 < CLEANUP_ATTEMPTS => { + thread::sleep(Duration::from_millis(20)); + } + Err(_) => return, + } + } + } + } + + impl Drop for BrowserProcess { + fn drop(&mut self) { + self.shutdown(); + self.remove_profile(); + } + } + + fn unique_profile_dir() -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let counter = PROFILE_COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "opentake-chromium-{}-{nanos}-{counter}", + std::process::id() + )) + } + + type CdpSocket = WebSocket>; + + fn set_socket_poll_timeout(socket: &CdpSocket) -> MotionResult<()> { + match socket.get_ref() { + MaybeTlsStream::Plain(stream) => stream + .set_read_timeout(Some(Duration::from_millis(50))) + .map_err(MotionError::Io), + _ => Err(MotionError::render_failed( + "the local Chromium CDP endpoint unexpectedly used TLS", + )), + } + } + + struct Cdp { + socket: CdpSocket, + next_id: u64, + policy: SandboxPolicy, + cancellation: MotionCancellationToken, + deadline: Instant, + blocked_url: Option, + pending_events: Vec, + } + + impl Cdp { + fn new( + socket: CdpSocket, + policy: SandboxPolicy, + cancellation: MotionCancellationToken, + deadline: Instant, + ) -> Self { + Self { + socket, + next_id: 1, + policy, + cancellation, + deadline, + blocked_url: None, + pending_events: Vec::new(), + } + } + + fn command( + &mut self, + method: &str, + params: Value, + session: Option<&str>, + ) -> MotionResult { + let id = self.next_id; + self.next_id += 1; + let mut message = json!({"id": id, "method": method, "params": params}); + if let Some(session) = session { + message["sessionId"] = Value::String(session.to_owned()); + } + self.send(message)?; + + loop { + let value = self.read()?; + if value.get("id").and_then(Value::as_u64) == Some(id) { + if let Some(error) = value.get("error") { + return Err(MotionError::render_failed(format!( + "Chromium CDP {method} failed: {error}" + ))); + } + return Ok(value.get("result").cloned().unwrap_or_else(|| json!({}))); + } + self.handle_event_or_queue(value)?; + } + } + + fn wait_for_event(&mut self, method: &str, session: Option<&str>) -> MotionResult { + if let Some(index) = self.pending_events.iter().position(|event| { + event.get("method").and_then(Value::as_str) == Some(method) + && session.is_none_or(|expected| { + event.get("sessionId").and_then(Value::as_str) == Some(expected) + }) + }) { + return Ok(self.pending_events.remove(index)); + } + loop { + let value = self.read()?; + if value.get("method").and_then(Value::as_str) == Some(method) + && session.is_none_or(|expected| { + value.get("sessionId").and_then(Value::as_str) == Some(expected) + }) + { + return Ok(value); + } + self.handle_event_or_queue(value)?; + } + } + + fn take_blocked_url(&mut self) -> Option { + self.blocked_url.take() + } + + fn send(&mut self, value: Value) -> MotionResult<()> { + self.socket + .send(Message::text(value.to_string())) + .map_err(|error| { + MotionError::render_failed(format!( + "failed to send Chromium CDP command: {error}" + )) + }) + } + + fn read(&mut self) -> MotionResult { + loop { + if self.cancellation.is_cancelled() { + return Err(MotionError::Cancelled); + } + if Instant::now() >= self.deadline { + return Err(MotionError::Timeout(self.policy.timeout)); + } + match self.socket.read() { + Ok(Message::Text(text)) => { + return serde_json::from_str(text.as_ref()).map_err(|error| { + MotionError::render_failed(format!( + "Chromium sent malformed CDP JSON: {error}" + )) + }); + } + Ok(Message::Ping(payload)) => { + self.socket.send(Message::Pong(payload)).map_err(|error| { + MotionError::render_failed(format!( + "failed to answer Chromium CDP ping: {error}" + )) + })?; + } + Ok(Message::Close(reason)) => { + return Err(MotionError::render_failed(format!( + "Chromium CDP connection closed unexpectedly: {reason:?}" + ))); + } + Ok(_) => {} + Err(tungstenite::Error::Io(error)) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => {} + Err(error) => { + return Err(MotionError::render_failed(format!( + "failed to read Chromium CDP response: {error}" + ))); + } + } + } + } + + fn handle_event_or_queue(&mut self, value: Value) -> MotionResult<()> { + match value.get("method").and_then(Value::as_str) { + Some("Fetch.requestPaused") => self.handle_request(&value), + Some("Log.entryAdded") => { + let text = value + .get("params") + .and_then(|params| params.get("entry")) + .and_then(|entry| entry.get("text")) + .and_then(Value::as_str) + .unwrap_or_default(); + if (text.contains("Content Security Policy") + || text.contains("Refused to load") + || text.contains("Not allowed to load local resource") + || text.contains("violates the following")) + && self.blocked_url.is_none() + { + self.blocked_url = Some(text.to_owned()); + } + Ok(()) + } + Some("Inspector.targetCrashed" | "Target.targetCrashed") => { + Err(MotionError::render_failed("Chromium render target crashed")) + } + Some(_) => { + if self.pending_events.len() >= 256 { + self.pending_events.remove(0); + } + self.pending_events.push(value); + Ok(()) + } + None => Ok(()), + } + } + + fn handle_request(&mut self, event: &Value) -> MotionResult<()> { + let params = event.get("params").ok_or_else(|| { + MotionError::render_failed("Fetch.requestPaused event has no params") + })?; + let request_id = required_string(params, "requestId")?; + let url = params + .get("request") + .and_then(|request| request.get("url")) + .and_then(Value::as_str) + .ok_or_else(|| { + MotionError::render_failed("Fetch.requestPaused event has no request URL") + })?; + let session = event.get("sessionId").and_then(Value::as_str); + let allowed = url == "about:blank" || self.policy.check_url(url).is_ok(); + let id = self.next_id; + self.next_id += 1; + let (method, params) = if allowed { + ("Fetch.continueRequest", json!({"requestId": request_id})) + } else { + if self.blocked_url.is_none() { + self.blocked_url = Some(url.to_owned()); + } + ( + "Fetch.failRequest", + json!({"requestId": request_id, "errorReason": "BlockedByClient"}), + ) + }; + let mut message = json!({"id": id, "method": method, "params": params}); + if let Some(session) = session { + message["sessionId"] = Value::String(session.to_owned()); + } + self.send(message) + } + } +} + /// Percent-encode HTML for a `data:` URL: keep unreserved chars, encode the rest. fn percent_encode_html(s: &str) -> String { let mut out = String::with_capacity(s.len()); @@ -444,6 +1266,7 @@ mod tests { for (fa, fb) in ca.frames.iter().zip(cb.frames.iter()) { let ba = std::fs::read(fa).unwrap(); let bb = std::fs::read(fb).unwrap(); + assert!(ba.starts_with(b"\x89PNG\r\n\x1a\n")); assert_eq!(ba, bb, "same request must produce identical bytes"); } } @@ -463,6 +1286,20 @@ mod tests { assert_eq!(first.get_pixel(0, 0)[3], 0); let last = image::open(clip.frames.last().unwrap()).unwrap().to_rgba8(); assert_eq!(last.get_pixel(0, 0)[3], 255); + + // 200x100 RGBA scanlines exceed one 65,535-byte stored-deflate block. + // Decode a direct encoder result to prove multi-block zlib framing and + // exact RGBA values, not just the tiny single-block fixture above. + let rgba = [17, 34, 51, 68]; + let big_a = encode_solid_rgba_png(200, 100, rgba); + let big_b = encode_solid_rgba_png(200, 100, rgba); + assert_eq!(big_a, big_b); + let big = image::load_from_memory_with_format(&big_a, image::ImageFormat::Png) + .unwrap() + .to_rgba8(); + assert_eq!(big.dimensions(), (200, 100)); + assert_eq!(big.get_pixel(0, 0).0, rgba); + assert_eq!(big.get_pixel(199, 99).0, rgba); } #[test] @@ -481,17 +1318,43 @@ mod tests { let r = HeadlessChromiumRenderer::new(MotionCache::new(tmp.path()), SandboxPolicy::default()); let req = MotionRenderRequest::new(MotionSource::code(""), 30, 2, 10, 10); - let err = r.render(&req).unwrap_err(); - assert!( - matches!(err, MotionError::RendererUnavailable(_)), - "expected RendererUnavailable, got {err:?}" - ); + #[cfg(not(feature = "chromium"))] + { + let err = r.render(&req).unwrap_err(); + assert!( + matches!(err, MotionError::RendererUnavailable(_)), + "expected RendererUnavailable, got {err:?}" + ); + } + #[cfg(feature = "chromium")] + { + if HeadlessChromiumRenderer::find_browser().is_some() { + let clip = r.render(&req).expect("feature-enabled browser render"); + assert_eq!(clip.frame_count(), 2); + } else { + assert!(matches!( + r.render(&req), + Err(MotionError::RendererUnavailable(_)) + )); + } + } } #[test] fn chromium_applies_sandbox_size_before_unavailable() { - // A document over the ceiling fails with a Sandbox error, proving the - // policy is enforced even though no browser runs. + // Stub checks the default ceiling before creating its content-hash dir. + let stub_tmp = tempfile::tempdir().unwrap(); + let stub = StubRenderer::new(MotionCache::new(stub_tmp.path())); + let oversized = "x".repeat(crate::sandbox::DEFAULT_MAX_DOCUMENT_BYTES + 1); + let stub_req = MotionRenderRequest::new(MotionSource::code(oversized), 30, 1, 10, 10); + assert!(matches!( + stub.render(&stub_req), + Err(MotionError::Sandbox(_)) + )); + assert_eq!(std::fs::read_dir(stub_tmp.path()).unwrap().count(), 0); + + // Chromium checks its policy before browser discovery/launch and before + // creating a content-hash dir, in both feature configurations. let tmp = tempfile::tempdir().unwrap(); let policy = SandboxPolicy { max_document_bytes: 4, @@ -502,6 +1365,7 @@ mod tests { MotionRenderRequest::new(MotionSource::code(""), 30, 1, 10, 10); let err = r.render(&req).unwrap_err(); assert!(matches!(err, MotionError::Sandbox(_)), "got {err:?}"); + assert_eq!(std::fs::read_dir(tmp.path()).unwrap().count(), 0); } #[test] diff --git a/crates/opentake-motion/src/sandbox.rs b/crates/opentake-motion/src/sandbox.rs index 9585b8b2..12c34d39 100644 --- a/crates/opentake-motion/src/sandbox.rs +++ b/crates/opentake-motion/src/sandbox.rs @@ -49,12 +49,18 @@ impl AllowedOrigin { /// local dev origin) URL — plaintext remote origins are refused outright. pub fn parse(origin: &str) -> Option { let lower = origin.trim().trim_end_matches('/').to_ascii_lowercase(); - let is_https = lower.starts_with("https://"); - // Allow http only for loopback dev servers; never for remote hosts. - let is_local_http = lower.starts_with("http://localhost") - || lower.starts_with("http://127.0.0.1") - || lower.starts_with("http://[::1]"); - if (is_https || is_local_http) && lower.len() > "https://".len() { + let (scheme, authority) = lower.split_once("://")?; + if authority.is_empty() + || authority.contains(['/', '?', '#', '@']) + || authority.chars().any(char::is_whitespace) + { + return None; + } + let is_https = scheme == "https"; + // Allow http only for exact loopback hosts (with an optional port), + // never for lookalikes such as localhost.evil.example. + let is_local_http = scheme == "http" && is_loopback_authority(authority); + if is_https || is_local_http { Some(AllowedOrigin(lower)) } else { None @@ -64,6 +70,28 @@ impl AllowedOrigin { pub fn as_str(&self) -> &str { &self.0 } + + fn matches_url(&self, url: &str) -> bool { + let Some(rest) = url.strip_prefix(self.as_str()) else { + return false; + }; + rest.is_empty() || rest.starts_with(['/', '?', '#']) + } +} + +fn is_loopback_authority(authority: &str) -> bool { + for host in ["localhost", "127.0.0.1", "[::1]"] { + if authority == host { + return true; + } + if let Some(port) = authority + .strip_prefix(host) + .and_then(|rest| rest.strip_prefix(':')) + { + return !port.is_empty() && port.bytes().all(|byte| byte.is_ascii_digit()); + } + } + false } /// The sandbox policy applied to a single render. @@ -128,7 +156,7 @@ impl SandboxPolicy { let allowed = self .allowed_origins .iter() - .any(|o| lower.starts_with(o.as_str())); + .any(|origin| origin.matches_url(&lower)); if allowed { Ok(()) } else { @@ -138,7 +166,9 @@ impl SandboxPolicy { } } - /// Reject an inline document larger than the configured ceiling. + /// Reject an inline document larger than the configured byte ceiling. + /// Equality is accepted; UTF-8 text is charged by encoded bytes, matching + /// what is handed to Chromium and what consumes memory. pub fn check_document_size(&self, document: &str) -> MotionResult<()> { if document.len() > self.max_document_bytes { return Err(MotionError::sandbox(format!( @@ -175,6 +205,9 @@ mod tests { assert!(p.check_url("https://unpkg.com/thing").is_err()); // origin stored without trailing slash, case-insensitive match assert!(p.check_url("HTTPS://CDN.JSDELIVR.NET/a").is_ok()); + assert!(p + .check_url("https://cdn.jsdelivr.net.evil.example/a") + .is_err()); } #[test] @@ -183,7 +216,9 @@ mod tests { assert!(AllowedOrigin::parse("http://cdn.evil.com").is_none()); // but loopback http is allowed for local dev servers assert!(AllowedOrigin::parse("http://localhost:5173").is_some()); + assert!(AllowedOrigin::parse("http://localhost.evil.example").is_none()); assert!(AllowedOrigin::parse("https://example.com").is_some()); + assert!(AllowedOrigin::parse("https://example.com/path").is_none()); // junk assert!(AllowedOrigin::parse("ftp://x").is_none()); assert!(AllowedOrigin::parse("https://").is_none()); @@ -203,8 +238,15 @@ mod tests { max_document_bytes: 10, ..Default::default() }; - assert!(p.check_document_size("under10").is_ok()); - assert!(p.check_document_size("this is way over ten bytes").is_err()); + assert!(p.check_document_size("0123456789").is_ok()); + assert!(p.check_document_size("01234567890").is_err()); + + let utf8 = SandboxPolicy { + max_document_bytes: 3, + ..Default::default() + }; + assert!(utf8.check_document_size("é").is_ok()); // 2 UTF-8 bytes + assert!(utf8.check_document_size("éé").is_err()); // 4 UTF-8 bytes } #[test] diff --git a/crates/opentake-motion/tests/chromium.rs b/crates/opentake-motion/tests/chromium.rs new file mode 100644 index 00000000..f463180f --- /dev/null +++ b/crates/opentake-motion/tests/chromium.rs @@ -0,0 +1,266 @@ +#[cfg(feature = "chromium")] +mod live { + use std::collections::BTreeSet; + use std::fs; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::path::PathBuf; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + use std::thread; + use std::time::Duration; + + use opentake_motion::{ + HeadlessChromiumRenderer, MotionCache, MotionCancellationToken, MotionClipSource, + MotionError, MotionRenderRequest, MotionRenderer, MotionSource, SandboxPolicy, + }; + use opentake_render::{DecodedFrame, FrameProvider}; + + fn browser() -> PathBuf { + HeadlessChromiumRenderer::find_browser() + .expect("the live chromium test requires Chrome, Chromium, or Edge") + } + + fn request(document: &str) -> MotionRenderRequest { + MotionRenderRequest::new(MotionSource::code(document), 10, 3, 48, 32) + } + + fn renderer(root: &std::path::Path) -> HeadlessChromiumRenderer { + HeadlessChromiumRenderer::new( + MotionCache::new(root), + SandboxPolicy::offline_with_timeout(Duration::from_secs(20)), + ) + .with_browser_path(browser()) + } + + fn live_profiles() -> BTreeSet { + let prefix = format!("opentake-chromium-{}-", std::process::id()); + fs::read_dir(std::env::temp_dir()) + .unwrap() + .flatten() + .filter_map(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(&prefix) + .then(|| entry.path()) + }) + .collect() + } + + fn decoded(path: &std::path::Path) -> Option { + let rgba = image::open(path).ok()?.to_rgba8(); + Some(DecodedFrame::new( + rgba.width(), + rgba.height(), + rgba.into_raw(), + false, + )) + } + + pub(super) fn run() { + let profiles_before = live_profiles(); + let animation = r#" +
+ + "#; + + let first_root = tempfile::tempdir().unwrap(); + let second_root = tempfile::tempdir().unwrap(); + let first = renderer(first_root.path()) + .render(&request(animation)) + .unwrap(); + let second = renderer(second_root.path()) + .render(&request(animation)) + .unwrap(); + + assert_eq!(first.frame_count(), 3); + assert_eq!(second.frame_count(), 3); + for (a, b) in first.frames.iter().zip(&second.frames) { + assert_eq!(fs::read(a).unwrap(), fs::read(b).unwrap()); + } + assert_ne!( + fs::read(&first.frames[0]).unwrap(), + fs::read(&first.frames[2]).unwrap(), + "virtual time must advance the visible animation" + ); + let first_png = image::open(&first.frames[0]).unwrap().to_rgba8(); + assert_eq!(first_png.get_pixel(0, 0)[3], 255); + assert_eq!( + first_png.get_pixel(47, 31)[3], + 0, + "surface capture must preserve the transparent canvas outside content" + ); + let source = MotionClipSource::new(first.clone(), decoded); + let composited = source + .decoded_frame("motion", 2) + .expect("Chromium PNG enters MotionClipSource"); + assert_eq!((composited.width, composited.height), (48, 32)); + assert_eq!(composited.rgba.len(), 48 * 32 * 4); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let served = Arc::new(AtomicBool::new(false)); + let server_observed = Arc::clone(&served); + let server = thread::spawn(move || { + for _ in 0..250 { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0u8; 2048]; + let _ = stream.read(&mut request); + let svg = b""; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: image/svg+xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + svg.len() + ); + // Chromium may close the socket as soon as the image is + // decoded and the frame is captured. A late BrokenPipe + // therefore confirms neither a server nor render + // failure; accepting the request is the network-policy + // boundary this fixture needs to prove. + server_observed.store(true, Ordering::Release); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.write_all(svg); + return; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(20)); + } + Err(error) => panic!("loopback server failed: {error}"), + } + } + }); + let allowed_root = tempfile::tempdir().unwrap(); + let allowed = HeadlessChromiumRenderer::new( + MotionCache::new(allowed_root.path()), + SandboxPolicy::offline_with_timeout(Duration::from_secs(20)).allow_origin(&origin), + ) + .with_browser_path(browser()) + .render(&request(&format!(""))) + .unwrap(); + assert_eq!(allowed.frame_count(), 3); + server.join().unwrap(); + assert!(served.load(Ordering::Acquire)); + + let blocked_root = tempfile::tempdir().unwrap(); + let blocked = renderer(blocked_root.path()) + .render(&request( + r#""#, + )) + .unwrap_err(); + assert!(matches!(blocked, MotionError::Sandbox(_)), "{blocked:?}"); + assert!( + fs::read_dir(blocked_root.path()) + .unwrap() + .all(|entry| fs::read_dir(entry.unwrap().path()) + .unwrap() + .next() + .is_none()), + "a rejected render must not leave partial frames" + ); + + let filesystem_root = tempfile::tempdir().unwrap(); + let filesystem = renderer(filesystem_root.path()) + .render(&request(r#""#)) + .unwrap_err(); + assert!( + matches!(filesystem, MotionError::Sandbox(_)), + "{filesystem:?}" + ); + + let timeout_root = tempfile::tempdir().unwrap(); + let timeout_renderer = HeadlessChromiumRenderer::new( + MotionCache::new(timeout_root.path()), + SandboxPolicy::offline_with_timeout(Duration::from_millis(500)), + ) + .with_browser_path(browser()); + assert!(matches!( + timeout_renderer.render(&request("")), + Err(MotionError::Timeout(_)) + )); + + let crash_root = tempfile::tempdir().unwrap(); + let crash_renderer = HeadlessChromiumRenderer::new( + MotionCache::new(crash_root.path()), + SandboxPolicy::default(), + ) + .with_browser_path(if cfg!(windows) { + PathBuf::from(r"C:\Windows\System32\where.exe") + } else { + PathBuf::from("/usr/bin/false") + }); + let crashed = crash_renderer + .render(&request("
crash
")) + .unwrap_err(); + assert!( + matches!(crashed, MotionError::RenderFailed(_)), + "{crashed:?}" + ); + + let malformed_root = tempfile::tempdir().unwrap(); + assert!(matches!( + renderer(malformed_root.path()).render(&request(" ")), + Err(MotionError::InvalidSource(_)) + )); + + let cancellation = MotionCancellationToken::new(); + let cancelled_root = tempfile::tempdir().unwrap(); + let cancellation_for_render = cancellation.clone(); + let cancelled_cache = cancelled_root.path().to_path_buf(); + let cancelled_browser = browser(); + let render_thread = thread::spawn(move || { + HeadlessChromiumRenderer::new( + MotionCache::new(cancelled_cache), + SandboxPolicy::offline_with_timeout(Duration::from_secs(20)), + ) + .with_browser_path(cancelled_browser) + .with_cancellation_token(cancellation_for_render) + .render(&request("")) + }); + thread::sleep(Duration::from_millis(200)); + cancellation.cancel(); + assert!(matches!( + render_thread.join().unwrap(), + Err(MotionError::Cancelled) + )); + + assert_eq!( + live_profiles(), + profiles_before, + "success, policy failure, timeout, crash, and cancellation must clean browser profiles" + ); + } +} + +#[cfg(feature = "chromium")] +#[test] +fn virtual_time_network_csp_timeout_cleanup_and_frame_identity() { + live::run(); +} + +#[cfg(not(feature = "chromium"))] +#[test] +fn virtual_time_network_csp_timeout_cleanup_and_frame_identity() { + use opentake_motion::{ + HeadlessChromiumRenderer, MotionCache, MotionError, MotionRenderRequest, MotionRenderer, + MotionSource, SandboxPolicy, + }; + + let root = tempfile::tempdir().unwrap(); + let renderer = + HeadlessChromiumRenderer::new(MotionCache::new(root.path()), SandboxPolicy::default()); + let request = MotionRenderRequest::new(MotionSource::code("
"), 30, 1, 16, 16); + assert!(matches!( + renderer.render(&request), + Err(MotionError::RendererUnavailable(_)) + )); +} diff --git a/crates/opentake-ops/src/command.rs b/crates/opentake-ops/src/command.rs index 57586f08..7edf8958 100644 --- a/crates/opentake-ops/src/command.rs +++ b/crates/opentake-ops/src/command.rs @@ -16,10 +16,14 @@ //! Ripple refusals (a sync-locked follower can't absorb the shift) abort like a //! validation error: `Err(EditError::Refused)`, document untouched. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use opentake_domain::{ - ChromaKey, ClipType, ColorGrade, Crop, Effect, Interpolation, Mask, Timeline, Transform, + AudioDenoise, CaptionTranslationInput, ChromaKey, Clip, ClipType, ColorGrade, ColorMatchInput, + Crop, Effect, Interpolation, LoudnessNormalization, LutReference, Mask, MaskShape, + MediaManifestEntry, NestedSequence, ScriptAssemblyPlan, StabilizationTrack, Timeline, Track, + Transform, Transition, TransitionKind, VoiceModelRecord, MAX_MASKS_PER_CLIP, + MAX_POLYGON_MASK_POINTS, }; use crate::editor_state::EditorState; @@ -40,6 +44,462 @@ pub enum EditError { Refused(String), } +#[cfg(test)] +mod motion_media_transaction_tests { + use super::*; + use crate::id::SeqIdGen; + use opentake_domain::{MediaSource, Track}; + + fn media(id: &str) -> MediaManifestEntry { + MediaManifestEntry { + id: id.into(), + name: format!("{id}.mp4"), + kind: ClipType::Video, + source: MediaSource::Project { + relative_path: format!("media/{id}.mp4"), + }, + duration: 1.0, + generation_input: None, + source_width: Some(64), + source_height: Some(36), + source_fps: Some(30.0), + has_audio: Some(false), + color: None, + proxy: None, + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + } + } + + fn clip(media_ref: &str, track_index: usize) -> ClipEntry { + ClipEntry { + media_ref: media_ref.into(), + media_type: ClipType::Video, + source_clip_type: ClipType::Video, + track_index, + start_frame: 0, + duration_frames: 30, + trim_start_frame: None, + trim_end_frame: None, + has_audio: false, + add_linked_audio: false, + transform: None, + } + } + + #[test] + fn register_and_add_is_one_undoable_document_transaction() { + let mut state = EditorState::default(); + let ids = SeqIdGen::default(); + let result = apply( + &mut state, + EditCommand::RegisterMediaAndAddClip { + media: media("motion-a"), + entry: clip("motion-a", 0), + auto_track: true, + }, + &ids, + ) + .unwrap(); + + assert_eq!(result.action_name, "Add Motion Graphic"); + assert_eq!(state.manifest.entries.len(), 1); + assert_eq!(state.timeline.tracks.len(), 1); + assert_eq!(state.timeline.tracks[0].clips.len(), 1); + assert_eq!(state.undo_depth(), 1); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert!(state.manifest.entries.is_empty()); + assert!(state.timeline.tracks.is_empty()); + } + + #[test] + fn failed_register_and_place_leaves_manifest_timeline_and_history_unchanged() { + let mut state = EditorState::default(); + state + .timeline + .tracks + .push(Track::new("audio", ClipType::Audio)); + let before = state.clone(); + let error = apply( + &mut state, + EditCommand::RegisterMediaAndAddClip { + media: media("motion-a"), + entry: clip("motion-a", 0), + auto_track: false, + }, + &SeqIdGen::default(), + ) + .unwrap_err(); + + assert!(error.to_string().contains("not compatible")); + assert_eq!(state.timeline, before.timeline); + assert_eq!(state.manifest, before.manifest); + assert_eq!(state.undo_depth(), before.undo_depth()); + assert_eq!(state.version(), before.version()); + } + + #[test] + fn register_and_swap_preserves_clip_identity_and_undo_restores_old_asset() { + let mut state = EditorState::default(); + let ids = SeqIdGen::default(); + let added = apply( + &mut state, + EditCommand::RegisterMediaAndAddClip { + media: media("motion-a"), + entry: clip("motion-a", 0), + auto_track: true, + }, + &ids, + ) + .unwrap(); + let clip_id = added.affected_clip_ids[0].clone(); + + let edited = apply( + &mut state, + EditCommand::RegisterMediaAndSwapClip { + media: media("motion-b"), + clip_id: clip_id.clone(), + }, + &ids, + ) + .unwrap(); + assert_eq!(edited.action_name, "Edit Motion Graphic"); + assert_eq!(edited.affected_clip_ids, vec![clip_id.clone()]); + assert_eq!(state.manifest.entries.len(), 2); + assert_eq!(state.timeline.tracks[0].clips[0].id, clip_id); + assert_eq!(state.timeline.tracks[0].clips[0].media_ref, "motion-b"); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert_eq!(state.manifest.entries.len(), 1); + assert_eq!(state.timeline.tracks[0].clips[0].media_ref, "motion-a"); + } + + #[test] + fn register_swap_and_clear_masks_is_one_reversible_transaction() { + let mut state = EditorState::default(); + let ids = SeqIdGen::default(); + let added = apply( + &mut state, + EditCommand::RegisterMediaAndAddClip { + media: media("source-a"), + entry: clip("source-a", 0), + auto_track: true, + }, + &ids, + ) + .unwrap(); + let clip_id = added.affected_clip_ids[0].clone(); + state.timeline.tracks[0].clips[0].masks = vec![Mask::default()]; + let undo_depth_before = state.undo_depth(); + + let edited = apply( + &mut state, + EditCommand::RegisterMediaAndSwapClipClearingMasks { + media: media("object-removed-b"), + clip_id: clip_id.clone(), + }, + &ids, + ) + .unwrap(); + + assert_eq!(edited.action_name, "Remove Masked Object"); + assert_eq!(edited.affected_clip_ids, vec![clip_id]); + assert_eq!(state.undo_depth(), undo_depth_before + 1); + assert_eq!(state.manifest.entries.len(), 2); + assert_eq!( + state.timeline.tracks[0].clips[0].media_ref, + "object-removed-b" + ); + assert!(state.timeline.tracks[0].clips[0].masks.is_empty()); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert_eq!(state.manifest.entries.len(), 1); + assert_eq!(state.timeline.tracks[0].clips[0].media_ref, "source-a"); + assert_eq!( + state.timeline.tracks[0].clips[0].masks, + vec![Mask::default()] + ); + + apply(&mut state, EditCommand::Redo, &ids).unwrap(); + assert_eq!(state.manifest.entries.len(), 2); + assert_eq!( + state.timeline.tracks[0].clips[0].media_ref, + "object-removed-b" + ); + assert!(state.timeline.tracks[0].clips[0].masks.is_empty()); + } +} + +#[cfg(test)] +mod color_match_command_tests { + use super::*; + use crate::id::SeqIdGen; + use opentake_domain::Rgb; + + fn input() -> ColorMatchInput { + ColorMatchInput { + reference_media_ref: "reference".into(), + reference_frame: 3, + target_frame: 12, + algorithm: "fixture-match".into(), + algorithm_version: 1, + target_mean_linear: Rgb::new(0.3, 0.2, 0.1), + reference_mean_linear: Rgb::new(0.1, 0.2, 0.3), + delta_e_before: 20.0, + delta_e_after: 1.0, + target_luma_before: 0.2, + target_luma_after: 0.2, + } + } + + #[test] + fn apply_color_match_is_one_undoable_edit_and_manual_grade_clears_provenance() { + let mut state = EditorState::default(); + let ids = SeqIdGen::default(); + apply( + &mut state, + EditCommand::InsertTrack { + kind: ClipType::Video, + at: None, + }, + &ids, + ) + .unwrap(); + let clip_id = ids.next_id(); + state.timeline.tracks[0] + .clips + .push(Clip::new(clip_id.clone(), "target", 10, 20)); + let grade = ColorGrade { + temperature: 0.2, + ..ColorGrade::default() + }; + let undo_before = state.undo_depth(); + + let result = apply( + &mut state, + EditCommand::ApplyColorMatch { + clip_id: clip_id.clone(), + grade, + input: input(), + }, + &ids, + ) + .unwrap(); + assert_eq!(result.action_name, "Match Color"); + assert_eq!(state.undo_depth(), undo_before + 1); + assert_eq!(state.timeline.tracks[0].clips[0].color_grade, Some(grade)); + assert_eq!( + state.timeline.tracks[0].clips[0] + .color_match_input + .as_ref() + .unwrap() + .reference_media_ref, + "reference" + ); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert!(state.timeline.tracks[0].clips[0].color_grade.is_none()); + assert!(state.timeline.tracks[0].clips[0] + .color_match_input + .is_none()); + apply(&mut state, EditCommand::Redo, &ids).unwrap(); + apply( + &mut state, + EditCommand::SetColorGrade { + clip_ids: vec![clip_id], + grade: Some(ColorGrade::default()), + }, + &ids, + ) + .unwrap(); + assert!(state.timeline.tracks[0].clips[0] + .color_match_input + .is_none()); + } +} + +#[cfg(test)] +mod aligned_stem_track_tests { + use super::*; + use crate::id::SeqIdGen; + + fn stem(media_ref: &str) -> ClipEntry { + ClipEntry { + media_ref: media_ref.into(), + media_type: ClipType::Audio, + source_clip_type: ClipType::Audio, + track_index: 0, + start_frame: 40, + duration_frames: 100, + trim_start_frame: None, + trim_end_frame: None, + has_audio: true, + add_linked_audio: false, + transform: None, + } + } + + #[test] + fn aligned_stems_use_separate_tracks_and_one_undo_entry() { + let mut state = EditorState::default(); + let ids = SeqIdGen::default(); + let result = apply( + &mut state, + EditCommand::AddClipsToSeparateAutoTracks { + entries: vec![stem("vocals"), stem("accompaniment")], + }, + &ids, + ) + .unwrap(); + assert_eq!(result.action_name, "Import Stems To Tracks"); + assert_eq!(result.affected_clip_ids.len(), 2); + assert_eq!(state.timeline.tracks.len(), 2); + assert!(state.timeline.tracks.iter().all(|track| { + track.kind == ClipType::Audio + && track.clips.len() == 1 + && track.clips[0].start_frame == 40 + && track.clips[0].duration_frames == 100 + })); + assert_eq!(state.undo_depth(), 1); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert!(state.timeline.tracks.is_empty()); + } +} + +#[cfg(test)] +mod loudness_command_tests { + use super::*; + use crate::id::SeqIdGen; + + fn normalization() -> LoudnessNormalization { + LoudnessNormalization { + target_lufs: -16.0, + true_peak_ceiling_dbtp: -1.0, + input_integrated_lufs: -23.0, + input_true_peak_dbtp: -8.0, + gain_db: 7.0, + output_integrated_lufs: -16.0, + output_true_peak_dbtp: -1.0, + } + } + + #[test] + fn loudness_apply_reset_and_undo_are_one_step_operations() { + let mut timeline = Timeline::new(); + let mut track = Track::new("a1", ClipType::Audio); + let mut clip = Clip::new("audio", "asset", 0, 90); + clip.media_type = ClipType::Audio; + clip.source_clip_type = ClipType::Audio; + track.clips.push(clip); + timeline.tracks.push(track); + let mut state = EditorState::from_timeline(timeline); + let ids = SeqIdGen::default(); + + apply( + &mut state, + EditCommand::SetLoudnessNormalization { + clip_id: "audio".to_string(), + normalization: Some(normalization()), + }, + &ids, + ) + .unwrap(); + assert_eq!( + state.timeline.tracks[0].clips[0].loudness_normalization, + Some(normalization()) + ); + assert_eq!(state.undo_depth(), 1); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert!(state.timeline.tracks[0].clips[0] + .loudness_normalization + .is_none()); + apply(&mut state, EditCommand::Redo, &ids).unwrap(); + assert_eq!( + state.timeline.tracks[0].clips[0].loudness_normalization, + Some(normalization()) + ); + + apply( + &mut state, + EditCommand::SetLoudnessNormalization { + clip_id: "audio".to_string(), + normalization: None, + }, + &ids, + ) + .unwrap(); + assert!(state.timeline.tracks[0].clips[0] + .loudness_normalization + .is_none()); + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert_eq!( + state.timeline.tracks[0].clips[0].loudness_normalization, + Some(normalization()) + ); + } +} + +#[cfg(test)] +mod denoise_command_tests { + use super::*; + use crate::id::SeqIdGen; + use opentake_domain::DenoiseMode; + + #[test] + fn denoise_apply_reset_and_undo_are_one_step_operations() { + let mut timeline = Timeline::new(); + let mut track = Track::new("a1", ClipType::Audio); + let mut clip = Clip::new("audio", "asset", 0, 90); + clip.media_type = ClipType::Audio; + clip.source_clip_type = ClipType::Audio; + track.clips.push(clip); + timeline.tracks.push(track); + let mut state = EditorState::from_timeline(timeline); + let config = AudioDenoise { + mode: DenoiseMode::Voice, + strength: 0.8, + preview_enabled: true, + }; + + apply( + &mut state, + EditCommand::SetAudioDenoise { + clip_id: "audio".to_string(), + denoise: Some(config), + }, + &SeqIdGen::default(), + ) + .unwrap(); + assert_eq!( + state.timeline.tracks[0].clips[0].audio_denoise, + Some(config) + ); + assert_eq!(state.undo_depth(), 1); + apply(&mut state, EditCommand::Undo, &SeqIdGen::default()).unwrap(); + assert!(state.timeline.tracks[0].clips[0].audio_denoise.is_none()); + apply(&mut state, EditCommand::Redo, &SeqIdGen::default()).unwrap(); + assert_eq!( + state.timeline.tracks[0].clips[0].audio_denoise, + Some(config) + ); + + apply( + &mut state, + EditCommand::SetAudioDenoise { + clip_id: "audio".to_string(), + denoise: None, + }, + &SeqIdGen::default(), + ) + .unwrap(); + assert!(state.timeline.tracks[0].clips[0].audio_denoise.is_none()); + } +} + impl std::fmt::Display for EditError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -154,6 +614,16 @@ pub struct CaptionEntry { pub caption_group_id: String, } +/// One reviewed caption text replacement. A batch is validated completely +/// before mutation, preserving clip identity and timing as one undo step. +#[derive(Clone, Debug)] +pub struct CaptionTranslationChange { + pub clip_id: String, + pub expected_source_text: String, + pub translated_text: String, + pub input: CaptionTranslationInput, +} + /// A single clip property assignment for [`EditCommand::SetClipProperties`]. /// `None` fields are left unchanged; setting a scalar clears the matching /// keyframe track (mirrors `applyPropertyChanges`). @@ -223,12 +693,66 @@ pub enum KeyframeValue { /// The unified editing command. Every editing surface routes through this. #[derive(Clone, Debug)] pub enum EditCommand { + /// Register an editable child timeline and place one compound clip that + /// references it. Sequence + clip creation is one undoable transaction. + CreateNestedSequence { + name: String, + timeline: Timeline, + track_index: usize, + start_frame: i32, + duration_frames: i32, + }, + /// Turn selected root clips into one editable compound clip. The child + /// timeline keeps their relative tracks/timing and source edits. + CreateNestedSequenceFromClips { name: String, clip_ids: Vec }, + /// Apply any ordinary edit command to a child timeline while preserving one + /// root-level undo snapshot and the shared media manifest. + EditNestedSequence { + sequence_id: String, + command: Box, + }, + /// Replace the editable contents of one existing nested sequence. + SetNestedSequenceTimeline { + sequence_id: String, + timeline: Timeline, + }, + /// Rename a nested sequence without changing references. + RenameNestedSequence { sequence_id: String, name: String }, + /// Replace one compound clip with clipped copies of its child timeline + /// tracks, preserving media edits and keeping the operation undoable. + DissolveNestedSequence { clip_id: String }, /// Overwrite-place clips (clears each destination range first). AddClips { entries: Vec }, /// Overwrite-place clips on fresh shared tracks chosen by media type. /// Visual entries share one new visual track; audio entries share one new /// audio track. Track insertion and placement commit as one transaction. AddClipsAutoTrack { entries: Vec }, + /// Place each entry on its own fresh compatible track in one transaction. + /// Used for aligned stems that intentionally overlap in time. + AddClipsToSeparateAutoTracks { entries: Vec }, + /// Register one already validated project-managed video and place it in the + /// same undo snapshot. Used by deterministic external renderers so undo + /// removes both the generated clip and its manifest record. + RegisterMediaAndAddClip { + media: MediaManifestEntry, + entry: ClipEntry, + auto_track: bool, + }, + /// Register a newly rendered replacement and swap an existing clip to it in + /// one undo snapshot. Undo restores the old media ref and removes the new + /// manifest record while leaving the prior rendered asset available. + RegisterMediaAndSwapClip { + media: MediaManifestEntry, + clip_id: String, + }, + /// Register a rendered replacement, swap an existing clip to it, and + /// remove the editable masks that were baked into that derivative. The + /// three mutations share one undo snapshot so undo restores both the + /// source media and its masks. + RegisterMediaAndSwapClipClearingMasks { + media: MediaManifestEntry, + clip_id: String, + }, /// Ripple-insert clips at `at_frame`, pushing later clips right. InsertClips { track_index: usize, @@ -322,6 +846,17 @@ pub enum EditCommand { clip_ids: Vec, grade: Option, }, + /// Apply a sampled reference match and its persisted provenance together. + ApplyColorMatch { + clip_id: String, + grade: ColorGrade, + input: ColorMatchInput, + }, + /// Set or clear one project-managed 3D LUT on one or more clips. + SetLut { + clip_ids: Vec, + lut: Option, + }, /// Set (or clear with `None`) the chroma key on one or more clips. SetChromaKey { clip_ids: Vec, @@ -337,6 +872,36 @@ pub enum EditCommand { clip_ids: Vec, effects: Vec, }, + /// Apply or reset one source analysis as an undoable audio operation. + SetLoudnessNormalization { + clip_id: String, + normalization: Option, + }, + /// Apply or reset local non-destructive denoise parameters. + SetAudioDenoise { + clip_id: String, + denoise: Option, + }, + /// Persist a source-bound, editable stabilization analysis on one video clip. + ApplyStabilization { + clip_id: String, + solution: StabilizationTrack, + }, + /// Change user-facing stabilization strength and/or safety crop margin. + AdjustStabilization { + clip_id: String, + strength: Option, + crop_margin: Option, + }, + /// Remove stabilization while preserving authored transforms and source media. + ResetStabilization { clip_id: String }, + /// Set or clear the visual transition at one exact adjacent clip boundary. + SetTransition { + from_clip_id: String, + to_clip_id: String, + kind: Option, + duration_frames: i32, + }, /// Ripple-delete project-frame ranges on a track, closing the gaps. RippleDeleteRanges { track_index: usize, @@ -371,6 +936,20 @@ pub enum EditCommand { /// composing `InsertTrack` + `AddTexts` would be two undo steps and could not /// stamp `caption_group_id`. Empty `entries` is a no-op (no track, no change). AddCaptions { entries: Vec }, + /// Accept a reviewed batch of translated captions without changing IDs, + /// track placement, or frame ranges. + ApplyCaptionTranslations { + changes: Vec, + }, + /// Persist a reviewable script assembly plan without placing any clips. + SaveScriptAssemblyPlan { plan: ScriptAssemblyPlan }, + /// Apply one already persisted plan to fresh visual/narration tracks as a + /// single timeline transaction. + ApplyScriptAssemblyPlan { plan_id: String }, + /// Persist one consent-bearing provider voice identity. + SaveVoiceModel { record: VoiceModelRecord }, + /// Permanently mark a provider voice as revoked in project metadata. + RevokeVoiceModel { voice_model_id: String }, /// Link clips into one group. Link { clip_ids: Vec }, /// Unlink clips (and their whole groups). @@ -497,13 +1076,59 @@ pub fn apply( )) } - EditCommand::AddClips { entries } => add_clips(state, entries, ids), - EditCommand::AddClipsAutoTrack { entries } => add_clips_auto_track(state, entries, ids), - EditCommand::InsertClips { + EditCommand::CreateNestedSequence { + name, + timeline, track_index, - at_frame, - entries, - } => insert_clips(state, track_index, at_frame, entries, ids), + start_frame, + duration_frames, + } => create_nested_sequence( + state, + name, + timeline, + track_index, + start_frame, + duration_frames, + ids, + ), + EditCommand::CreateNestedSequenceFromClips { name, clip_ids } => { + create_nested_sequence_from_clips(state, name, clip_ids, ids) + } + EditCommand::EditNestedSequence { + sequence_id, + command, + } => edit_nested_sequence(state, sequence_id, *command, ids), + EditCommand::SetNestedSequenceTimeline { + sequence_id, + timeline, + } => set_nested_sequence_timeline(state, sequence_id, timeline), + EditCommand::RenameNestedSequence { sequence_id, name } => { + rename_nested_sequence(state, sequence_id, name) + } + EditCommand::DissolveNestedSequence { clip_id } => { + dissolve_nested_sequence(state, clip_id, ids) + } + EditCommand::AddClips { entries } => add_clips(state, entries, ids), + EditCommand::AddClipsAutoTrack { entries } => add_clips_auto_track(state, entries, ids), + EditCommand::AddClipsToSeparateAutoTracks { entries } => { + add_clips_to_separate_auto_tracks(state, entries, ids) + } + EditCommand::RegisterMediaAndAddClip { + media, + entry, + auto_track, + } => register_media_and_add_clip(state, media, entry, auto_track, ids), + EditCommand::RegisterMediaAndSwapClip { media, clip_id } => { + register_media_and_swap_clip(state, media, clip_id, false, ids) + } + EditCommand::RegisterMediaAndSwapClipClearingMasks { media, clip_id } => { + register_media_and_swap_clip(state, media, clip_id, true, ids) + } + EditCommand::InsertClips { + track_index, + at_frame, + entries, + } => insert_clips(state, track_index, at_frame, entries, ids), EditCommand::MoveClips { moves } => move_clips(state, moves, ids), EditCommand::DuplicateClips { clip_ids, @@ -557,12 +1182,40 @@ pub fn apply( interpolation, } => set_keyframe_interpolation(state, clip_id, property, frame, interpolation), EditCommand::SetColorGrade { clip_ids, grade } => set_color_grade(state, clip_ids, grade), + EditCommand::ApplyColorMatch { + clip_id, + grade, + input, + } => apply_color_match(state, clip_id, grade, input), + EditCommand::SetLut { clip_ids, lut } => set_lut(state, clip_ids, lut), EditCommand::SetChromaKey { clip_ids, chroma_key, } => set_chroma_key(state, clip_ids, chroma_key), EditCommand::SetMasks { clip_ids, masks } => set_masks(state, clip_ids, masks), EditCommand::SetEffects { clip_ids, effects } => set_effects(state, clip_ids, effects), + EditCommand::SetLoudnessNormalization { + clip_id, + normalization, + } => set_loudness_normalization(state, clip_id, normalization), + EditCommand::SetAudioDenoise { clip_id, denoise } => { + set_audio_denoise(state, clip_id, denoise) + } + EditCommand::ApplyStabilization { clip_id, solution } => { + apply_stabilization(state, clip_id, solution) + } + EditCommand::AdjustStabilization { + clip_id, + strength, + crop_margin, + } => adjust_stabilization(state, clip_id, strength, crop_margin), + EditCommand::ResetStabilization { clip_id } => reset_stabilization(state, clip_id), + EditCommand::SetTransition { + from_clip_id, + to_clip_id, + kind, + duration_frames, + } => set_transition(state, from_clip_id, to_clip_id, kind, duration_frames), EditCommand::RippleDeleteRanges { track_index, ranges, @@ -571,6 +1224,17 @@ pub fn apply( EditCommand::AddTexts { entries } => add_texts(state, entries, ids), EditCommand::AddTextsAutoTrack { entries } => add_texts_auto_track(state, entries, ids), EditCommand::AddCaptions { entries } => add_captions(state, entries, ids), + EditCommand::ApplyCaptionTranslations { changes } => { + apply_caption_translations(state, changes) + } + EditCommand::SaveScriptAssemblyPlan { plan } => save_script_assembly_plan(state, plan), + EditCommand::ApplyScriptAssemblyPlan { plan_id } => { + apply_script_assembly_plan(state, plan_id, ids) + } + EditCommand::SaveVoiceModel { record } => save_voice_model(state, record), + EditCommand::RevokeVoiceModel { voice_model_id } => { + revoke_voice_model(state, voice_model_id) + } EditCommand::Link { clip_ids } => link(state, clip_ids, ids), EditCommand::Unlink { clip_ids } => unlink(state, clip_ids), EditCommand::RemoveTracks { track_indexes } => remove_tracks(state, track_indexes), @@ -603,6 +1267,470 @@ pub fn apply( } } +fn create_nested_sequence( + state: &mut EditorState, + name: String, + timeline: Timeline, + track_index: usize, + start_frame: i32, + duration_frames: i32, + ids: &dyn IdGen, +) -> Result { + let name = name.trim().to_string(); + if name.is_empty() { + return Err(EditError::Invalid( + "nested sequence name must not be empty".into(), + )); + } + if track_index >= state.timeline.tracks.len() { + return Err(EditError::Invalid(format!( + "track index out of range: {track_index}" + ))); + } + if state.timeline.tracks[track_index].kind == ClipType::Audio { + return Err(EditError::Invalid( + "a compound clip requires a visual track".into(), + )); + } + if start_frame < 0 || duration_frames < 1 { + return Err(EditError::Invalid( + "compound timing requires startFrame >= 0 and durationFrames >= 1".into(), + )); + } + if !timeline.nested_sequences.is_empty() { + return Err(EditError::Invalid( + "child timelines must reference the root nested sequence registry".into(), + )); + } + + transact( + state, + "Create Compound Clip", + |affected| format!("Created compound clip {}", affected.join(", ")), + |st| { + let sequence_id = ids.next_id(); + let clip_id = ids.next_id(); + st.timeline.nested_sequences.push(NestedSequence::new( + sequence_id.clone(), + name, + timeline, + )); + ops::clear_region( + &mut st.timeline, + track_index, + start_frame, + start_frame.saturating_add(duration_frames), + false, + ids, + ); + st.timeline.tracks[track_index].clips.push(Clip::new_nested( + clip_id.clone(), + sequence_id, + start_frame, + duration_frames, + )); + ops::sort_clips(&mut st.timeline.tracks[track_index]); + Ok(vec![clip_id]) + }, + ) +} + +fn create_nested_sequence_from_clips( + state: &mut EditorState, + name: String, + clip_ids: Vec, + ids: &dyn IdGen, +) -> Result { + let name = name.trim().to_string(); + if name.is_empty() { + return Err(EditError::Invalid( + "nested sequence name must not be empty".into(), + )); + } + if clip_ids.is_empty() { + return Err(EditError::Invalid( + "at least one clip is required to create a compound".into(), + )); + } + let requested: HashSet = clip_ids.iter().cloned().collect(); + if requested.len() != clip_ids.len() { + return Err(EditError::Invalid( + "compound clip selection contains duplicate ids".into(), + )); + } + // Keep linked A/V partners inside the same edit boundary. Leaving one half + // at root would create a link group spanning independent timelines, which + // child edit commands cannot preserve safely. + let selected = ops::expand_to_link_group(&state.timeline, &requested); + + let mut start_frame = i32::MAX; + let mut end_frame = i32::MIN; + let mut target_track = None; + let mut child = Timeline::new(); + child.fps = state.timeline.fps; + child.width = state.timeline.width; + child.height = state.timeline.height; + child.settings_configured = state.timeline.settings_configured; + for (track_index, track) in state.timeline.tracks.iter().enumerate() { + let clips: Vec = track + .clips + .iter() + .filter(|clip| selected.contains(&clip.id)) + .cloned() + .collect(); + if clips.is_empty() { + continue; + } + if target_track.is_none() && track.kind != ClipType::Audio { + target_track = Some(track_index); + } + for clip in &clips { + start_frame = start_frame.min(clip.start_frame); + end_frame = end_frame.max(clip.end_frame()); + } + let mut child_track = Track::new(ids.next_id(), track.kind); + child_track.muted = track.muted; + child_track.hidden = track.hidden; + child_track.sync_locked = track.sync_locked; + child_track.clips = clips; + child.tracks.push(child_track); + } + let found: usize = child.tracks.iter().map(|track| track.clips.len()).sum(); + if found != selected.len() { + return Err(EditError::Invalid( + "one or more clips selected for the compound no longer exist".into(), + )); + } + let target_track = target_track.ok_or_else(|| { + EditError::Invalid("a compound clip requires at least one visual clip".into()) + })?; + if state.timeline.tracks[target_track] + .clips + .iter() + .any(|clip| { + !selected.contains(&clip.id) + && clip.start_frame < end_frame + && start_frame < clip.end_frame() + }) + { + return Err(EditError::Invalid( + "compound selection span overlaps an unselected clip on its destination track".into(), + )); + } + for track in &mut child.tracks { + for clip in &mut track.clips { + clip.start_frame -= start_frame; + } + } + let duration_frames = end_frame - start_frame; + + transact( + state, + "Create Compound Clip", + |affected| format!("Created compound clip {}", affected.join(", ")), + |st| { + for track in &mut st.timeline.tracks { + track.clips.retain(|clip| !selected.contains(&clip.id)); + } + let sequence_id = ids.next_id(); + let compound_id = ids.next_id(); + st.timeline.nested_sequences.push(NestedSequence::new( + sequence_id.clone(), + name, + child, + )); + ops::clear_region( + &mut st.timeline, + target_track, + start_frame, + end_frame, + false, + ids, + ); + st.timeline.tracks[target_track] + .clips + .push(Clip::new_nested( + compound_id.clone(), + sequence_id, + start_frame, + duration_frames, + )); + ops::sort_clips(&mut st.timeline.tracks[target_track]); + ops::prune_empty_tracks(&mut st.timeline); + Ok(vec![compound_id]) + }, + ) +} + +fn edit_nested_sequence( + state: &mut EditorState, + sequence_id: String, + command: EditCommand, + ids: &dyn IdGen, +) -> Result { + if matches!( + &command, + EditCommand::Undo + | EditCommand::Redo + | EditCommand::CreateNestedSequence { .. } + | EditCommand::CreateNestedSequenceFromClips { .. } + | EditCommand::EditNestedSequence { .. } + | EditCommand::SetNestedSequenceTimeline { .. } + | EditCommand::RenameNestedSequence { .. } + | EditCommand::DissolveNestedSequence { .. } + | EditCommand::CreateFolder { .. } + | EditCommand::MoveToFolder { .. } + | EditCommand::RenameMedia { .. } + | EditCommand::RenameFolder { .. } + | EditCommand::DeleteMedia { .. } + | EditCommand::DeleteFolder { .. } + | EditCommand::SetTimelineSettings { .. } + | EditCommand::SaveScriptAssemblyPlan { .. } + | EditCommand::ApplyScriptAssemblyPlan { .. } + | EditCommand::SaveVoiceModel { .. } + | EditCommand::RevokeVoiceModel { .. } + ) { + return Err(EditError::Invalid( + "nested-sequence, media-library, and project-settings commands must target the root timeline".into(), + )); + } + let child = state + .timeline + .nested_sequences + .iter() + .find(|sequence| sequence.id == sequence_id) + .map(|sequence| sequence.timeline.clone()) + .ok_or_else(|| EditError::Invalid(format!("Nested sequence not found: {sequence_id}")))?; + + transact( + state, + "Edit Compound Clip", + |_| format!("Edited nested sequence {sequence_id}"), + |st| { + // Supply the root registry during the inner transaction so child + // references resolve against the same identities as preview/export. + // The target sequence's stored (pre-edit) contents are blanked in + // this temporary view: `editable` already represents those clips, + // and counting both copies would manufacture duplicate clip ids. + // The enclosing root transaction validates the fully replaced graph + // (including cycles through this sequence) before it can commit. + let mut editable = child; + editable.nested_sequences = st.timeline.nested_sequences.clone(); + editable + .nested_sequences + .iter_mut() + .find(|sequence| sequence.id == sequence_id) + .expect("sequence was resolved before transaction") + .timeline = Timeline::new(); + let mut child_state = EditorState::new(editable, st.manifest.clone()); + let inner = apply(&mut child_state, command, ids)?; + child_state.timeline.nested_sequences.clear(); + let sequence = st + .timeline + .nested_sequences + .iter_mut() + .find(|sequence| sequence.id == sequence_id) + .expect("sequence was resolved before transaction"); + sequence.timeline = child_state.timeline; + st.manifest = child_state.manifest; + Ok(inner.affected_clip_ids) + }, + ) +} + +fn set_nested_sequence_timeline( + state: &mut EditorState, + sequence_id: String, + timeline: Timeline, +) -> Result { + if !timeline.nested_sequences.is_empty() { + return Err(EditError::Invalid( + "child timelines must reference the root nested sequence registry".into(), + )); + } + transact( + state, + "Edit Compound Clip", + |_| format!("Edited nested sequence {sequence_id}"), + |st| { + let sequence = st + .timeline + .nested_sequences + .iter_mut() + .find(|sequence| sequence.id == sequence_id) + .ok_or_else(|| { + EditError::Invalid(format!("Nested sequence not found: {sequence_id}")) + })?; + sequence.timeline = timeline; + Ok(Vec::new()) + }, + ) +} + +fn rename_nested_sequence( + state: &mut EditorState, + sequence_id: String, + name: String, +) -> Result { + let name = name.trim().to_string(); + if name.is_empty() { + return Err(EditError::Invalid( + "nested sequence name must not be empty".into(), + )); + } + transact( + state, + "Rename Compound Clip", + |_| format!("Renamed nested sequence {sequence_id}"), + |st| { + let sequence = st + .timeline + .nested_sequences + .iter_mut() + .find(|sequence| sequence.id == sequence_id) + .ok_or_else(|| { + EditError::Invalid(format!("Nested sequence not found: {sequence_id}")) + })?; + sequence.name = name; + Ok(Vec::new()) + }, + ) +} + +fn dissolve_nested_sequence( + state: &mut EditorState, + clip_id: String, + ids: &dyn IdGen, +) -> Result { + let location = state + .find_clip(&clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + let compound = state.timeline.tracks[location.track_index].clips[location.clip_index].clone(); + let sequence_id = compound + .nested_sequence_id + .as_deref() + .ok_or_else(|| EditError::Invalid(format!("Clip is not a compound clip: {clip_id}")))?; + if (compound.speed - 1.0).abs() > f64::EPSILON || compound.reversed { + return Err(EditError::Invalid( + "retimed or reversed compound clips must be normalized before dissolve".into(), + )); + } + let has_parent_edits = (compound.volume - 1.0).abs() > f64::EPSILON + || compound.fade_in_frames != 0 + || compound.fade_out_frames != 0 + || (compound.opacity - 1.0).abs() > f64::EPSILON + || compound.transform != Transform::default() + || compound.crop != Crop::default() + || compound.link_group_id.is_some() + || compound.caption_group_id.is_some() + || compound.text_content.is_some() + || compound.text_style.is_some() + || compound.opacity_track.is_some() + || compound.position_track.is_some() + || compound.scale_track.is_some() + || compound.rotation_track.is_some() + || compound.crop_track.is_some() + || compound.volume_track.is_some() + || compound.color_grade.is_some() + || compound.chroma_key.is_some() + || !compound.masks.is_empty() + || !compound.effects.is_empty() + || compound.transition_out.is_some(); + if has_parent_edits { + return Err(EditError::Invalid( + "compound clips with parent-level edits must be normalized before dissolve".into(), + )); + } + let child = state + .timeline + .nested_sequences + .iter() + .find(|sequence| sequence.id == sequence_id) + .map(|sequence| sequence.timeline.clone()) + .ok_or_else(|| EditError::Invalid(format!("Nested sequence not found: {sequence_id}")))?; + + transact( + state, + "Dissolve Compound Clip", + |affected| format!("Dissolved compound into {} clip(s)", affected.len()), + |st| { + let source_start = compound.trim_start_frame; + let source_end = source_start.saturating_add(compound.duration_frames); + let mut id_map = HashMap::new(); + let mut link_counts: HashMap = HashMap::new(); + for child_clip in child.tracks.iter().flat_map(|track| &track.clips) { + let visible_start = child_clip.start_frame.max(source_start); + let visible_end = child_clip.end_frame().min(source_end); + if visible_end <= visible_start { + continue; + } + id_map.insert(child_clip.id.clone(), ids.next_id()); + if let Some(group) = &child_clip.link_group_id { + *link_counts.entry(group.clone()).or_default() += 1; + } + } + let mut link_map: HashMap = HashMap::new(); + + ops::clear_region::remove_clip(&mut st.timeline, &clip_id); + let mut affected = Vec::new(); + + for child_track in child.tracks { + let requested = st.timeline.tracks.len(); + let target = ops::insert_track(&mut st.timeline, requested, child_track.kind, ids); + st.timeline.tracks[target].muted = child_track.muted; + st.timeline.tracks[target].hidden = child_track.hidden; + st.timeline.tracks[target].sync_locked = child_track.sync_locked; + + for mut child_clip in child_track.clips { + let visible_start = child_clip.start_frame.max(source_start); + let visible_end = child_clip.end_frame().min(source_end); + if visible_end <= visible_start { + continue; + } + let clipped_left = visible_start - child_clip.start_frame; + let clipped_right = child_clip.end_frame() - visible_end; + let old_id = child_clip.id.clone(); + child_clip.id = id_map + .get(&old_id) + .expect("visible child clip received a replacement id") + .clone(); + child_clip.link_group_id = child_clip.link_group_id.take().and_then(|group| { + (link_counts.get(&group).copied().unwrap_or(0) > 1).then(|| { + link_map + .entry(group) + .or_insert_with(|| ids.next_id()) + .clone() + }) + }); + child_clip.transition_out = + child_clip.transition_out.take().and_then(|mut transition| { + id_map.get(&transition.to_clip_id).map(|to_id| { + transition.from_clip_id = child_clip.id.clone(); + transition.to_clip_id = to_id.clone(); + transition + }) + }); + child_clip.start_frame = compound + .start_frame + .saturating_add(visible_start - source_start); + child_clip.duration_frames = visible_end - visible_start; + child_clip.trim_start_frame = child_clip + .trim_start_frame + .saturating_add((clipped_left as f64 * child_clip.speed).round() as i32); + child_clip.trim_end_frame = child_clip + .trim_end_frame + .saturating_add((clipped_right as f64 * child_clip.speed).round() as i32); + affected.push(child_clip.id.clone()); + st.timeline.tracks[target].clips.push(child_clip); + } + ops::sort_clips(&mut st.timeline.tracks[target]); + } + ops::prune_empty_tracks(&mut st.timeline); + Ok(affected) + }, + ) +} + // MARK: - Transaction helper /// Run `work` inside a transaction: snapshot, mutate, commit-if-changed. `work` @@ -622,6 +1750,11 @@ fn transact( return Err(error); } }; + prune_invalid_transitions(&mut state.timeline); + if let Err(reason) = state.timeline.validate_nested_sequences() { + state.restore(before); + return Err(EditError::Invalid(reason)); + } let after = state.snapshot(); let timeline_changed = before.timeline != after.timeline; let manifest_changed = before.manifest != after.manifest; @@ -640,6 +1773,65 @@ fn transact( )) } +/// Keep transition pair identity aligned with the actual cut graph after every +/// transactional edit. A move/delete/trim must never leave a dormant transition +/// that could later bind to a different neighbor. +fn prune_invalid_transitions(timeline: &mut Timeline) { + for sequence in &mut timeline.nested_sequences { + prune_invalid_transitions(&mut sequence.timeline); + } + for track in &mut timeline.tracks { + if track.kind == ClipType::Audio { + for clip in &mut track.clips { + clip.transition_out = None; + } + continue; + } + let mut order: Vec = (0..track.clips.len()).collect(); + order.sort_by_key(|&index| { + ( + track.clips[index].start_frame, + track.clips[index].id.clone(), + ) + }); + let mut valid: HashMap = HashMap::new(); + for pair in order.windows(2) { + let from = &track.clips[pair[0]]; + let to = &track.clips[pair[1]]; + if from.end_frame() != to.start_frame + || matches!(from.media_type, ClipType::Audio | ClipType::Text) + || matches!(to.media_type, ClipType::Audio | ClipType::Text) + { + continue; + } + valid.insert( + from.id.clone(), + ( + to.id.clone(), + (from.duration_frames.min(to.duration_frames) / 2).max(1), + ), + ); + } + for clip in &mut track.clips { + let Some(transition) = &mut clip.transition_out else { + continue; + }; + let Some((to_id, maximum)) = valid.get(&clip.id) else { + clip.transition_out = None; + continue; + }; + if (!transition.from_clip_id.is_empty() && transition.from_clip_id != clip.id) + || transition.to_clip_id != *to_id + { + clip.transition_out = None; + continue; + } + transition.from_clip_id = clip.id.clone(); + transition.duration_frames = transition.duration_frames.clamp(1, *maximum); + } + } +} + fn result( state: &EditorState, timeline_changed: bool, @@ -815,6 +2007,54 @@ fn add_clips_auto_track( ) } +fn add_clips_to_separate_auto_tracks( + state: &mut EditorState, + entries: Vec, + ids: &dyn IdGen, +) -> Result { + if entries.is_empty() { + return Err(EditError::Invalid( + "Missing or empty 'entries' array".into(), + )); + } + for (index, entry) in entries.iter().enumerate() { + validate_auto_track_entry(entry, index)?; + } + transact( + state, + "Import Stems To Tracks", + |added| { + format!( + "Imported {} aligned stem(s): {}", + added.len(), + added.join(", ") + ) + }, + |current| { + let mut placed = Vec::with_capacity(entries.len()); + for entry in &entries { + let kind = if entry.source_clip_type == ClipType::Audio { + ClipType::Audio + } else { + ClipType::Video + }; + let at = current.timeline.tracks.len(); + let track_index = ops::insert_track(&mut current.timeline, at, kind, ids); + let mut entry = entry.clone(); + entry.track_index = track_index; + placed.extend(ops::place_clip( + &mut current.timeline, + &entry.to_spec(), + track_index, + None, + ids, + )); + } + Ok(placed) + }, + ) +} + fn insert_track_cmd( state: &mut EditorState, kind: ClipType, @@ -904,6 +2144,137 @@ fn swap_clips(state: &mut EditorState, a: String, b: String) -> Result Result<(), EditError> { + if media.id.trim().is_empty() { + return Err(EditError::Invalid( + "generated media id must not be empty".into(), + )); + } + if state + .manifest + .entries + .iter() + .any(|entry| entry.id == media.id) + { + return Err(EditError::Invalid(format!( + "Media already exists: {}", + media.id + ))); + } + if state + .manifest + .entries + .iter() + .any(|entry| entry.source == media.source) + { + return Err(EditError::Invalid( + "generated media source is already registered".into(), + )); + } + Ok(()) +} + +fn register_media_and_add_clip( + state: &mut EditorState, + media: MediaManifestEntry, + entry: ClipEntry, + auto_track: bool, + ids: &dyn IdGen, +) -> Result { + validate_registered_media(state, &media)?; + if media.kind != entry.media_type || media.kind != entry.source_clip_type { + return Err(EditError::Invalid( + "generated media type must match the placed clip type".into(), + )); + } + if entry.media_ref != media.id { + return Err(EditError::Invalid( + "placed clip must reference the generated media id".into(), + )); + } + + // Run the existing placement command against a disposable state so all of + // its overlap, track, duration, and manifest validation remains the single + // source of truth. Only its resulting document is copied into the one outer + // transaction; its temporary undo/version bookkeeping is discarded. + let mut candidate = state.clone(); + candidate.manifest.entries.push(media); + let placement = if auto_track { + add_clips_auto_track(&mut candidate, vec![entry], ids)? + } else { + add_clips(&mut candidate, vec![entry], ids)? + }; + let timeline = candidate.timeline; + let manifest = candidate.manifest; + let affected = placement.affected_clip_ids; + + transact( + state, + "Add Motion Graphic", + |ids| format!("Added motion graphic: {}", ids.join(", ")), + move |current| { + current.timeline = timeline; + current.manifest = manifest; + Ok(affected) + }, + ) +} + +fn register_media_and_swap_clip( + state: &mut EditorState, + media: MediaManifestEntry, + clip_id: String, + clear_masks: bool, + ids: &dyn IdGen, +) -> Result { + validate_registered_media(state, &media)?; + let mut candidate = state.clone(); + let media_ref = media.id.clone(); + candidate.manifest.entries.push(media); + let seed_clip_id = clip_id.clone(); + let replacement = swap_media(&mut candidate, clip_id, media_ref)?; + if clear_masks { + let clip = candidate + .timeline + .tracks + .iter_mut() + .flat_map(|track| track.clips.iter_mut()) + .find(|clip| clip.id == seed_clip_id) + .ok_or_else(|| EditError::Invalid("replacement clip disappeared".into()))?; + clip.masks.clear(); + } + let timeline = candidate.timeline; + let manifest = candidate.manifest; + let affected = replacement.affected_clip_ids; + + // `ids` is intentionally accepted to keep the external-render commands on + // the same command signature; SwapMedia itself does not mint ids. + let _ = ids; + transact( + state, + if clear_masks { + "Remove Masked Object" + } else { + "Edit Motion Graphic" + }, + move |ids| { + if clear_masks { + format!("Removed masked object: {}", ids.join(", ")) + } else { + format!("Edited motion graphic: {}", ids.join(", ")) + } + }, + move |current| { + current.timeline = timeline; + current.manifest = manifest; + Ok(affected) + }, + ) +} + fn insert_clips( state: &mut EditorState, track_index: usize, @@ -928,7 +2299,7 @@ fn insert_clips( } let target_type = state.timeline.tracks[track_index].kind; for (i, e) in entries.iter().enumerate() { - if !e.source_clip_type.is_compatible(target_type) { + if !e.media_type.is_compatible(target_type) { return Err(EditError::Invalid(format!( "entries[{i}]: asset type is not compatible with the target track" ))); @@ -1189,8 +2560,22 @@ fn set_clip_properties( )); } for id in &clip_ids { - if state.find_clip(id).is_none() { - return Err(EditError::Invalid(format!("Clip not found: {id}"))); + let location = state + .find_clip(id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {id}")))?; + let clip = &state.timeline.tracks[location.track_index].clips[location.clip_index]; + if clip.nested_sequence_id.is_some() + && (props + .speed + .is_some_and(|speed| (speed - 1.0).abs() > f64::EPSILON) + || props.reversed == Some(true) + || props.crop.is_some_and(|crop| crop != Crop::default()) + || props.text_content.is_some() + || props.text_style.is_some()) + { + return Err(EditError::Invalid(format!( + "compound clip {id} does not support retime, reverse, crop, or text properties" + ))); } } if let Some(df) = props.duration_frames { @@ -1223,111 +2608,520 @@ fn set_clip_properties( for id in &clip_ids { apply_property_changes(&mut st.timeline, id, &props, false); } - for pid in &partners { - let is_text = st - .find_clip(pid) - .map(|l| st.timeline.tracks[l.track_index].clips[l.clip_index].media_type) - == Some(ClipType::Text); - // Partners receive only timing (and drop it when text). - let partner_props = ClipProperties { - duration_frames: if is_text { None } else { props.duration_frames }, - trim_start_frame: if is_text { - None - } else { - props.trim_start_frame - }, - trim_end_frame: if is_text { None } else { props.trim_end_frame }, - speed: if is_text { None } else { props.speed }, - ..Default::default() + for pid in &partners { + let is_text = st + .find_clip(pid) + .map(|l| st.timeline.tracks[l.track_index].clips[l.clip_index].media_type) + == Some(ClipType::Text); + // Partners receive only timing (and drop it when text). + let partner_props = ClipProperties { + duration_frames: if is_text { None } else { props.duration_frames }, + trim_start_frame: if is_text { + None + } else { + props.trim_start_frame + }, + trim_end_frame: if is_text { None } else { props.trim_end_frame }, + speed: if is_text { None } else { props.speed }, + ..Default::default() + }; + apply_property_changes(&mut st.timeline, pid, &partner_props, true); + } + Ok(clip_ids.clone()) + }, + ) +} + +/// Apply a property bundle to one clip in place. `partner` marks the call as a +/// linked-partner propagation (only timing fields are set then). 1:1 port of +/// `applyPropertyChanges`. +fn apply_property_changes( + timeline: &mut Timeline, + clip_id: &str, + props: &ClipProperties, + _partner: bool, +) { + let Some((ti, ci)) = find(timeline, clip_id) else { + return; + }; + let clip = &mut timeline.tracks[ti].clips[ci]; + + if props.duration_frames.is_some() + || props.trim_start_frame.is_some() + || props.trim_end_frame.is_some() + || props.speed.is_some() + || props.reversed.is_some() + { + clip.loudness_normalization = None; + } + + if let Some(v) = props.duration_frames { + clip.duration_frames = v; + clip.clamp_keyframes_to_duration(); + clip.clamp_fades_to_duration(); + } + if let Some(v) = props.trim_start_frame { + clip.trim_start_frame = v; + } + if let Some(v) = props.trim_end_frame { + clip.trim_end_frame = v; + } + if let Some(v) = props.speed { + // When no explicit duration is given, recompute duration so the same + // source span plays at the new speed (mirrors applyPropertyChanges). + if props.duration_frames.is_none() && v > 0.0 { + let source_consumed = clip.duration_frames as f64 * clip.speed; + clip.duration_frames = (1).max((source_consumed / v).round() as i32); + clip.clamp_keyframes_to_duration(); + clip.clamp_fades_to_duration(); + } + clip.speed = v; + } + // Setting a scalar clears the matching keyframe track. + if let Some(v) = props.volume { + clip.volume = v; + clip.volume_track = None; + } + if let Some(v) = props.opacity { + clip.opacity = v; + clip.opacity_track = None; + } + if let Some(t) = props.transform { + clip.transform = t; + } + if let Some(c) = props.crop { + clip.crop = c; + clip.crop_track = None; + } + if let Some(v) = props.fade_in_frames { + clip.fade_in_frames = v.max(0); + clip.clamp_fades_to_duration(); + } + if let Some(v) = props.fade_out_frames { + clip.fade_out_frames = v.max(0); + clip.clamp_fades_to_duration(); + } + if let Some(i) = props.fade_in_interpolation { + clip.fade_in_interpolation = i; + } + if let Some(i) = props.fade_out_interpolation { + clip.fade_out_interpolation = i; + } + if let Some(f) = props.flip_horizontal { + clip.transform.flip_horizontal = f; + } + if let Some(f) = props.flip_vertical { + clip.transform.flip_vertical = f; + } + if let Some(reversed) = props.reversed { + clip.reversed = reversed; + } + if let Some(c) = &props.text_content { + clip.text_content = Some(c.clone()); + clip.caption_translation_input = None; + } + if let Some(s) = &props.text_style { + clip.text_style = Some(s.clone()); + } +} + +fn apply_caption_translations( + state: &mut EditorState, + changes: Vec, +) -> Result { + if changes.is_empty() { + return Err(EditError::Invalid( + "Missing or empty caption translation changes".into(), + )); + } + let mut seen = HashSet::new(); + for change in &changes { + if !seen.insert(change.clip_id.as_str()) { + return Err(EditError::Invalid(format!( + "Duplicate caption clip: {}", + change.clip_id + ))); + } + if change.translated_text.trim().is_empty() { + return Err(EditError::Invalid(format!( + "Translated text is empty for clip {}", + change.clip_id + ))); + } + let location = state + .find_clip(&change.clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {}", change.clip_id)))?; + let clip = &state.timeline.tracks[location.track_index].clips[location.clip_index]; + if clip.media_type != ClipType::Text || clip.caption_group_id.is_none() { + return Err(EditError::Invalid(format!( + "Clip is not a caption: {}", + change.clip_id + ))); + } + if clip.text_content.as_deref() != Some(change.expected_source_text.as_str()) { + return Err(EditError::Invalid(format!( + "Caption text changed during review: {}", + change.clip_id + ))); + } + if change.input.source_text != change.expected_source_text { + return Err(EditError::Invalid(format!( + "Caption provenance does not match source text: {}", + change.clip_id + ))); + } + } + let n = changes.len(); + transact( + state, + "Translate Captions", + move |_| format!("Translated {n} caption(s)"), + move |st| { + for change in &changes { + let (track_index, clip_index) = find(&st.timeline, &change.clip_id) + .expect("caption translations were prevalidated"); + let clip = &mut st.timeline.tracks[track_index].clips[clip_index]; + clip.text_content = Some(change.translated_text.clone()); + clip.caption_translation_input = Some(change.input.clone()); + } + Ok(changes + .iter() + .map(|change| change.clip_id.clone()) + .collect()) + }, + ) +} + +fn validate_script_assembly_plan(plan: &ScriptAssemblyPlan) -> Result<(), EditError> { + if plan.id.is_empty() + || plan.id.len() > 128 + || plan.plan_hash.len() != 64 + || !plan.plan_hash.bytes().all(|byte| byte.is_ascii_hexdigit()) + || plan.planner.trim().is_empty() + || plan.planner.len() > 128 + || plan.planner_version == 0 + || plan.start_frame < 0 + || plan.segments.is_empty() + || plan.segments.len() > 100 + { + return Err(EditError::Invalid("invalid script assembly plan".into())); + } + for (index, segment) in plan.segments.iter().enumerate() { + if segment.script.trim().is_empty() + || segment.script.len() > 20_000 + || segment.media_ref.is_empty() + || segment.media_ref.len() > 256 + || segment + .narration_media_ref + .as_ref() + .is_some_and(|value| value.is_empty() || value.len() > 256) + || !(1..=36_000).contains(&segment.duration_frames) + { + return Err(EditError::Invalid(format!( + "invalid script assembly segment {index}" + ))); + } + if segment.transition.is_some() + && (index + 1 == plan.segments.len() + || segment.duration_frames < 2 + || plan.segments[index + 1].duration_frames < 2) + { + return Err(EditError::Invalid(format!( + "segment {index} transition requires a following segment with at least two frames" + ))); + } + } + Ok(()) +} + +fn save_script_assembly_plan( + state: &mut EditorState, + plan: ScriptAssemblyPlan, +) -> Result { + validate_script_assembly_plan(&plan)?; + let plan_id = plan.id.clone(); + transact( + state, + "Plan Script Video", + |_| format!("Saved script assembly plan {plan_id}"), + move |st| { + if let Some(existing) = st + .timeline + .script_assembly_plans + .iter_mut() + .find(|existing| existing.id == plan.id) + { + *existing = plan.clone(); + } else { + if st.timeline.script_assembly_plans.len() >= 20 { + st.timeline.script_assembly_plans.remove(0); + } + st.timeline.script_assembly_plans.push(plan.clone()); + } + Ok(Vec::new()) + }, + ) +} + +fn apply_script_assembly_plan( + state: &mut EditorState, + plan_id: String, + ids: &dyn IdGen, +) -> Result { + let plan = state + .timeline + .script_assembly_plans + .iter() + .find(|plan| plan.id == plan_id) + .cloned() + .ok_or_else(|| EditError::Invalid(format!("script assembly plan not found: {plan_id}")))?; + validate_script_assembly_plan(&plan)?; + let fps = state.timeline.fps.max(1) as f64; + for (index, segment) in plan.segments.iter().enumerate() { + let visual = state + .manifest + .entries + .iter() + .find(|entry| entry.id == segment.media_ref) + .ok_or_else(|| { + EditError::Invalid(format!( + "script segment {index} visual media not found: {}", + segment.media_ref + )) + })?; + if !matches!( + visual.kind, + ClipType::Image | ClipType::Video | ClipType::Lottie + ) { + return Err(EditError::Invalid(format!( + "script segment {index} media must be visual" + ))); + } + if visual.kind == ClipType::Video + && visual.duration > 0.0 + && (visual.duration * fps).round() as i32 + 1 < segment.duration_frames + { + return Err(EditError::Invalid(format!( + "script segment {index} is longer than its video source" + ))); + } + if let Some(narration_ref) = &segment.narration_media_ref { + let narration = state + .manifest + .entries + .iter() + .find(|entry| entry.id == *narration_ref) + .ok_or_else(|| { + EditError::Invalid(format!( + "script segment {index} narration media not found: {narration_ref}" + )) + })?; + if !narration + .has_audio + .unwrap_or(narration.kind == ClipType::Audio) + || !matches!(narration.kind, ClipType::Audio | ClipType::Video) + { + return Err(EditError::Invalid(format!( + "script segment {index} narration must contain audio" + ))); + } + let narration_frames = (narration.duration * fps).round() as i32; + if narration.duration > 0.0 && (narration_frames - segment.duration_frames).abs() > 1 { + return Err(EditError::Invalid(format!( + "script segment {index} narration duration must match within one frame" + ))); + } + } + } + + transact( + state, + "Build Script Video", + |affected| format!("Built script video with {} clip(s)", affected.len()), + move |st| { + let visual_track_id = ids.next_id(); + let mut visual_track = Track::new(visual_track_id, ClipType::Video); + let mut narration_track = plan + .segments + .iter() + .any(|segment| segment.narration_media_ref.is_some()) + .then(|| Track::new(ids.next_id(), ClipType::Audio)); + let mut cursor = plan.start_frame; + let mut affected = Vec::new(); + for segment in &plan.segments { + let media = st + .manifest + .entries + .iter() + .find(|entry| entry.id == segment.media_ref) + .expect("script assembly media was prevalidated"); + let mut clip = Clip::new( + ids.next_id(), + segment.media_ref.clone(), + cursor, + segment.duration_frames, + ); + clip.media_type = media.kind; + clip.source_clip_type = media.kind; + if segment.narration_media_ref.is_some() { + clip.volume = 0.0; + } + affected.push(clip.id.clone()); + visual_track.clips.push(clip); + if let (Some(track), Some(narration_ref)) = + (&mut narration_track, &segment.narration_media_ref) + { + let narration = st + .manifest + .entries + .iter() + .find(|entry| entry.id == *narration_ref) + .expect("script narration was prevalidated"); + let mut clip = Clip::new( + ids.next_id(), + narration_ref.clone(), + cursor, + segment.duration_frames, + ); + clip.media_type = ClipType::Audio; + clip.source_clip_type = narration.kind; + affected.push(clip.id.clone()); + track.clips.push(clip); + } + cursor = cursor.saturating_add(segment.duration_frames); + } + for index in 0..visual_track.clips.len().saturating_sub(1) { + let Some(kind) = plan.segments[index].transition else { + continue; }; - apply_property_changes(&mut st.timeline, pid, &partner_props, true); + let from = &visual_track.clips[index]; + let to = &visual_track.clips[index + 1]; + let duration_frames = 12 + .min(from.duration_frames / 2) + .min(to.duration_frames / 2) + .max(1); + visual_track.clips[index].transition_out = Some(Transition { + from_clip_id: from.id.clone(), + to_clip_id: to.id.clone(), + kind, + duration_frames, + }); } - Ok(clip_ids.clone()) + st.timeline.tracks.insert(0, visual_track); + if let Some(track) = narration_track { + st.timeline.tracks.push(track); + } + Ok(affected) }, ) } -/// Apply a property bundle to one clip in place. `partner` marks the call as a -/// linked-partner propagation (only timing fields are set then). 1:1 port of -/// `applyPropertyChanges`. -fn apply_property_changes( - timeline: &mut Timeline, - clip_id: &str, - props: &ClipProperties, - _partner: bool, -) { - let Some((ti, ci)) = find(timeline, clip_id) else { - return; - }; - let clip = &mut timeline.tracks[ti].clips[ci]; +fn valid_sha256(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} - if let Some(v) = props.duration_frames { - clip.duration_frames = v; - clip.clamp_keyframes_to_duration(); - clip.clamp_fades_to_duration(); - } - if let Some(v) = props.trim_start_frame { - clip.trim_start_frame = v; - } - if let Some(v) = props.trim_end_frame { - clip.trim_end_frame = v; - } - if let Some(v) = props.speed { - // When no explicit duration is given, recompute duration so the same - // source span plays at the new speed (mirrors applyPropertyChanges). - if props.duration_frames.is_none() && v > 0.0 { - let source_consumed = clip.duration_frames as f64 * clip.speed; - clip.duration_frames = (1).max((source_consumed / v).round() as i32); - clip.clamp_keyframes_to_duration(); - clip.clamp_fades_to_duration(); - } - clip.speed = v; - } - // Setting a scalar clears the matching keyframe track. - if let Some(v) = props.volume { - clip.volume = v; - clip.volume_track = None; - } - if let Some(v) = props.opacity { - clip.opacity = v; - clip.opacity_track = None; - } - if let Some(t) = props.transform { - clip.transform = t; - } - if let Some(c) = props.crop { - clip.crop = c; - clip.crop_track = None; - } - if let Some(v) = props.fade_in_frames { - clip.fade_in_frames = v.max(0); - clip.clamp_fades_to_duration(); - } - if let Some(v) = props.fade_out_frames { - clip.fade_out_frames = v.max(0); - clip.clamp_fades_to_duration(); - } - if let Some(i) = props.fade_in_interpolation { - clip.fade_in_interpolation = i; - } - if let Some(i) = props.fade_out_interpolation { - clip.fade_out_interpolation = i; - } - if let Some(f) = props.flip_horizontal { - clip.transform.flip_horizontal = f; +fn validate_voice_model(record: &VoiceModelRecord) -> Result<(), EditError> { + let valid_token = |value: &str, max: usize| { + !value.trim().is_empty() + && value.len() <= max + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.:/".contains(&byte)) + }; + if !valid_token(&record.id, 128) + || !valid_token(&record.provider, 64) + || record.provider_voice_id.is_empty() + || record.provider_voice_id.len() > 256 + || !record + .provider_voice_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + || !valid_token(&record.model, 256) + || !valid_token(&record.consent_id, 256) + || !valid_token(&record.source_audio_asset_id, 256) + || !valid_sha256(&record.source_audio_sha256) + || !valid_sha256(&record.request_hash) + || record.voice_name.trim().is_empty() + || record.voice_name.len() > 128 + { + return Err(EditError::Invalid("invalid cloned voice metadata".into())); } - if let Some(f) = props.flip_vertical { - clip.transform.flip_vertical = f; + Ok(()) +} + +fn save_voice_model( + state: &mut EditorState, + record: VoiceModelRecord, +) -> Result { + validate_voice_model(&record)?; + let source = state + .manifest + .entries + .iter() + .find(|entry| entry.id == record.source_audio_asset_id) + .ok_or_else(|| EditError::Invalid("voice reference audio does not exist".into()))?; + if source.kind != ClipType::Audio || !source.has_audio.unwrap_or(true) { + return Err(EditError::Invalid( + "voice reference must be an audio asset".into(), + )); } - if let Some(reversed) = props.reversed { - clip.reversed = reversed; + if state.timeline.voice_models.iter().any(|existing| { + existing.id == record.id + || (existing.provider == record.provider + && existing.provider_voice_id == record.provider_voice_id) + }) { + return Err(EditError::Invalid( + "provider voice identity is already registered".into(), + )); } - if let Some(c) = &props.text_content { - clip.text_content = Some(c.clone()); + if state.timeline.voice_models.len() >= 100 { + return Err(EditError::Invalid( + "project voice model limit reached".into(), + )); } - if let Some(s) = &props.text_style { - clip.text_style = Some(s.clone()); + let record_id = record.id.clone(); + state.timeline.voice_models.push(record); + state.commit_irreversible(); + Ok(result( + state, + true, + false, + "Enroll Voice Clone", + Vec::new(), + &format!("Enrolled voice clone {record_id}"), + )) +} + +fn revoke_voice_model( + state: &mut EditorState, + voice_model_id: String, +) -> Result { + let record = state + .timeline + .voice_models + .iter_mut() + .find(|record| record.id == voice_model_id) + .ok_or_else(|| EditError::Invalid("voice model not found".into()))?; + if record.revoked { + return Ok(result( + state, + false, + false, + "Revoke Voice Clone", + Vec::new(), + "Voice clone was already revoked", + )); } + record.revoked = true; + state.commit_irreversible(); + Ok(result( + state, + true, + false, + "Revoke Voice Clone", + Vec::new(), + &format!("Revoked voice clone {voice_model_id}"), + )) } fn set_keyframes( @@ -1336,8 +3130,17 @@ fn set_keyframes( property: KeyframeProperty, payload: KeyframePayload, ) -> Result { - if state.find_clip(&clip_id).is_none() { - return Err(EditError::Invalid(format!("Clip not found: {clip_id}"))); + let location = state + .find_clip(&clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + if property == KeyframeProperty::Crop + && state.timeline.tracks[location.track_index].clips[location.clip_index] + .nested_sequence_id + .is_some() + { + return Err(EditError::Invalid( + "compound clips do not support crop keyframes".into(), + )); } // Type/property agreement check. let ok = matches!( @@ -1398,6 +3201,11 @@ fn stamp_keyframe( .find_clip(&clip_id) .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; let clip = &state.timeline.tracks[loc.track_index].clips[loc.clip_index]; + if property == KeyframeProperty::Crop && clip.nested_sequence_id.is_some() { + return Err(EditError::Invalid( + "compound clips do not support crop keyframes".into(), + )); + } if !clip.contains(frame) { return Err(EditError::Invalid(format!( "Frame {frame} is outside clip range ({}..{})", @@ -1481,6 +3289,11 @@ fn upsert_keyframe( .find_clip(&clip_id) .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; let clip = &state.timeline.tracks[loc.track_index].clips[loc.clip_index]; + if property == KeyframeProperty::Crop && clip.nested_sequence_id.is_some() { + return Err(EditError::Invalid( + "compound clips do not support crop keyframes".into(), + )); + } if !clip.contains(frame) { return Err(EditError::Invalid(format!( "Frame {frame} is outside clip range ({}..{})", @@ -1791,78 +3604,433 @@ fn set_keyframe_interpolation( // inside the shared `withTimelineSwap` transaction (snapshot -> mutate -> // commit-if-changed + version bump), so undo/redo and versioning come for free. -/// Validate that `clip_ids` is non-empty and every id resolves, then run `mutate` -/// for each clip inside one transaction. Shared by the four effect setters. -fn set_clip_effect_field( +/// Validate that `clip_ids` is non-empty and every id resolves, then run `mutate` +/// for each clip inside one transaction. Shared by the four effect setters. +fn set_clip_effect_field( + state: &mut EditorState, + clip_ids: Vec, + action_name: &'static str, + mutate: impl Fn(&mut opentake_domain::Clip), +) -> Result { + if clip_ids.is_empty() { + return Err(EditError::Invalid( + "Missing or empty 'clipIds' array".into(), + )); + } + for id in &clip_ids { + if state.find_clip(id).is_none() { + return Err(EditError::Invalid(format!("Clip not found: {id}"))); + } + } + let n = clip_ids.len(); + transact( + state, + action_name, + move |_| format!("Updated {n} clip(s)"), + move |st| { + for id in &clip_ids { + if let Some((ti, ci)) = find(&st.timeline, id) { + mutate(&mut st.timeline.tracks[ti].clips[ci]); + } + } + Ok(clip_ids.clone()) + }, + ) +} + +fn reject_compound_effect_targets( + state: &EditorState, + clip_ids: &[String], + adding_effect: bool, +) -> Result<(), EditError> { + if !adding_effect { + return Ok(()); + } + for id in clip_ids { + let location = state + .find_clip(id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {id}")))?; + if state.timeline.tracks[location.track_index].clips[location.clip_index] + .nested_sequence_id + .is_some() + { + return Err(EditError::Invalid(format!( + "compound clip {id} does not support direct pixel effects" + ))); + } + } + Ok(()) +} + +fn set_color_grade( + state: &mut EditorState, + clip_ids: Vec, + grade: Option, +) -> Result { + if let Some(grade) = grade { + grade + .validate() + .map_err(|error| EditError::Invalid(format!("invalid color grade: {error}")))?; + } + reject_compound_effect_targets(state, &clip_ids, grade.is_some())?; + set_clip_effect_field(state, clip_ids, "Set Color Grade", move |clip| { + clip.color_grade = grade; + clip.color_match_input = None; + }) +} + +fn apply_color_match( + state: &mut EditorState, + clip_id: String, + grade: ColorGrade, + input: ColorMatchInput, +) -> Result { + grade + .validate() + .map_err(|error| EditError::Invalid(format!("invalid color match grade: {error}")))?; + reject_compound_effect_targets(state, std::slice::from_ref(&clip_id), true)?; + set_clip_effect_field(state, vec![clip_id], "Match Color", move |clip| { + clip.color_grade = Some(grade); + clip.color_match_input = Some(input.clone()); + }) +} + +fn set_lut( + state: &mut EditorState, + clip_ids: Vec, + lut: Option, +) -> Result { + if let Some(reference) = &lut { + reference + .validate() + .map_err(|error| EditError::Invalid(format!("invalid LUT reference: {error}")))?; + } + reject_compound_effect_targets(state, &clip_ids, lut.is_some())?; + set_clip_effect_field(state, clip_ids, "Set LUT", move |clip| { + clip.lut = lut.clone(); + }) +} + +fn set_chroma_key( + state: &mut EditorState, + clip_ids: Vec, + chroma_key: Option, +) -> Result { + reject_compound_effect_targets(state, &clip_ids, chroma_key.is_some())?; + set_clip_effect_field(state, clip_ids, "Set Chroma Key", move |clip| { + clip.chroma_key = chroma_key; + }) +} + +fn set_masks( + state: &mut EditorState, + clip_ids: Vec, + masks: Vec, +) -> Result { + if masks.len() > MAX_MASKS_PER_CLIP { + return Err(EditError::Invalid(format!( + "a clip supports at most {MAX_MASKS_PER_CLIP} masks" + ))); + } + for (index, mask) in masks.iter().enumerate() { + if let MaskShape::Poly { points } = &mask.shape { + if points.len() < 3 || points.len() > MAX_POLYGON_MASK_POINTS { + return Err(EditError::Invalid(format!( + "mask {index} polygon must contain 3..={MAX_POLYGON_MASK_POINTS} points" + ))); + } + } + } + reject_compound_effect_targets(state, &clip_ids, !masks.is_empty())?; + set_clip_effect_field(state, clip_ids, "Set Masks", move |clip| { + clip.masks = masks.clone(); + }) +} + +fn set_effects( + state: &mut EditorState, + clip_ids: Vec, + effects: Vec, +) -> Result { + opentake_domain::validate_effect_chain(&effects) + .map_err(|error| EditError::Invalid(error.to_string()))?; + reject_compound_effect_targets(state, &clip_ids, !effects.is_empty())?; + set_clip_effect_field(state, clip_ids, "Set Effects", move |clip| { + clip.effects = effects.clone(); + }) +} + +fn set_loudness_normalization( + state: &mut EditorState, + clip_id: String, + normalization: Option, +) -> Result { + let location = state + .find_clip(&clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + let clip = &state.timeline.tracks[location.track_index].clips[location.clip_index]; + if !matches!(clip.media_type, ClipType::Audio | ClipType::Video) + || clip.nested_sequence_id.is_some() + { + return Err(EditError::Invalid( + "loudness normalization requires an ordinary audio-bearing clip".to_string(), + )); + } + if let Some(value) = normalization { + value.validate().map_err(|error| { + EditError::Invalid(format!("invalid loudness normalization: {error}")) + })?; + } + transact( + state, + if normalization.is_some() { + "Normalize Loudness" + } else { + "Reset Loudness" + }, + |_| "Updated clip loudness".to_string(), + move |st| { + let (track_index, clip_index) = find(&st.timeline, &clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + st.timeline.tracks[track_index].clips[clip_index].loudness_normalization = + normalization; + Ok(vec![clip_id.clone()]) + }, + ) +} + +fn set_audio_denoise( state: &mut EditorState, - clip_ids: Vec, - action_name: &'static str, - mutate: impl Fn(&mut opentake_domain::Clip), + clip_id: String, + denoise: Option, ) -> Result { - if clip_ids.is_empty() { + let location = state + .find_clip(&clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + let clip = &state.timeline.tracks[location.track_index].clips[location.clip_index]; + if !matches!(clip.media_type, ClipType::Audio | ClipType::Video) + || clip.nested_sequence_id.is_some() + { return Err(EditError::Invalid( - "Missing or empty 'clipIds' array".into(), + "audio denoise requires an ordinary audio-bearing clip".to_string(), )); } - for id in &clip_ids { - if state.find_clip(id).is_none() { - return Err(EditError::Invalid(format!("Clip not found: {id}"))); - } + if let Some(value) = denoise { + value + .validate() + .map_err(|error| EditError::Invalid(format!("invalid audio denoise: {error}")))?; } - let n = clip_ids.len(); transact( state, - action_name, - move |_| format!("Updated {n} clip(s)"), + if denoise.is_some() { + "Apply Audio Denoise" + } else { + "Reset Audio Denoise" + }, + |_| "Updated clip audio denoise".to_string(), move |st| { - for id in &clip_ids { - if let Some((ti, ci)) = find(&st.timeline, id) { - mutate(&mut st.timeline.tracks[ti].clips[ci]); - } - } - Ok(clip_ids.clone()) + let (track_index, clip_index) = find(&st.timeline, &clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + st.timeline.tracks[track_index].clips[clip_index].audio_denoise = denoise; + Ok(vec![clip_id.clone()]) }, ) } -fn set_color_grade( +fn stabilization_clip<'a>(state: &'a EditorState, clip_id: &str) -> Result<&'a Clip, EditError> { + let location = state + .find_clip(clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {clip_id}")))?; + let clip = &state.timeline.tracks[location.track_index].clips[location.clip_index]; + if clip.media_type != ClipType::Video || clip.nested_sequence_id.is_some() { + return Err(EditError::Invalid(format!( + "stabilization requires an ordinary video clip: {clip_id}" + ))); + } + Ok(clip) +} + +fn apply_stabilization( state: &mut EditorState, - clip_ids: Vec, - grade: Option, + clip_id: String, + solution: StabilizationTrack, ) -> Result { - set_clip_effect_field(state, clip_ids, "Set Color Grade", move |clip| { - clip.color_grade = grade; + solution.validate().map_err(EditError::Invalid)?; + let clip = stabilization_clip(state, &clip_id)?; + if solution.source_identity != clip.media_ref { + return Err(EditError::Invalid(format!( + "stabilization source identity {} does not match clip source {}", + solution.source_identity, clip.media_ref + ))); + } + set_clip_effect_field(state, vec![clip_id], "Apply Stabilization", move |clip| { + clip.stabilization = Some(solution.clone()); }) } -fn set_chroma_key( +fn adjust_stabilization( state: &mut EditorState, - clip_ids: Vec, - chroma_key: Option, + clip_id: String, + strength: Option, + crop_margin: Option, ) -> Result { - set_clip_effect_field(state, clip_ids, "Set Chroma Key", move |clip| { - clip.chroma_key = chroma_key; + if strength.is_none() && crop_margin.is_none() { + return Err(EditError::Invalid( + "strength or cropMargin is required".to_string(), + )); + } + if strength.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) { + return Err(EditError::Invalid( + "stabilization strength must be finite and within 0..=1".to_string(), + )); + } + if crop_margin.is_some_and(|value| !value.is_finite() || !(0.0..=0.5).contains(&value)) { + return Err(EditError::Invalid( + "stabilization crop margin must be finite and within 0..=0.5".to_string(), + )); + } + let clip = stabilization_clip(state, &clip_id)?; + if clip.stabilization.is_none() { + return Err(EditError::Invalid(format!( + "Clip has no stabilization analysis: {clip_id}" + ))); + } + set_clip_effect_field(state, vec![clip_id], "Adjust Stabilization", move |clip| { + if let Some(solution) = &mut clip.stabilization { + if let Some(value) = strength { + solution.strength = value; + } + if let Some(value) = crop_margin { + solution.crop_margin = value; + } + } }) } -fn set_masks( - state: &mut EditorState, - clip_ids: Vec, - masks: Vec, -) -> Result { - set_clip_effect_field(state, clip_ids, "Set Masks", move |clip| { - clip.masks = masks.clone(); +fn reset_stabilization(state: &mut EditorState, clip_id: String) -> Result { + stabilization_clip(state, &clip_id)?; + set_clip_effect_field(state, vec![clip_id], "Reset Stabilization", |clip| { + clip.stabilization = None; }) } -fn set_effects( +fn set_transition( state: &mut EditorState, - clip_ids: Vec, - effects: Vec, + from_clip_id: String, + to_clip_id: String, + kind: Option, + duration_frames: i32, ) -> Result { - set_clip_effect_field(state, clip_ids, "Set Effects", move |clip| { - clip.effects = effects.clone(); - }) + let from_location = state + .find_clip(&from_clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {from_clip_id}")))?; + + if kind.is_some() + && state.timeline.tracks[from_location.track_index].clips[from_location.clip_index] + .nested_sequence_id + .is_some() + { + return Err(EditError::Invalid( + "compound clips do not support direct transitions".into(), + )); + } + + // Clearing is allowed even after the pair stopped being adjacent so stale + // metadata can always be removed safely. Pair identity must still match. + if kind.is_none() { + return transact( + state, + "Remove Transition", + |_| "Removed transition".to_string(), + |st| { + let clip = &mut st.timeline.tracks[from_location.track_index].clips + [from_location.clip_index]; + if clip.transition_out.as_ref().is_some_and(|transition| { + (transition.from_clip_id.is_empty() || transition.from_clip_id == from_clip_id) + && transition.to_clip_id == to_clip_id + }) { + clip.transition_out = None; + } + Ok(vec![from_clip_id.clone(), to_clip_id.clone()]) + }, + ); + } + + if duration_frames < 1 { + return Err(EditError::Invalid(format!( + "durationFrames must be >= 1 (got {duration_frames})" + ))); + } + let to_location = state + .find_clip(&to_clip_id) + .ok_or_else(|| EditError::Invalid(format!("Clip not found: {to_clip_id}")))?; + if state.timeline.tracks[to_location.track_index].clips[to_location.clip_index] + .nested_sequence_id + .is_some() + { + return Err(EditError::Invalid( + "compound clips do not support direct transitions".into(), + )); + } + if from_location.track_index != to_location.track_index { + return Err(EditError::Invalid( + "A transition requires clips on the same track".into(), + )); + } + let track = &state.timeline.tracks[from_location.track_index]; + if track.kind == ClipType::Audio { + return Err(EditError::Invalid( + "Visual transitions are unavailable on audio tracks".into(), + )); + } + let from = &track.clips[from_location.clip_index]; + let to = &track.clips[to_location.clip_index]; + if matches!(from.media_type, ClipType::Audio | ClipType::Text) + || matches!(to.media_type, ClipType::Audio | ClipType::Text) + { + return Err(EditError::Invalid( + "A transition requires two visual source clips".into(), + )); + } + if from.end_frame() != to.start_frame { + return Err(EditError::Invalid( + "A transition requires an exact adjacent clip boundary".into(), + )); + } + let mut ordered: Vec<&opentake_domain::Clip> = track.clips.iter().collect(); + ordered.sort_by_key(|clip| (clip.start_frame, clip.id.as_str())); + let successor = ordered + .iter() + .position(|clip| clip.id == from_clip_id) + .and_then(|index| ordered.get(index + 1)); + if successor.map(|clip| clip.id.as_str()) != Some(to_clip_id.as_str()) { + return Err(EditError::Invalid( + "A transition requires the immediate next clip".into(), + )); + } + + let maximum = (from.duration_frames.min(to.duration_frames) / 2).max(1); + if duration_frames > maximum { + return Err(EditError::Invalid(format!( + "durationFrames exceeds the available transition handle ({duration_frames} > {maximum})" + ))); + } + let kind = kind.expect("kind checked above"); + transact( + state, + "Set Transition", + |_| format!("Set transition from {from_clip_id} to {to_clip_id}"), + |st| { + st.timeline.tracks[from_location.track_index].clips[from_location.clip_index] + .transition_out = Some(Transition { + from_clip_id: from_clip_id.clone(), + to_clip_id: to_clip_id.clone(), + kind, + duration_frames, + }); + Ok(vec![from_clip_id.clone(), to_clip_id.clone()]) + }, + ) } fn ripple_delete_ranges( @@ -2583,6 +4751,8 @@ fn swap_media( if let Some(loc) = st.find_clip(tid) { st.timeline.tracks[loc.track_index].clips[loc.clip_index].media_ref = media_ref.clone(); + st.timeline.tracks[loc.track_index].clips[loc.clip_index] + .loudness_normalization = None; affected.push(tid.clone()); } } @@ -2650,7 +4820,10 @@ fn validate_entry(state: &EditorState, e: &ClipEntry, i: usize) -> Result<(), Ed ))); } let target = state.timeline.tracks[e.track_index].kind; - if !e.source_clip_type.is_compatible(target) { + // Destination compatibility is determined by the placed lane type. A + // linked audio clip can legitimately retain `source_clip_type = Video` + // because it still resolves audio from the original video asset. + if !e.media_type.is_compatible(target) { return Err(EditError::Invalid(format!( "entries[{i}]: asset type is not compatible with the destination track" ))); @@ -2685,12 +4858,12 @@ fn validate_entry(state: &EditorState, e: &ClipEntry, i: usize) -> Result<(), Ed } fn validate_auto_track_entry(e: &ClipEntry, i: usize) -> Result<(), EditError> { - let target = if e.source_clip_type == ClipType::Audio { + let target = if e.media_type == ClipType::Audio { ClipType::Audio } else { ClipType::Video }; - if !e.source_clip_type.is_compatible(target) { + if !e.media_type.is_compatible(target) { return Err(EditError::Invalid(format!( "entries[{i}]: asset type is not compatible with an auto-created track" ))); @@ -4248,6 +6421,305 @@ mod add_captions_tests { // State untouched by the refusal. assert_eq!(state.timeline.tracks.len(), 2); } + + #[test] + fn caption_translation_preserves_identity_timing_and_is_one_undo_step() { + let mut state = state_with_video_and_audio(); + let ids = SeqIdGen::new("cap-"); + apply( + &mut state, + EditCommand::AddCaptions { + entries: vec![caption("Hello", 3, 17, "g"), caption("World", 25, 19, "g")], + }, + &ids, + ) + .unwrap(); + let captions = state.timeline.tracks[0].clips.clone(); + let changes = captions + .iter() + .zip(["你好", "世界"]) + .map(|(clip, translated)| { + let source = clip.text_content.clone().unwrap(); + CaptionTranslationChange { + clip_id: clip.id.clone(), + expected_source_text: source.clone(), + translated_text: translated.into(), + input: CaptionTranslationInput { + source_text: source, + source_locale: "en-US".into(), + target_locale: "zh-CN".into(), + provider: "mock".into(), + model: "mock-v1".into(), + }, + } + }) + .collect(); + let result = apply( + &mut state, + EditCommand::ApplyCaptionTranslations { changes }, + &ids, + ) + .unwrap(); + assert_eq!(result.action_name, "Translate Captions"); + for (before, after) in captions.iter().zip(&state.timeline.tracks[0].clips) { + assert_eq!(after.id, before.id); + assert_eq!(after.start_frame, before.start_frame); + assert_eq!(after.duration_frames, before.duration_frames); + assert_eq!(after.caption_group_id, before.caption_group_id); + assert_eq!( + after + .caption_translation_input + .as_ref() + .unwrap() + .source_text, + before.text_content.clone().unwrap() + ); + } + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert_eq!(state.timeline.tracks[0].clips, captions); + } + + #[test] + fn caption_translation_stale_batch_is_atomic_and_manual_edit_clears_provenance() { + let mut state = state_with_video_and_audio(); + let ids = SeqIdGen::new("cap-"); + apply( + &mut state, + EditCommand::AddCaptions { + entries: vec![caption("One", 0, 10, "g"), caption("Two", 10, 10, "g")], + }, + &ids, + ) + .unwrap(); + let before = state.timeline.clone(); + let caption_ids: Vec<_> = state.timeline.tracks[0] + .clips + .iter() + .map(|clip| clip.id.clone()) + .collect(); + let change = |clip_id: String, source: &str, translated: &str| CaptionTranslationChange { + clip_id, + expected_source_text: source.into(), + translated_text: translated.into(), + input: CaptionTranslationInput { + source_text: source.into(), + source_locale: "en".into(), + target_locale: "fr".into(), + provider: "mock".into(), + model: "mock-v1".into(), + }, + }; + assert!(apply( + &mut state, + EditCommand::ApplyCaptionTranslations { + changes: vec![ + change(caption_ids[0].clone(), "One", "Un"), + change(caption_ids[1].clone(), "stale", "Deux"), + ], + }, + &ids, + ) + .is_err()); + assert_eq!(state.timeline, before); + + apply( + &mut state, + EditCommand::ApplyCaptionTranslations { + changes: vec![change(caption_ids[0].clone(), "One", "Un")], + }, + &ids, + ) + .unwrap(); + apply( + &mut state, + EditCommand::SetClipProperties { + clip_ids: vec![caption_ids[0].clone()], + properties: Box::new(ClipProperties { + text_content: Some("Edited".into()), + ..ClipProperties::default() + }), + }, + &ids, + ) + .unwrap(); + assert!(state.timeline.tracks[0].clips[0] + .caption_translation_input + .is_none()); + } +} + +#[cfg(test)] +mod script_assembly_command_tests { + use super::*; + use crate::id::SeqIdGen; + use opentake_domain::{MediaSource, ScriptAssemblySegment}; + + fn media(id: &str, kind: ClipType, duration: f64, has_audio: bool) -> MediaManifestEntry { + MediaManifestEntry { + id: id.into(), + name: id.into(), + kind, + source: MediaSource::Project { + relative_path: format!("media/{id}"), + }, + duration, + generation_input: None, + source_width: None, + source_height: None, + source_fps: None, + has_audio: Some(has_audio), + color: None, + proxy: None, + folder_id: None, + cached_remote_url: None, + cached_remote_url_expires_at: None, + } + } + + fn plan() -> ScriptAssemblyPlan { + ScriptAssemblyPlan { + id: "script-plan-test".into(), + plan_hash: "a".repeat(64), + planner: "test-planner".into(), + planner_version: 1, + start_frame: 0, + segments: (0..3) + .map(|index| ScriptAssemblySegment { + script: format!("Scene {}", index + 1), + media_ref: format!("visual-{index}"), + narration_media_ref: Some(format!("voice-{index}")), + duration_frames: 30, + transition: (index < 2).then_some(TransitionKind::CrossDissolve), + }) + .collect(), + } + } + + fn state() -> EditorState { + let mut state = EditorState::default(); + state.timeline.settings_configured = true; + for index in 0..3 { + state.manifest.entries.push(media( + &format!("visual-{index}"), + ClipType::Image, + 0.0, + false, + )); + state.manifest.entries.push(media( + &format!("voice-{index}"), + ClipType::Audio, + 1.0, + true, + )); + } + state + } + + #[test] + fn reviewed_three_segment_plan_applies_tracks_sync_transitions_and_one_undo() { + let mut state = state(); + let ids = SeqIdGen::new("script-"); + apply( + &mut state, + EditCommand::SaveScriptAssemblyPlan { plan: plan() }, + &ids, + ) + .unwrap(); + assert!(state.timeline.tracks.is_empty()); + assert_eq!(state.timeline.script_assembly_plans, vec![plan()]); + let undo_depth_before_apply = state.undo_depth(); + let result = apply( + &mut state, + EditCommand::ApplyScriptAssemblyPlan { + plan_id: "script-plan-test".into(), + }, + &ids, + ) + .unwrap(); + assert_eq!(result.action_name, "Build Script Video"); + assert_eq!(state.undo_depth(), undo_depth_before_apply + 1); + assert_eq!(state.timeline.tracks.len(), 2); + let visual = &state.timeline.tracks[0]; + let narration = &state.timeline.tracks[1]; + assert_eq!(visual.kind, ClipType::Video); + assert_eq!(narration.kind, ClipType::Audio); + assert_eq!(visual.clips.len(), 3); + assert_eq!(narration.clips.len(), 3); + for index in 0..3 { + assert_eq!(visual.clips[index].start_frame, index as i32 * 30); + assert_eq!(narration.clips[index].start_frame, index as i32 * 30); + assert_eq!(visual.clips[index].duration_frames, 30); + assert_eq!(narration.clips[index].duration_frames, 30); + assert_eq!(visual.clips[index].volume, 0.0); + } + for index in 0..2 { + let transition = visual.clips[index].transition_out.as_ref().unwrap(); + assert_eq!(transition.from_clip_id, visual.clips[index].id); + assert_eq!(transition.to_clip_id, visual.clips[index + 1].id); + assert_eq!(transition.kind, TransitionKind::CrossDissolve); + assert_eq!(transition.duration_frames, 12); + } + assert!(visual.clips[2].transition_out.is_none()); + + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert!(state.timeline.tracks.is_empty()); + assert_eq!(state.timeline.script_assembly_plans, vec![plan()]); + } + + #[test] + fn missing_media_refuses_atomically_and_narration_must_match_within_one_frame() { + let mut state = state(); + let ids = SeqIdGen::new("script-"); + let mut invalid = plan(); + invalid.id = "missing-plan".into(); + invalid.segments[1].media_ref = "missing".into(); + apply( + &mut state, + EditCommand::SaveScriptAssemblyPlan { + plan: invalid.clone(), + }, + &ids, + ) + .unwrap(); + let before = state.timeline.clone(); + assert!(apply( + &mut state, + EditCommand::ApplyScriptAssemblyPlan { + plan_id: invalid.id, + }, + &ids, + ) + .is_err()); + assert_eq!(state.timeline, before); + + let mut mismatch = plan(); + mismatch.id = "mismatch-plan".into(); + state + .manifest + .entries + .iter_mut() + .find(|entry| entry.id == "voice-0") + .unwrap() + .duration = 2.0; + apply( + &mut state, + EditCommand::SaveScriptAssemblyPlan { + plan: mismatch.clone(), + }, + &ids, + ) + .unwrap(); + let before = state.timeline.clone(); + assert!(apply( + &mut state, + EditCommand::ApplyScriptAssemblyPlan { + plan_id: mismatch.id, + }, + &ids, + ) + .is_err()); + assert_eq!(state.timeline, before); + } } /// Tests for [`EditCommand::AddTextsAutoTrack`] (#194): the all-omitted- diff --git a/crates/opentake-ops/src/editor_state.rs b/crates/opentake-ops/src/editor_state.rs index f6693160..6af47d67 100644 --- a/crates/opentake-ops/src/editor_state.rs +++ b/crates/opentake-ops/src/editor_state.rs @@ -97,32 +97,48 @@ impl EditorState { self.version += 1; } + /// Commit an irreversible audit mutation without adding an undo entry. + /// Earlier undo snapshots are retained, but restore paths keep provider + /// voice records outside ordinary document undo/redo. + pub(crate) fn commit_irreversible(&mut self) { + self.redo_stack.clear(); + self.version += 1; + } + /// Undo the most recent committed change. Returns `true` if anything was /// undone. Pushes the pre-undo document onto the redo stack and bumps the /// version. pub(crate) fn undo(&mut self) -> bool { - let Some(prev) = self.undo_stack.pop() else { - return false; - }; let current = self.snapshot(); - self.restore(prev); - self.redo_stack.push(current); - self.version += 1; - true + while let Some(mut prev) = self.undo_stack.pop() { + preserve_voice_models(&mut prev, ¤t); + if prev == current { + continue; + } + self.restore(prev); + self.redo_stack.push(current); + self.version += 1; + return true; + } + false } /// Redo the most recently undone change. Returns `true` if anything was /// redone. Pushes the pre-redo document onto the undo stack and bumps the /// version. pub(crate) fn redo(&mut self) -> bool { - let Some(next) = self.redo_stack.pop() else { - return false; - }; let current = self.snapshot(); - self.restore(next); - self.undo_stack.push(current); - self.version += 1; - true + while let Some(mut next) = self.redo_stack.pop() { + preserve_voice_models(&mut next, ¤t); + if next == current { + continue; + } + self.restore(next); + self.undo_stack.push(current); + self.version += 1; + return true; + } + false } // MARK: - Lookups (1:1 port of EditorViewModel.findClip) @@ -143,10 +159,14 @@ impl EditorState { } } +fn preserve_voice_models(target: &mut DocSnapshot, current: &DocSnapshot) { + target.timeline.voice_models = current.timeline.voice_models.clone(); +} + #[cfg(test)] mod tests { use super::*; - use opentake_domain::{Clip, ClipType, Track}; + use opentake_domain::{Clip, ClipType, Track, VoiceModelRecord}; fn state_with_clip() -> EditorState { let mut tl = Timeline::new(); @@ -194,6 +214,61 @@ mod tests { assert_eq!(s.version(), 3); } + #[test] + fn permanent_voice_revocation_survives_all_undo_snapshots() { + let mut state = EditorState::default(); + let before_enroll = state.snapshot(); + state.timeline.voice_models.push(VoiceModelRecord { + id: "voice-1".into(), + provider: "elevenlabs".into(), + provider_voice_id: "provider-1".into(), + model: "model".into(), + consent_id: "consent-1".into(), + source_audio_asset_id: "audio-1".into(), + source_audio_sha256: "a".repeat(64), + request_hash: "b".repeat(64), + voice_name: "Narrator".into(), + revoked: false, + }); + state.commit(before_enroll); + state.timeline.voice_models[0].revoked = true; + state.commit_irreversible(); + + let version = state.version(); + assert!(!state.undo()); + assert_eq!(state.timeline.voice_models.len(), 1); + assert!(state.timeline.voice_models[0].revoked); + assert!(!state.can_undo()); + assert!(!state.redo()); + assert_eq!(state.version(), version); + } + + #[test] + fn active_provider_voice_survives_undo_of_an_earlier_edit() { + let mut state = state_with_clip(); + let before_edit = state.snapshot(); + state.timeline.tracks[0].clips[0].start_frame = 12; + state.commit(before_edit); + state.timeline.voice_models.push(VoiceModelRecord { + id: "voice-1".into(), + provider: "elevenlabs".into(), + provider_voice_id: "provider-1".into(), + model: "model".into(), + consent_id: "consent-1".into(), + source_audio_asset_id: "audio-1".into(), + source_audio_sha256: "a".repeat(64), + request_hash: "b".repeat(64), + voice_name: "Narrator".into(), + revoked: false, + }); + state.commit_irreversible(); + + assert!(state.undo()); + assert_eq!(state.timeline.tracks[0].clips[0].start_frame, 0); + assert_eq!(state.timeline.voice_models.len(), 1); + assert!(!state.timeline.voice_models[0].revoked); + } + #[test] fn new_edit_clears_redo_stack() { let mut s = state_with_clip(); diff --git a/crates/opentake-ops/src/intent.rs b/crates/opentake-ops/src/intent.rs index b88d426a..f8b95899 100644 --- a/crates/opentake-ops/src/intent.rs +++ b/crates/opentake-ops/src/intent.rs @@ -276,7 +276,10 @@ fn validate_intent_entry( "entries[{index}]: track index {track_index} out of range" ))); }; - if !entry.source_clip_type.is_compatible(track.kind) { + // A placed audio lane can come from a video asset (linked audio), so + // track compatibility follows the placed media type rather than the + // source container type. + if !entry.media_type.is_compatible(track.kind) { return Err(EditError::Invalid(format!( "entries[{index}]: asset type is not compatible with the destination track" ))); diff --git a/crates/opentake-ops/src/lib.rs b/crates/opentake-ops/src/lib.rs index 6c286947..baf872b0 100644 --- a/crates/opentake-ops/src/lib.rs +++ b/crates/opentake-ops/src/lib.rs @@ -31,8 +31,9 @@ pub use engines::{ // --- Command layer --- pub use command::{ - apply, CaptionEntry, ClipEntry, ClipProperties, EditCommand, EditError, EditResult, - KeyframePayload, KeyframeProperty, KeyframeValue, RenameEntry, TextAutoTrackEntry, TextEntry, + apply, CaptionEntry, CaptionTranslationChange, ClipEntry, ClipProperties, EditCommand, + EditError, EditResult, KeyframePayload, KeyframeProperty, KeyframeValue, RenameEntry, + TextAutoTrackEntry, TextEntry, }; pub use editor_state::{DocSnapshot, EditorState}; pub use id::{IdGen, SeqIdGen}; diff --git a/crates/opentake-ops/src/ops/clear_region.rs b/crates/opentake-ops/src/ops/clear_region.rs index 77590901..1f4e348e 100644 --- a/crates/opentake-ops/src/ops/clear_region.rs +++ b/crates/opentake-ops/src/ops/clear_region.rs @@ -49,6 +49,7 @@ pub fn clear_region( let new_trim_end = clip.trim_end_frame + source_delta; let c = &mut timeline.tracks[ti].clips[ci]; c.trim_end_frame = new_trim_end; + c.loudness_normalization = None; c.set_duration(new_duration); } } @@ -63,6 +64,7 @@ pub fn clear_region( let c = &mut timeline.tracks[ti].clips[ci]; c.start_frame = new_start_frame; c.trim_start_frame = new_trim_start; + c.loudness_normalization = None; c.set_duration(new_duration); } } @@ -71,13 +73,17 @@ pub fn clear_region( if find(timeline, &clip_id).is_some() { // Split at `start`; the right half is what now covers the region. split_clip(timeline, &clip_id, start, ids); - // Locate the freshly created right half (starts at `start`, not the original id). - let right = timeline - .tracks - .iter() - .flat_map(|t| &t.clips) - .find(|c| c.start_frame == start && c.id != clip_id) - .map(|c| (c.id.clone(), c.end_frame())); + // Locate the freshly created right half on the original + // clip's track. Linked splits mint a right half on every + // partner track; a global search can select the wrong + // partner and leave a duplicate middle fragment behind. + let right = find(timeline, &clip_id).and_then(|(ti, _)| { + timeline.tracks[ti] + .clips + .iter() + .find(|c| c.start_frame == start && c.id != clip_id) + .map(|c| (c.id.clone(), c.end_frame())) + }); if let Some((right_id, right_end)) = right { if right_end > end { // Right half overruns the region — split again at `end`, diff --git a/crates/opentake-ops/src/ops/duplicate.rs b/crates/opentake-ops/src/ops/duplicate.rs index 9097a5b6..b984768c 100644 --- a/crates/opentake-ops/src/ops/duplicate.rs +++ b/crates/opentake-ops/src/ops/duplicate.rs @@ -338,8 +338,9 @@ mod tests { }, feather: 0.05, invert: false, + ..Mask::default() }]; - src.effects = vec![Effect::new("gaussianBlur").with_param("radius", 4.0)]; + src.effects = vec![Effect::new("grayscale").with_param("amount", 0.4)]; let orig_color_grade = src.color_grade; let orig_chroma_key = src.chroma_key; let g = SeqIdGen::default(); diff --git a/crates/opentake-ops/src/ops/folders.rs b/crates/opentake-ops/src/ops/folders.rs index 42d6dcce..87615c7a 100644 --- a/crates/opentake-ops/src/ops/folders.rs +++ b/crates/opentake-ops/src/ops/folders.rs @@ -131,9 +131,15 @@ pub fn delete_folder( (folders_removed, assets_removed, clips_removed) } -/// Remove every clip whose `media_ref` is in `asset_ids`, then prune any tracks -/// left empty (mirroring `remove_clips`). Returns the count of clips removed. +/// Remove every clip whose `media_ref` is in `asset_ids` from the root and all +/// registered nested timelines, then prune tracks left empty (mirroring +/// `remove_clips`). Returns the total count of clips removed across the graph. fn cascade_remove_clips(timeline: &mut Timeline, asset_ids: &HashSet) -> usize { + let nested_count = timeline + .nested_sequences + .iter_mut() + .map(|sequence| cascade_remove_clips(&mut sequence.timeline, asset_ids)) + .sum::(); let doomed: Vec = timeline .tracks .iter() @@ -148,7 +154,7 @@ fn cascade_remove_clips(timeline: &mut Timeline, asset_ids: &HashSet) -> if count > 0 { crate::ops::prune_empty_tracks(timeline); } - count + nested_count + count } /// Expand a set of root folder ids to include all transitive descendant folders @@ -192,6 +198,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -311,6 +319,40 @@ mod tests { assert_eq!(tl.tracks[0].clips[0].id, "clip-2"); } + #[test] + fn delete_media_cascades_through_nested_timelines() { + use opentake_domain::{Clip, NestedSequence, Track}; + + let mut m = MediaManifest::new(); + m.entries.push(entry("a")); + m.entries.push(entry("b")); + + let mut child = timeline_with_clip("nested-a", "a"); + child.tracks[0] + .clips + .push(Clip::new("nested-b", "b", 40, 30)); + let mut root = Timeline::new(); + let mut root_track = Track::new("root-track", ClipType::Video); + root_track + .clips + .push(Clip::new_nested("compound", "sequence", 0, 70)); + root.tracks.push(root_track); + root.nested_sequences + .push(NestedSequence::new("sequence", "Nested", child)); + + let (assets, clips) = + delete_media(&mut root, &mut m, &["a".to_string()].into_iter().collect()); + + assert_eq!(assets, 1); + assert_eq!(clips, 1); + assert_eq!(root.nested_sequences[0].timeline.tracks.len(), 1); + assert_eq!( + root.nested_sequences[0].timeline.tracks[0].clips[0].id, + "nested-b" + ); + assert_eq!(root.tracks[0].clips[0].id, "compound"); + } + #[test] fn delete_folder_recurses_and_cascades() { let mut m = MediaManifest::new(); diff --git a/crates/opentake-ops/src/ops/ripple.rs b/crates/opentake-ops/src/ops/ripple.rs index 8e6b7629..b758328d 100644 --- a/crates/opentake-ops/src/ops/ripple.rs +++ b/crates/opentake-ops/src/ops/ripple.rs @@ -488,6 +488,71 @@ mod tests { } } + #[test] + fn ripple_delete_ranges_keeps_linked_av_frame_exact() { + let mut tl = Timeline::new(); + let mut video = Track::new("video", ClipType::Video); + let mut video_clip = clip("video-clip", 0, 900); + video_clip.link_group_id = Some("av".into()); + video.clips.push(video_clip); + let mut audio = Track::new("audio", ClipType::Audio); + let mut audio_clip = clip("audio-clip", 0, 900); + audio_clip.media_type = ClipType::Audio; + audio_clip.link_group_id = Some("av".into()); + audio.clips.push(audio_clip); + tl.tracks.extend([video, audio]); + + let g = SeqIdGen::new("r-"); + let out = ripple_delete_ranges_on_track(&mut tl, 1, &[FrameRange::new(6, 12)], &label, &g); + assert!(matches!(out, RippleOutcome::Ok(_))); + let spans = |track: &Track| { + track + .clips + .iter() + .map(|clip| (clip.start_frame, clip.end_frame())) + .collect::>() + }; + assert_eq!(spans(&tl.tracks[0]), vec![(0, 6), (6, 894)]); + assert_eq!(spans(&tl.tracks[1]), vec![(0, 6), (6, 894)]); + } + + #[test] + fn ripple_delete_multiple_ranges_with_sync_locked_captions_is_atomic() { + let mut tl = Timeline::new(); + let mut captions = Track::new("captions", ClipType::Video); + captions.sync_locked = true; + captions.clips.extend([ + clip("caption-1", 0, 86), + clip("caption-2", 146, 116), + clip("caption-3", 350, 21), + clip("caption-4", 371, 57), + clip("caption-5", 428, 82), + ]); + let mut video = Track::new("video", ClipType::Video); + let mut video_clip = clip("video-clip", 0, 900); + video_clip.link_group_id = Some("av".into()); + video.clips.push(video_clip); + let mut audio = Track::new("audio", ClipType::Audio); + let mut audio_clip = clip("audio-clip", 0, 900); + audio_clip.media_type = ClipType::Audio; + audio_clip.link_group_id = Some("av".into()); + audio.clips.push(audio_clip); + tl.tracks.extend([captions, video, audio]); + let before = tl.clone(); + + let g = SeqIdGen::new("r-"); + let out = ripple_delete_ranges_on_track( + &mut tl, + 2, + &[FrameRange::new(151, 154), FrameRange::new(358, 365)], + &label, + &g, + ); + + assert!(matches!(out, RippleOutcome::Refused(_))); + assert_eq!(tl, before, "a follower collision must be side-effect free"); + } + #[test] fn ripple_delete_ranges_refuses_on_locked_follower_collision() { let mut tl = Timeline::new(); diff --git a/crates/opentake-ops/src/ops/settings.rs b/crates/opentake-ops/src/ops/settings.rs index 9e35fa7b..094c74b3 100644 --- a/crates/opentake-ops/src/ops/settings.rs +++ b/crates/opentake-ops/src/ops/settings.rs @@ -33,13 +33,21 @@ pub fn set_timeline_settings(timeline: &mut Timeline, fps: i32, width: i32, heig return false; } + // Nested timelines share one project timebase and output canvas. Keep every + // stored child synchronized (including frame/keyframe rescaling) so entering + // a compound never exposes stale settings after the root changes. + let mut nested_changed = false; + for sequence in &mut timeline.nested_sequences { + nested_changed |= set_timeline_settings(&mut sequence.timeline, fps, width, height); + } + let prev_fps = timeline.fps; let prev_width = timeline.width; let prev_height = timeline.height; let prev_configured = timeline.settings_configured; if fps == prev_fps && width == prev_width && height == prev_height && prev_configured { - return false; + return nested_changed; } // Rescale all frame-based values when FPS changes (upstream :26-52). @@ -62,6 +70,10 @@ pub fn set_timeline_settings(timeline: &mut Timeline, fps: i32, width: i32, heig clip.rescale_keyframes(scale); clip.fade_in_frames = round_scale(clip.fade_in_frames, scale); clip.fade_out_frames = round_scale(clip.fade_out_frames, scale); + if let Some(transition) = &mut clip.transition_out { + transition.duration_frames = + round_scale(transition.duration_frames, scale).max(1); + } clip.clamp_keyframes_to_duration(); clip.clamp_fades_to_duration(); previous_end = Some(clip.end_frame()); @@ -85,7 +97,9 @@ fn round_scale(value: i32, scale: f64) -> i32 { #[cfg(test)] mod tests { use super::*; - use opentake_domain::{Clip, ClipType, Keyframe, KeyframeTrack, Track}; + use opentake_domain::{ + Clip, ClipType, Keyframe, KeyframeTrack, Track, Transition, TransitionKind, + }; fn track(id: &str, kind: ClipType, clips: Vec) -> Track { let mut track = Track::new(id, kind); @@ -117,6 +131,29 @@ mod tests { assert_eq!((c.start_frame, c.duration_frames), (10, 40)); } + #[test] + fn settings_change_rescales_registered_nested_timelines() { + use opentake_domain::NestedSequence; + + let mut child = Timeline::new(); + child.tracks.push(track( + "child", + ClipType::Video, + vec![clip("nested", 15, 30)], + )); + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence", "Scene", child)); + + assert!(set_timeline_settings(&mut root, 60, 1280, 720)); + + let child = &root.nested_sequences[0].timeline; + assert_eq!((child.fps, child.width, child.height), (60, 1280, 720)); + assert!(child.settings_configured); + assert_eq!(child.tracks[0].clips[0].start_frame, 30); + assert_eq!(child.tracks[0].clips[0].duration_frames, 60); + } + #[test] fn fps_doubling_scales_clip_start_and_duration() { let mut tl = Timeline::new(); @@ -147,6 +184,29 @@ mod tests { assert_eq!(c.fade_out_frames, 24); } + #[test] + fn fps_change_scales_transition_duration() { + let mut tl = Timeline::new(); + let mut a = clip("a", 0, 60); + a.transition_out = Some(Transition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: TransitionKind::CrossDissolve, + duration_frames: 15, + }); + tl.tracks + .push(track("v", ClipType::Video, vec![a, clip("b", 60, 60)])); + assert!(set_timeline_settings(&mut tl, 60, 1920, 1080)); + assert_eq!( + tl.tracks[0].clips[0] + .transition_out + .as_ref() + .unwrap() + .duration_frames, + 30 + ); + } + #[test] fn fps_change_rescales_keyframe_offsets() { let mut tl = Timeline::new(); diff --git a/crates/opentake-ops/src/ops/trim.rs b/crates/opentake-ops/src/ops/trim.rs index e5d90f3d..2bea5f5d 100644 --- a/crates/opentake-ops/src/ops/trim.rs +++ b/crates/opentake-ops/src/ops/trim.rs @@ -48,6 +48,7 @@ pub fn trim_clip_internal( let c = &mut timeline.tracks[ti].clips[ci]; c.trim_start_frame = trim_start_frame; c.trim_end_frame = trim_end_frame; + c.loudness_normalization = None; c.start_frame = new_start_frame; c.set_duration(new_duration); diff --git a/crates/opentake-ops/tests/command_apply.rs b/crates/opentake-ops/tests/command_apply.rs index d0ac67e9..d968dd56 100644 --- a/crates/opentake-ops/tests/command_apply.rs +++ b/crates/opentake-ops/tests/command_apply.rs @@ -3,8 +3,11 @@ //! resulting `Timeline` / `MediaManifest`, undo/redo behavior, versioning, and //! the refusal path — the behaviors the port must match upstream. -use opentake_domain::{AnimPair, Interpolation, Keyframe, KeyframeTrack}; -use opentake_domain::{ChromaKey, ColorGrade, Effect, Mask, MaskShape, Point2}; +use opentake_domain::{AnimPair, Interpolation, Keyframe, KeyframeTrack, TransitionKind}; +use opentake_domain::{ + ChromaKey, ColorGrade, Effect, HslSecondary, LiftGammaGain, LutReference, Mask, MaskShape, + Point2, Rgb, +}; use opentake_domain::{ Clip, ClipType, MediaManifest, MediaManifestEntry, MediaSource, Timeline, Track, Transform, }; @@ -55,6 +58,463 @@ fn entry(track_index: usize, media_type: ClipType, start: i32, dur: i32) -> Clip } } +#[test] +fn compound_create_edit_move_trim_duplicate_dissolve_and_undo_share_one_command_path() { + let mut child = Timeline::new(); + child.tracks = vec![video_track( + "child-track", + true, + vec![Clip::new("child", "asset-child", 5, 20)], + )]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + + let created = apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene A".into(), + timeline: child, + track_index: 0, + start_frame: 100, + duration_frames: 30, + }, + &ids, + ) + .unwrap(); + let compound_id = created.affected_clip_ids[0].clone(); + assert_eq!(st.timeline.nested_sequences.len(), 1); + assert_eq!(st.undo_depth(), 1); + + let sequence_id = st.timeline.nested_sequences[0].id.clone(); + let mut edited = st.timeline.nested_sequences[0].timeline.clone(); + edited.tracks[0].clips[0].media_ref = "asset-edited".into(); + apply( + &mut st, + EditCommand::SetNestedSequenceTimeline { + sequence_id: sequence_id.clone(), + timeline: edited, + }, + &ids, + ) + .unwrap(); + apply( + &mut st, + EditCommand::RenameNestedSequence { + sequence_id, + name: "Edited scene".into(), + }, + &ids, + ) + .unwrap(); + apply( + &mut st, + EditCommand::MoveClips { + moves: vec![ClipMove { + clip_id: compound_id.clone(), + to_track: 0, + to_frame: 110, + }], + }, + &ids, + ) + .unwrap(); + apply( + &mut st, + EditCommand::TrimClips { + edits: vec![(compound_id.clone(), 5, 0)], + }, + &ids, + ) + .unwrap(); + let duplicate = apply( + &mut st, + EditCommand::DuplicateClips { + clip_ids: vec![compound_id.clone()], + offset_frames: 40, + target_track_indexes: vec![0], + }, + &ids, + ) + .unwrap(); + assert_eq!(duplicate.affected_clip_ids.len(), 1); + + let before_dissolve = st.timeline.clone(); + let dissolved = apply( + &mut st, + EditCommand::DissolveNestedSequence { + clip_id: compound_id.clone(), + }, + &ids, + ) + .unwrap(); + assert_eq!(dissolved.affected_clip_ids.len(), 1); + let leaf = st + .timeline + .tracks + .iter() + .flat_map(|track| &track.clips) + .find(|clip| dissolved.affected_clip_ids.contains(&clip.id)) + .unwrap(); + assert_eq!(leaf.media_ref, "asset-edited"); + assert_eq!(leaf.start_frame, 115); + assert_eq!(leaf.duration_frames, 20); + assert_eq!(leaf.trim_start_frame, 0); + + apply(&mut st, EditCommand::Undo, &ids).unwrap(); + assert_eq!(st.timeline, before_dissolve); +} + +#[test] +fn dissolve_refuses_parent_edits_without_changing_history() { + let mut child = Timeline::new(); + child.tracks = vec![video_track( + "child-track", + true, + vec![Clip::new("child", "asset-child", 0, 20)], + )]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + let created = apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 20, + }, + &ids, + ) + .unwrap(); + let compound_id = created.affected_clip_ids[0].clone(); + apply( + &mut st, + EditCommand::SetClipProperties { + clip_ids: vec![compound_id.clone()], + properties: Box::new(ClipProperties { + opacity: Some(0.5), + ..ClipProperties::default() + }), + }, + &ids, + ) + .unwrap(); + let before = st.timeline.clone(); + let undo_depth = st.undo_depth(); + let version = st.version(); + + let error = apply( + &mut st, + EditCommand::DissolveNestedSequence { + clip_id: compound_id, + }, + &ids, + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("parent-level edits must be normalized")); + assert_eq!(st.timeline, before); + assert_eq!(st.undo_depth(), undo_depth); + assert_eq!(st.version(), version); +} + +#[test] +fn compound_edit_refuses_properties_that_cannot_render() { + let mut child = Timeline::new(); + child.tracks = vec![video_track( + "child-track", + true, + vec![Clip::new("child", "asset-child", 0, 20)], + )]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + let created = apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 20, + }, + &ids, + ) + .unwrap(); + let compound_id = created.affected_clip_ids[0].clone(); + let before = st.timeline.clone(); + let undo_depth = st.undo_depth(); + + let error = apply( + &mut st, + EditCommand::SetClipProperties { + clip_ids: vec![compound_id.clone()], + properties: Box::new(ClipProperties { + speed: Some(2.0), + ..ClipProperties::default() + }), + }, + &ids, + ) + .unwrap_err(); + assert!(error.to_string().contains("does not support retime")); + assert_eq!(st.timeline, before); + assert_eq!(st.undo_depth(), undo_depth); + + let error = apply( + &mut st, + EditCommand::SetColorGrade { + clip_ids: vec![compound_id], + grade: Some(ColorGrade::default()), + }, + &ids, + ) + .unwrap_err(); + assert!(error.to_string().contains("direct pixel effects")); + assert_eq!(st.timeline, before); + assert_eq!(st.undo_depth(), undo_depth); +} + +#[test] +fn compound_creation_expands_linked_partners_and_refuses_unselected_overlap() { + let mut video = Clip::new("video", "video-asset", 0, 10); + video.link_group_id = Some("av".into()); + let blocker = Clip::new("blocker", "blocker-asset", 15, 5); + let later = Clip::new("later", "later-asset", 20, 10); + let mut audio = Clip::new("audio", "audio-asset", 0, 10); + audio.media_type = ClipType::Audio; + audio.source_clip_type = ClipType::Audio; + audio.link_group_id = Some("av".into()); + let mut st = state(vec![ + video_track("v1", true, vec![video, blocker]), + video_track("v2", true, vec![later]), + audio_track("a1", true, vec![audio]), + ]); + let ids = SeqIdGen::new("nested-"); + let before = st.timeline.clone(); + + let error = apply( + &mut st, + EditCommand::CreateNestedSequenceFromClips { + name: "Blocked".into(), + clip_ids: vec!["video".into(), "later".into()], + }, + &ids, + ) + .unwrap_err(); + assert!(error.to_string().contains("overlaps an unselected clip")); + assert_eq!(st.timeline, before); + + apply( + &mut st, + EditCommand::CreateNestedSequenceFromClips { + name: "Linked".into(), + clip_ids: vec!["video".into()], + }, + &ids, + ) + .unwrap(); + let child_clips = st.timeline.nested_sequences[0] + .timeline + .tracks + .iter() + .flat_map(|track| &track.clips) + .collect::>(); + assert_eq!(child_clips.len(), 2); + assert!(child_clips.iter().any(|clip| clip.id == "video")); + assert!(child_clips.iter().any(|clip| clip.id == "audio")); +} + +#[test] +fn dissolve_remaps_link_groups_and_transition_targets() { + let mut first = Clip::new("first", "first-asset", 0, 10); + first.link_group_id = Some("av".into()); + first.transition_out = Some(opentake_domain::Transition { + from_clip_id: "first".into(), + to_clip_id: "second".into(), + kind: TransitionKind::CrossDissolve, + duration_frames: 3, + }); + let second = Clip::new("second", "second-asset", 10, 10); + let mut audio = Clip::new("audio", "audio-asset", 0, 10); + audio.media_type = ClipType::Audio; + audio.source_clip_type = ClipType::Audio; + audio.link_group_id = Some("av".into()); + let mut child = Timeline::new(); + child.tracks = vec![ + video_track("child-video", true, vec![first, second]), + audio_track("child-audio", true, vec![audio]), + ]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + let created = apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 20, + }, + &ids, + ) + .unwrap(); + apply( + &mut st, + EditCommand::DissolveNestedSequence { + clip_id: created.affected_clip_ids[0].clone(), + }, + &ids, + ) + .unwrap(); + + let clips = st + .timeline + .tracks + .iter() + .flat_map(|track| &track.clips) + .collect::>(); + let first = clips + .iter() + .find(|clip| clip.media_ref == "first-asset") + .unwrap(); + let second = clips + .iter() + .find(|clip| clip.media_ref == "second-asset") + .unwrap(); + let audio = clips + .iter() + .find(|clip| clip.media_ref == "audio-asset") + .unwrap(); + assert_eq!(first.transition_out.as_ref().unwrap().to_clip_id, second.id); + assert!(first.link_group_id.is_some()); + assert_eq!(first.link_group_id, audio.link_group_id); + assert_ne!(first.link_group_id.as_deref(), Some("av")); +} + +#[test] +fn invalid_nested_edit_restores_document_and_history() { + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + let mut child = Timeline::new(); + child.tracks.push(video_track( + "child-track", + true, + vec![Clip::new_nested("bad-ref", "missing", 0, 10)], + )); + let before = st.timeline.clone(); + let error = apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Invalid".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 10, + }, + &ids, + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("missing nested sequence reference")); + assert_eq!(st.timeline, before); + assert_eq!(st.undo_depth(), 0); +} + +#[test] +fn nested_child_command_edits_in_place_and_root_undo_restores_it() { + let mut child = Timeline::new(); + child.tracks = vec![video_track( + "child-track", + true, + vec![Clip::new("child-a", "asset", 0, 20)], + )]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 20, + }, + &ids, + ) + .unwrap(); + let sequence_id = st.timeline.nested_sequences[0].id.clone(); + let before = st.timeline.clone(); + + let result = apply( + &mut st, + EditCommand::EditNestedSequence { + sequence_id, + command: Box::new(EditCommand::MoveClips { + moves: vec![ClipMove { + clip_id: "child-a".into(), + to_track: 0, + to_frame: 7, + }], + }), + }, + &ids, + ) + .unwrap(); + assert!(result.changed); + assert_eq!( + st.timeline.nested_sequences[0].timeline.tracks[0].clips[0].start_frame, + 7 + ); + + apply(&mut st, EditCommand::Undo, &ids).unwrap(); + assert_eq!(st.timeline, before); +} + +#[test] +fn nested_child_refuses_root_scoped_commands() { + let mut child = Timeline::new(); + child.tracks = vec![video_track("child-track", true, vec![])]; + let mut st = state(vec![video_track("root", true, vec![])]); + let ids = SeqIdGen::new("nested-"); + apply( + &mut st, + EditCommand::CreateNestedSequence { + name: "Scene".into(), + timeline: child, + track_index: 0, + start_frame: 0, + duration_frames: 20, + }, + &ids, + ) + .unwrap(); + let sequence_id = st.timeline.nested_sequences[0].id.clone(); + let before_timeline = st.timeline.clone(); + let before_manifest = st.manifest.clone(); + let undo_depth = st.undo_depth(); + + let error = apply( + &mut st, + EditCommand::EditNestedSequence { + sequence_id, + command: Box::new(EditCommand::DeleteMedia { + asset_ids: vec!["asset".into()], + }), + }, + &ids, + ) + .unwrap_err(); + + assert!(error.to_string().contains("must target the root timeline")); + assert_eq!(st.timeline, before_timeline); + assert_eq!(st.manifest, before_manifest); + assert_eq!(st.undo_depth(), undo_depth); +} + // ---- add_clips + overwrite ------------------------------------------------ #[test] @@ -109,6 +569,23 @@ fn add_clips_applies_supplied_transform() { assert_eq!(placed.transform.height, 1.0); } +#[test] +fn add_clips_accepts_audio_lane_derived_from_video_asset() { + let mut st = state(vec![audio_track("a", true, vec![])]); + let g = SeqIdGen::new("n-"); + let mut e = entry(0, ClipType::Audio, 15, 110); + e.source_clip_type = ClipType::Video; + e.trim_start_frame = Some(10); + + apply(&mut st, EditCommand::AddClips { entries: vec![e] }, &g).unwrap(); + + let placed = &st.timeline.tracks[0].clips[0]; + assert_eq!(placed.media_type, ClipType::Audio); + assert_eq!(placed.source_clip_type, ClipType::Video); + assert_eq!(placed.start_frame, 15); + assert_eq!(placed.trim_start_frame, 10); +} + #[test] fn add_clips_rejects_out_of_range_track() { let mut st = state(vec![video_track("v", true, vec![])]); @@ -753,6 +1230,116 @@ fn set_clip_properties_multiple_fields_at_once() { assert!(c.opacity_track.is_none()); // opacity scalar cleared its track } +#[test] +fn set_transition_validates_pair_rejects_oversize_and_undoes() { + let mut st = state(vec![video_track( + "v", + true, + vec![clip("a", 0, 100), clip("b", 100, 40)], + )]); + let g = SeqIdGen::default(); + + let result = apply( + &mut st, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 20, + }, + &g, + ) + .unwrap(); + + assert!(result.changed); + assert_eq!(result.action_name, "Set Transition"); + let transition = st.timeline.tracks[0].clips[0] + .transition_out + .as_ref() + .expect("transition stored on outgoing clip"); + assert_eq!(transition.to_clip_id, "b"); + assert_eq!(transition.from_clip_id, "a"); + assert_eq!(transition.kind, TransitionKind::CrossDissolve); + assert_eq!(transition.duration_frames, 20); + + apply(&mut st, EditCommand::Undo, &g).unwrap(); + assert!(st.timeline.tracks[0].clips[0].transition_out.is_none()); + + let error = apply( + &mut st, + EditCommand::SetTransition { + from_clip_id: "b".into(), + to_clip_id: "a".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 10, + }, + &g, + ) + .unwrap_err(); + assert!(matches!(error, EditError::Invalid(_))); + + let oversized = apply( + &mut st, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 21, + }, + &g, + ) + .unwrap_err(); + assert!(matches!(oversized, EditError::Invalid(_))); +} + +#[test] +fn moving_either_side_of_a_transition_prunes_it_and_undo_restores_it() { + let mut st = state(vec![video_track( + "v", + true, + vec![clip("a", 0, 100), clip("b", 100, 40)], + )]); + let g = SeqIdGen::default(); + apply( + &mut st, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 12, + }, + &g, + ) + .unwrap(); + + apply( + &mut st, + EditCommand::MoveClips { + moves: vec![ClipMove { + clip_id: "b".into(), + to_track: 0, + to_frame: 110, + }], + }, + &g, + ) + .unwrap(); + let outgoing = st.timeline.tracks[0] + .clips + .iter() + .find(|clip| clip.id == "a") + .unwrap(); + assert!(outgoing.transition_out.is_none()); + + apply(&mut st, EditCommand::Undo, &g).unwrap(); + let outgoing = st.timeline.tracks[0] + .clips + .iter() + .find(|clip| clip.id == "a") + .unwrap(); + assert_eq!(outgoing.transition_out.as_ref().unwrap().to_clip_id, "b"); +} + // ---- set_keyframes -------------------------------------------------------- #[test] @@ -847,6 +1434,8 @@ fn create_folder_and_move_asset_into_it() { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -1089,6 +1678,11 @@ fn set_color_grade_applies_and_undoes() { let grade = ColorGrade { exposure: 0.5, saturation: 1.2, + hsl_secondary: Some(HslSecondary { + hue_center: 0.65, + hue_shift: 0.15, + ..Default::default() + }), ..Default::default() }; let res = apply( @@ -1108,6 +1702,84 @@ fn set_color_grade_applies_and_undoes() { // Undo restores the cleared grade. apply(&mut st, EditCommand::Undo, &g).unwrap(); assert_eq!(find_clip(&st, "c").color_grade, None); + apply(&mut st, EditCommand::Redo, &g).unwrap(); + assert_eq!(find_clip(&st, "c").color_grade, Some(grade)); +} + +#[test] +fn set_lut_applies_adjusts_removes_and_round_trips_history() { + let mut state = one_clip_state(); + let ids = SeqIdGen::default(); + let reference = + LutReference::new("0123456789abcdef".repeat(4), "Known Transform", 1.0).unwrap(); + apply( + &mut state, + EditCommand::SetLut { + clip_ids: vec!["c".into()], + lut: Some(reference.clone()), + }, + &ids, + ) + .unwrap(); + assert_eq!(find_clip(&state, "c").lut.as_ref(), Some(&reference)); + + let adjusted = LutReference { + intensity: 0.35, + ..reference.clone() + }; + apply( + &mut state, + EditCommand::SetLut { + clip_ids: vec!["c".into()], + lut: Some(adjusted.clone()), + }, + &ids, + ) + .unwrap(); + assert_eq!(find_clip(&state, "c").lut.as_ref(), Some(&adjusted)); + apply(&mut state, EditCommand::Undo, &ids).unwrap(); + assert_eq!(find_clip(&state, "c").lut.as_ref(), Some(&reference)); + apply(&mut state, EditCommand::Redo, &ids).unwrap(); + assert_eq!(find_clip(&state, "c").lut.as_ref(), Some(&adjusted)); + + apply( + &mut state, + EditCommand::SetLut { + clip_ids: vec!["c".into()], + lut: None, + }, + &ids, + ) + .unwrap(); + assert!(find_clip(&state, "c").lut.is_none()); +} + +#[test] +fn set_color_grade_rejects_invalid_without_mutation() { + let mut st = one_clip_state(); + let g = SeqIdGen::default(); + let error = apply( + &mut st, + EditCommand::SetColorGrade { + clip_ids: vec!["c".into()], + grade: Some(ColorGrade { + lift_gamma_gain: LiftGammaGain { + gamma: Rgb::new(0.0, 1.0, 1.0), + ..Default::default() + }, + ..Default::default() + }), + }, + &g, + ) + .expect_err("zero gamma must be rejected before mutation"); + assert_eq!( + error.to_string(), + "invalid color grade: liftGammaGain.gamma.r must be finite and within (0, 4]" + ); + assert_eq!(find_clip(&st, "c").color_grade, None); + assert_eq!(st.version(), 0); + assert!(!st.can_undo()); } #[test] @@ -1227,6 +1899,7 @@ fn set_masks_replaces_list() { }, feather: 0.05, invert: false, + ..Mask::default() }]; let res = apply( &mut st, @@ -1253,6 +1926,29 @@ fn set_masks_replaces_list() { .unwrap(); assert!(res2.changed); assert!(find_clip(&st, "c").masks.is_empty()); + + apply(&mut st, EditCommand::Undo, &g).unwrap(); + assert_eq!(find_clip(&st, "c").masks, masks); + apply(&mut st, EditCommand::Redo, &g).unwrap(); + assert!(find_clip(&st, "c").masks.is_empty()); + + let oversized_polygon = Mask { + shape: MaskShape::Poly { + points: vec![Point2::new(0.5, 0.5); 17], + }, + ..Mask::default() + }; + let err = apply( + &mut st, + EditCommand::SetMasks { + clip_ids: vec!["c".into()], + masks: vec![oversized_polygon], + }, + &g, + ) + .unwrap_err(); + assert!(err.to_string().contains("3..=16 points")); + assert!(find_clip(&st, "c").masks.is_empty()); } #[test] @@ -1260,8 +1956,8 @@ fn set_effects_replaces_chain() { let mut st = one_clip_state(); let g = SeqIdGen::default(); let effects = vec![ - Effect::new("gaussianBlur").with_param("radius", 4.0), - Effect::new("glow").with_param("intensity", 0.6), + Effect::new("grayscale").with_param("amount", 0.4), + Effect::new("sepia").with_param("amount", 0.6), ]; let res = apply( &mut st, @@ -1277,6 +1973,30 @@ fn set_effects_replaces_chain() { assert_eq!(find_clip(&st, "c").effects, effects); } +#[test] +fn set_effects_rejects_unknown_names_and_invalid_parameters_without_history() { + let g = SeqIdGen::default(); + for effect in [ + Effect::new("blur"), + Effect::new("sepia").with_param("radius", 2.0), + Effect::new("invert").with_param("amount", 1.1), + ] { + let mut st = one_clip_state(); + let error = apply( + &mut st, + EditCommand::SetEffects { + clip_ids: vec!["c".into()], + effects: vec![effect], + }, + &g, + ) + .unwrap_err(); + assert!(matches!(error, EditError::Invalid(_))); + assert!(find_clip(&st, "c").effects.is_empty()); + assert_eq!(st.version(), 0); + } +} + #[test] fn advanced_effect_commands_reject_empty_and_missing() { let mut st = one_clip_state(); @@ -1299,7 +2019,7 @@ fn advanced_effect_commands_reject_empty_and_missing() { &mut st, EditCommand::SetEffects { clip_ids: vec!["nope".into()], - effects: vec![Effect::new("blur")] + effects: vec![Effect::new("grayscale")] }, &g ), @@ -1368,6 +2088,8 @@ fn media_entry(id: &str, kind: ClipType, duration_secs: f64) -> MediaManifestEnt source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/src/archive.rs b/crates/opentake-project/src/archive.rs index 841e6a94..1086d70f 100644 --- a/crates/opentake-project/src/archive.rs +++ b/crates/opentake-project/src/archive.rs @@ -404,6 +404,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/src/bundle.rs b/crates/opentake-project/src/bundle.rs index 075a40c6..c903699b 100644 --- a/crates/opentake-project/src/bundle.rs +++ b/crates/opentake-project/src/bundle.rs @@ -174,6 +174,11 @@ impl Project { } /// Decode every project component from one retained root capability. + /// + /// Persisted compatibility is applied by each component's deserializer: + /// missing optional fields receive their legacy defaults, while explicit + /// schema versions are preserved. Saving the returned project therefore + /// writes the decoded state without silently promoting legacy versions. pub fn open_from_root(root: &ProjectRoot) -> Result { Self::open_from_root_with_hook(root, |_| {}) } @@ -192,6 +197,12 @@ impl Project { let (mut timeline, timeline_blockers, timeline_document) = decode_component::(&timeline_bytes, layout::TIMELINE_FILE)?; compatibility::repair_timeline_ids(&mut timeline, &timeline_document); + timeline + .validate_nested_sequences() + .map_err(|reason| ProjectError::InvalidTimeline { + file: layout::TIMELINE_FILE, + reason, + })?; after_component(layout::TIMELINE_FILE); let mut compatibility = ProjectCompatibility::default(); compatibility.extend(timeline_blockers); @@ -396,6 +407,13 @@ impl EncodedProject { /// Produce the exact byte snapshot before any destination path is created. fn prepare(project: &Project) -> Result { project.compatibility.ensure_writable()?; + project + .timeline + .validate_nested_sequences() + .map_err(|reason| ProjectError::InvalidTimeline { + file: layout::TIMELINE_FILE, + reason, + })?; Ok(Self { timeline: encode_component(layout::TIMELINE_FILE, &project.timeline)?, manifest: encode_component(layout::MANIFEST_FILE, &project.manifest)?, diff --git a/crates/opentake-project/src/compatibility.rs b/crates/opentake-project/src/compatibility.rs index 4bf6cf6b..716b9359 100644 --- a/crates/opentake-project/src/compatibility.rs +++ b/crates/opentake-project/src/compatibility.rs @@ -5,8 +5,8 @@ //! that serde cannot see after a Swift-compatible `try?` fallback. use opentake_domain::{ - AnimPair, Clip, Crop, Fill, Keyframe, KeyframeTrack, KeyframeValueWireShape, Rgba, Shadow, - TextStyle, Timeline, Track, Transform, + AnimPair, Clip, Crop, Fill, Keyframe, KeyframeTrack, KeyframeValueWireShape, NestedSequence, + Rgba, Shadow, TextStyle, Timeline, Track, Transform, }; use serde_json::Value; use uuid::Uuid; @@ -27,6 +27,25 @@ pub(crate) struct TimelineFallback { /// and clip ordering; a Track.clips fallback yields an empty decoded vector and /// is therefore skipped safely. pub(crate) fn repair_timeline_ids(timeline: &mut Timeline, document: &Value) { + repair_timeline_ids_inner(timeline, document); +} + +fn repair_timeline_ids_inner(timeline: &mut Timeline, document: &Value) { + if let Some(raw_sequences) = document + .get(Timeline::NESTED_SEQUENCES_WIRE_FIELD) + .and_then(Value::as_array) + { + for (sequence_index, sequence) in timeline.nested_sequences.iter_mut().enumerate() { + let Some(raw_timeline) = raw_sequences + .get(sequence_index) + .and_then(|value| value.get(NestedSequence::TIMELINE_WIRE_FIELD)) + else { + continue; + }; + repair_timeline_ids_inner(&mut sequence.timeline, raw_timeline); + } + } + let Some(raw_tracks) = document .get(Timeline::TRACKS_WIRE_FIELD) .and_then(Value::as_array) @@ -112,6 +131,44 @@ pub(crate) fn scan_timeline( failed_tracks: &[bool], ignored: &mut Vec, ) { + scan_timeline_inner(document, "", file, failed_tracks, ignored); +} + +fn scan_timeline_inner( + document: &Value, + prefix: &str, + file: &str, + failed_tracks: &[bool], + ignored: &mut Vec, +) { + if let Some(sequences) = document + .get(Timeline::NESTED_SEQUENCES_WIRE_FIELD) + .and_then(Value::as_array) + { + for (sequence_index, sequence) in sequences.iter().enumerate() { + let sequence_path = prefixed( + prefix, + &format!("{}.{sequence_index}", Timeline::NESTED_SEQUENCES_WIRE_FIELD), + ); + scan_object_keys( + Some(sequence), + &sequence_path, + NestedSequence::WIRE_FIELDS, + file, + ignored, + ); + if let Some(child) = sequence.get(NestedSequence::TIMELINE_WIRE_FIELD) { + scan_timeline_inner( + child, + &format!("{sequence_path}.{}", NestedSequence::TIMELINE_WIRE_FIELD), + file, + &[], + ignored, + ); + } + } + } + let Some(tracks) = document .get(Timeline::TRACKS_WIRE_FIELD) .and_then(Value::as_array) @@ -120,7 +177,10 @@ pub(crate) fn scan_timeline( }; for (track_index, track) in tracks.iter().enumerate() { - let track_path = format!("{}.{track_index}", Timeline::TRACKS_WIRE_FIELD); + let track_path = prefixed( + prefix, + &format!("{}.{track_index}", Timeline::TRACKS_WIRE_FIELD), + ); for field in Track::TOLERANT_SCALAR_WIRE_FIELDS { scan_future_scalar_shape( track.get(*field), @@ -219,6 +279,14 @@ pub(crate) fn scan_timeline( } } +fn prefixed(prefix: &str, suffix: &str) -> String { + if prefix.is_empty() { + suffix.to_string() + } else { + format!("{prefix}.{suffix}") + } +} + fn scan_decodable_clip_unknowns(clip: &Value, path: &str, file: &str, ignored: &mut Vec) { let Ok(bytes) = serde_json::to_vec(clip) else { return; diff --git a/crates/opentake-project/src/edl.rs b/crates/opentake-project/src/edl.rs index bde2bdf6..ba4eaa68 100644 --- a/crates/opentake-project/src/edl.rs +++ b/crates/opentake-project/src/edl.rs @@ -48,7 +48,7 @@ //! no cross-platform tape/source-timecode reader — see `fcpxml.rs`); the source //! window is `[trim_start, trim_start + source_frames_consumed)`. -use opentake_domain::{Clip, MediaManifest, MediaResolver, Timeline, Track}; +use opentake_domain::{Clip, ClipType, MediaManifest, MediaResolver, Timeline, Track}; /// Reel name for every event. Real source-tape names need a tape-timecode /// reader OpenTake lacks; `AX` ("auxiliary") is the CMX3600 convention for @@ -100,14 +100,30 @@ impl Builder<'_> { out } - /// Clips of the topmost video track, sorted by start frame. CMX3600 holds a - /// single video track, so we pick the first visual track in timeline order. + /// Clips of the topmost editorial-media track, sorted by start frame. + /// + /// `ClipType::is_visual()` also includes text and Lottie tracks. Those are + /// overlays rather than CMX3600 video events, so selecting the first visual + /// track would export captions as `Offline` clips and hide the real video + /// track below. Pick the first track that actually contains video/image + /// media and omit overlay clips even when a mixed visual track contains + /// them. fn top_video_clips(&self) -> Vec { - let track: Option<&Track> = self.timeline.tracks.iter().find(|t| t.kind.is_visual()); - let Some(track) = track else { - return Vec::new(); - }; - let mut clips: Vec = track.clips.clone(); + let mut clips = self + .timeline + .tracks + .iter() + .filter(|track: &&Track| track.kind.is_visual()) + .find_map(|track| { + let clips: Vec = track + .clips + .iter() + .filter(|clip| matches!(clip.media_type, ClipType::Video | ClipType::Image)) + .cloned() + .collect(); + (!clips.is_empty()).then_some(clips) + }) + .unwrap_or_default(); clips.sort_by_key(|c| c.start_frame); clips } @@ -183,7 +199,7 @@ fn format_timecode(frame: i32, fps: i32, drop_frame: bool) -> String { #[cfg(test)] mod tests { use super::*; - use opentake_domain::{ClipType, MediaManifestEntry, MediaSource}; + use opentake_domain::{MediaManifestEntry, MediaSource}; fn entry(id: &str, name: &str, kind: ClipType, duration: f64) -> MediaManifestEntry { MediaManifestEntry { @@ -199,6 +215,8 @@ mod tests { source_height: Some(1080), source_fps: Some(30.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -367,6 +385,37 @@ mod tests { assert!(!edl.contains("002 AX")); } + #[test] + fn caption_overlay_track_does_not_shadow_video_track() { + let mut tl = Timeline::new(); + tl.fps = 30; + + let mut captions = Track::new("captions", ClipType::Text); + let mut caption = Clip::new("caption", "caption-media", 0, 90); + caption.media_type = ClipType::Text; + caption.source_clip_type = ClipType::Text; + caption.text_content = Some("Do not export me as Offline".into()); + captions.clips.push(caption); + + let mut video = Track::new("video", ClipType::Video); + video.clips.push(Clip::new("shot", "v1", 0, 120)); + + // Top-to-bottom order matches the real editor: captions above video. + tl.tracks.push(captions); + tl.tracks.push(video); + + let edl = export_edl( + &tl, + &manifest(vec![entry("v1", "talking-head.mp4", ClipType::Video, 4.0)]), + ); + + assert!(edl.contains("* FROM CLIP NAME: talking-head.mp4")); + assert!(!edl.contains("Do not export me")); + assert!(!edl.contains("* FROM CLIP NAME: Offline")); + assert!(edl.contains("001 AX")); + assert!(!edl.contains("002 AX")); + } + #[test] fn empty_timeline_is_header_only() { let tl = Timeline::new(); diff --git a/crates/opentake-project/src/error.rs b/crates/opentake-project/src/error.rs index d0543a58..81bad137 100644 --- a/crates/opentake-project/src/error.rs +++ b/crates/opentake-project/src/error.rs @@ -50,6 +50,10 @@ pub enum ProjectError { blockers: Vec, }, + /// The decoded timeline graph is structurally unsafe to edit or render. + #[error("invalid timeline graph in {file}: {reason}")] + InvalidTimeline { file: &'static str, reason: String }, + /// Publication could not install the staged bundle or restore the prior /// target. The retained backup is deliberately left in place and the next /// save attempt will recover it before doing new work. diff --git a/crates/opentake-project/src/fcpxml.rs b/crates/opentake-project/src/fcpxml.rs index 17985e53..9f9d6dd7 100644 --- a/crates/opentake-project/src/fcpxml.rs +++ b/crates/opentake-project/src/fcpxml.rs @@ -1194,6 +1194,8 @@ mod tests { source_height: Some(1080), source_fps: Some(30.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, @@ -1272,6 +1274,8 @@ mod tests { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/src/fcpxml_modern_tests.rs b/crates/opentake-project/src/fcpxml_modern_tests.rs index a6cb97de..e32e3bb4 100644 --- a/crates/opentake-project/src/fcpxml_modern_tests.rs +++ b/crates/opentake-project/src/fcpxml_modern_tests.rs @@ -20,6 +20,8 @@ fn entry(id: &str, name: &str, kind: ClipType, duration: f64) -> MediaManifestEn source_height: Some(1080), source_fps: Some(30.0), has_audio: Some(kind == ClipType::Video || kind == ClipType::Audio), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/src/layout.rs b/crates/opentake-project/src/layout.rs index 4c096214..d50b1771 100644 --- a/crates/opentake-project/src/layout.rs +++ b/crates/opentake-project/src/layout.rs @@ -28,6 +28,9 @@ pub const THUMBNAIL_FILE: &str = "thumbnail.jpg"; /// convention point inside this directory. pub const MEDIA_DIR: &str = "media"; +/// `media/luts/` — content-addressed, project-managed `.cube` files. +pub const LUTS_DIR: &str = "luts"; + /// `chat-sessions/` — one `.json` per agent chat session. /// /// OpenTake-specific: upstream stores these under `chat/` @@ -61,6 +64,11 @@ pub fn media_dir(bundle: &Path) -> PathBuf { bundle.join(MEDIA_DIR) } +/// Absolute path to the project-managed LUT directory. +pub fn luts_dir(bundle: &Path) -> PathBuf { + media_dir(bundle).join(LUTS_DIR) +} + /// Absolute path to the `chat-sessions/` directory inside `bundle`. pub fn chat_sessions_dir(bundle: &Path) -> PathBuf { bundle.join(CHAT_SESSIONS_DIR) diff --git a/crates/opentake-project/src/otio.rs b/crates/opentake-project/src/otio.rs index 6d446e92..24ea2f13 100644 --- a/crates/opentake-project/src/otio.rs +++ b/crates/opentake-project/src/otio.rs @@ -279,6 +279,8 @@ mod tests { source_height: Some(1080), source_fps: Some(24.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/src/project_root.rs b/crates/opentake-project/src/project_root.rs index 4faa1e9c..e0184918 100644 --- a/crates/opentake-project/src/project_root.rs +++ b/crates/opentake-project/src/project_root.rs @@ -436,6 +436,99 @@ impl ProjectRoot { Ok(()) } + /// Read one project-managed LUT through retained no-follow directories. + pub fn read_lut(&self, name: &str, max_bytes: usize) -> Result>> { + validate_leaf(name).map_err(|error| { + ProjectError::io( + self.path + .join(crate::layout::MEDIA_DIR) + .join(crate::layout::LUTS_DIR) + .join(name), + error, + ) + })?; + let Some(directory) = self.luts_directory(false)? else { + return Ok(None); + }; + let path = self + .path + .join(crate::layout::MEDIA_DIR) + .join(crate::layout::LUTS_DIR) + .join(name); + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + #[cfg(unix)] + options.custom_flags(libc::O_NONBLOCK); + let mut file = match directory.open_with(name, &options) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(ProjectError::io(path, error)), + }; + let metadata = file + .metadata() + .map_err(|error| ProjectError::io(&path, error))?; + if !metadata.is_file() || metadata.len() > max_bytes as u64 { + return Err(ProjectError::io( + path, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "LUT is not a bounded nofollow regular file", + ), + )); + } + let mut bytes = Vec::with_capacity(metadata.len() as usize); + Read::by_ref(&mut file) + .take(max_bytes as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|error| ProjectError::io(&path, error))?; + if bytes.len() > max_bytes { + return Err(ProjectError::io( + path, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "LUT grew beyond the configured byte limit", + ), + )); + } + Ok(Some(bytes)) + } + + /// Atomically publish one validated, content-addressed LUT under + /// `media/luts/`. Callers validate both the bytes and digest before entry. + pub fn write_lut_atomic(&self, name: &str, bytes: &[u8]) -> Result<()> { + validate_leaf(name).map_err(|error| { + ProjectError::io( + self.path + .join(crate::layout::MEDIA_DIR) + .join(crate::layout::LUTS_DIR) + .join(name), + error, + ) + })?; + let directory = self + .luts_directory(true)? + .expect("create=true returns a directory"); + let directory_path = self + .path + .join(crate::layout::MEDIA_DIR) + .join(crate::layout::LUTS_DIR); + let tmp_name = unique_temp_name(name); + let mut tmp = TransactionLeaf::create(&directory, &tmp_name) + .map_err(|error| ProjectError::io(directory_path.join(&tmp_name), error))?; + tmp.handle + .as_file_mut() + .write_all(bytes) + .map_err(|error| ProjectError::io(directory_path.join(&tmp_name), error))?; + tmp.handle + .as_file() + .sync_all() + .map_err(|error| ProjectError::io(directory_path.join(&tmp_name), error))?; + tmp.replace(&directory, Path::new(name)) + .map_err(|error| ProjectError::io(directory_path.join(name), error))?; + tmp.cleanup_on_drop = false; + Ok(()) + } + /// List no-follow regular leaves in `chat-sessions/`. Callers own the /// filename policy (for example selecting only `.json`). pub fn list_chat_session_files(&self, max_entries: usize) -> Result> { @@ -508,6 +601,71 @@ impl ProjectRoot { .map(Some) .map_err(|error| ProjectError::io(path, error)) } + + fn luts_directory(&self, create: bool) -> Result> { + let media_path = self.path.join(crate::layout::MEDIA_DIR); + match self.dir.symlink_metadata(crate::layout::MEDIA_DIR) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => { + return Err(ProjectError::io( + media_path, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "media must be a nofollow directory", + ), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound && create => { + self.dir + .create_dir(crate::layout::MEDIA_DIR) + .or_else(|error| { + if error.kind() == std::io::ErrorKind::AlreadyExists { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| ProjectError::io(&media_path, error))?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(ProjectError::io(&media_path, error)), + } + let media = self + .dir + .open_dir_nofollow(crate::layout::MEDIA_DIR) + .map_err(|error| ProjectError::io(&media_path, error))?; + let luts_path = media_path.join(crate::layout::LUTS_DIR); + match media.symlink_metadata(crate::layout::LUTS_DIR) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => { + return Err(ProjectError::io( + luts_path, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "luts must be a nofollow directory", + ), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound && create => { + media + .create_dir(crate::layout::LUTS_DIR) + .or_else(|error| { + if error.kind() == std::io::ErrorKind::AlreadyExists { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| ProjectError::io(&luts_path, error))?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(ProjectError::io(&luts_path, error)), + } + media + .open_dir_nofollow(crate::layout::LUTS_DIR) + .map(Some) + .map_err(|error| ProjectError::io(luts_path, error)) + } } /// One complete-bundle sibling publication. The persistent lock leaf @@ -642,15 +800,33 @@ impl BundlePublisher { } pub(crate) fn publish(mut self) -> Result { - #[cfg(test)] - if FAIL_PUBLISH_AFTER_BACKUP.with(|fail| fail.replace(false)) { - return self.publish_with_hook(|| { - Err(std::io::Error::other( - "injected publication failure after backup", - )) - }); + let result = { + #[cfg(test)] + if FAIL_PUBLISH_AFTER_BACKUP.with(|fail| fail.replace(false)) { + self.publish_with_hook(|| { + Err(std::io::Error::other( + "injected publication failure after backup", + )) + }) + } else { + self.publish_with_hook(|| Ok(())) + } + + #[cfg(not(test))] + self.publish_with_hook(|| Ok(())) + }; + + // A caller may immediately start the next complete-bundle save while + // retaining the returned ProjectRoot. Make that successful handoff + // explicit instead of depending on the two cloned lock handles being + // dropped at the end of this function. Error paths retain the lock + // through Drop's staged-artifact cleanup. Closing the handles remains + // the fallback; an unlock error cannot safely turn an already + // committed publication into a reported failure. + if result.is_ok() { + let _ = self._lock.unlock(); } - self.publish_with_hook(|| Ok(())) + result } fn publish_with_hook( @@ -2176,6 +2352,38 @@ mod tests { .exists()); } + #[test] + fn publish_releases_the_transaction_lock_before_returning() { + let tmp = TmpDir::new("publish-lock-handoff"); + let target = tmp.path().join("Existing.opentake"); + fs::create_dir_all(&target).unwrap(); + fs::write(target.join("project.json"), b"initial timeline").unwrap(); + let lock_path = tmp.path().join(".Existing.opentake.opentake-lock"); + + for generation in 0..32 { + let publisher = ProjectRoot::begin_replace(&target).unwrap(); + let expected = format!("timeline generation {generation}"); + publisher + .stage() + .write_atomic("project.json", expected.as_bytes()) + .unwrap(); + let root = publisher.publish().unwrap(); + + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + lock.try_lock() + .expect("publish must hand off its transaction lock before returning"); + lock.unlock().unwrap(); + assert_eq!( + root.read_optional("project.json").unwrap().unwrap(), + expected.as_bytes() + ); + } + } + #[test] fn postcommit_backup_cleanup_failure_returns_success_and_recovers_on_retry() { let tmp = TmpDir::new("postcommit-cleanup"); diff --git a/crates/opentake-project/src/safe_fs/tests.rs b/crates/opentake-project/src/safe_fs/tests.rs index 9deb61b3..bdd69b84 100644 --- a/crates/opentake-project/src/safe_fs/tests.rs +++ b/crates/opentake-project/src/safe_fs/tests.rs @@ -109,6 +109,119 @@ fn component_accepts_safe_names_and_rejects_too_long_and_unsafe_names() { } } +#[cfg(windows)] +struct WindowsContractDir(std::path::PathBuf); + +#[cfg(windows)] +impl WindowsContractDir { + fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "opentake-windows-contract-{}-{id}", + std::process::id() + )); + std::fs::create_dir(&path).expect("create Windows contract fixture"); + Self(path) + } +} + +#[cfg(windows)] +impl Drop for WindowsContractDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[cfg(windows)] +#[test] +fn windows_contract() { + use std::io::SeekFrom; + + let fixture = WindowsContractDir::new(); + let root = capture_absolute_directory(&fixture.0, DirectoryAccess::MutateChildren) + .expect("capture local Windows fixture"); + + let stage_name = ComponentName::new("stage").unwrap(); + let quarantine_name = ComponentName::new("quarantine").unwrap(); + let nested_name = ComponentName::new("nested").unwrap(); + let leaf_name = ComponentName::new("leaf.bin").unwrap(); + let stage = create_stage_dir_new(&root, &stage_name, CreatePermissions::Inherit) + .expect("create retained stage"); + let nested = create_dir_new( + stage.directory(), + &nested_name, + CreatePermissions::Inherit, + DirectoryAccess::MutateChildren, + ) + .expect("create nested directory"); + let mut leaf = create_file_new(&nested, &leaf_name, CreatePermissions::Inherit) + .expect("create retained file"); + leaf.write_all(b"windows-capability-relative") + .expect("write retained file"); + leaf.flush().expect("flush retained file"); + leaf.sync_all().expect("sync retained file"); + leaf.seek(SeekFrom::Start(0)).expect("rewind retained file"); + let mut bytes = [0; 27]; + assert_eq!(leaf.read(&mut bytes).unwrap(), bytes.len()); + assert_eq!(&bytes, b"windows-capability-relative"); + drop(leaf); + drop(nested); + + let quarantine = quarantine_stage(stage, &root, quarantine_name.clone()) + .expect("quarantine retained stage without replacement"); + cleanup_quarantined_tree(quarantine).expect("delete quarantined tree by retained handles"); + assert!(matches!( + query_child_nofollow(&root, &quarantine_name).unwrap(), + ChildState::Absent + )); + + let published_name = ComponentName::new("published").unwrap(); + let published_stage_name = ComponentName::new("publish-stage").unwrap(); + let published_stage = + create_stage_dir_new(&root, &published_stage_name, CreatePermissions::Inherit) + .expect("create publish stage"); + publish_stage_noreplace(published_stage, &root, published_name.clone()) + .expect("publish retained stage without replacement"); + assert!(matches!( + query_child_nofollow(&root, &published_name).unwrap(), + ChildState::Present(EntryMetadata { + kind: EntryKind::Directory, + .. + }) + )); + + let collision_stage_name = ComponentName::new("collision-stage").unwrap(); + let collision_stage = + create_stage_dir_new(&root, &collision_stage_name, CreatePermissions::Inherit) + .expect("create collision stage"); + assert!(matches!( + publish_stage_noreplace(collision_stage, &root, published_name), + Err(SafeFsError::AlreadyExists { + operation: SafeFsOperation::RenameNoReplaceSameParent, + }) + )); + assert!(matches!( + query_child_nofollow(&root, &collision_stage_name).unwrap(), + ChildState::Present(EntryMetadata { + kind: EntryKind::Directory, + .. + }) + )); +} + +#[cfg(windows)] +#[test] +fn synchronous_nt_pending_is_invariant_error() { + assert!(matches!( + super::windows::synchronous_pending_contract_for_test(), + Err(SafeFsError::Os { + operation: SafeFsOperation::ReadFile, + raw: RawOsError::NtStatus { .. }, + }) + )); +} + #[cfg(any(target_os = "linux", target_os = "macos"))] mod unix_contract { use super::super::capability::CleanupCapability; diff --git a/crates/opentake-project/src/safe_fs/windows.rs b/crates/opentake-project/src/safe_fs/windows.rs index e5be68c7..a0865f60 100644 --- a/crates/opentake-project/src/safe_fs/windows.rs +++ b/crates/opentake-project/src/safe_fs/windows.rs @@ -23,22 +23,24 @@ use windows_sys::Win32::Foundation::{ STATUS_OBJECT_NAME_NOT_FOUND, STATUS_OBJECT_PATH_NOT_FOUND, STATUS_OBJECT_TYPE_MISMATCH, STATUS_PENDING, STATUS_REPARSE_POINT_ENCOUNTERED, STATUS_SHARING_VIOLATION, UNICODE_STRING, }; -use windows_sys::Win32::Security::SECURITY_DESCRIPTOR; +use windows_sys::Win32::Security::*; use windows_sys::Win32::Storage::FileSystem::{ CreateFileW, FileAttributeTagInfo, FileIdInfo, FileRemoteProtocolInfo, FileStandardInfo, GetDriveTypeW, GetFileInformationByHandleEx, GetVolumeInformationByHandleW, GetVolumeNameForVolumeMountPointW, GetVolumePathNameW, DELETE, FILE_ACCESS_RIGHTS, - FILE_ADD_FILE, FILE_ADD_SUBDIRECTORY, FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_REPARSE_POINT, - FILE_ATTRIBUTE_TAG_INFO, FILE_DELETE_CHILD, FILE_FLAGS_AND_ATTRIBUTES, - FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_ID_INFO, FILE_LIST_DIRECTORY, - FILE_READ_ATTRIBUTES, FILE_READ_DATA, FILE_REMOTE_PROTOCOL_INFO, FILE_SHARE_MODE, - FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_STANDARD_INFO, FILE_TRAVERSE, FILE_WRITE_DATA, - GET_FILEEX_INFO_LEVELS, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, OPEN_EXISTING, READ_CONTROL, - SYNCHRONIZE, + FILE_ADD_FILE, FILE_ADD_SUBDIRECTORY, FILE_ALL_ACCESS, FILE_ATTRIBUTE_NORMAL, + FILE_ATTRIBUTE_REPARSE_POINT, FILE_ATTRIBUTE_TAG_INFO, FILE_DELETE_CHILD, + FILE_FLAGS_AND_ATTRIBUTES, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_ID_INFO, FILE_LIST_DIRECTORY, FILE_READ_ATTRIBUTES, FILE_READ_DATA, + FILE_REMOTE_PROTOCOL_INFO, FILE_SHARE_MODE, FILE_SHARE_READ, FILE_SHARE_WRITE, + FILE_STANDARD_INFO, FILE_TRAVERSE, FILE_WRITE_DATA, GET_FILEEX_INFO_LEVELS, + MAXIMUM_REPARSE_DATA_BUFFER_SIZE, OPEN_EXISTING, READ_CONTROL, SYNCHRONIZE, }; use windows_sys::Win32::System::Ioctl::FSCTL_GET_REPARSE_POINT; -use windows_sys::Win32::System::SystemServices::FILE_CS_FLAG_CASE_SENSITIVE_DIR; -use windows_sys::Win32::System::Threading::GetCurrentProcess; +use windows_sys::Win32::System::SystemServices::{ + ACCESS_ALLOWED_ACE_TYPE, FILE_CS_FLAG_CASE_SENSITIVE_DIR, SECURITY_DESCRIPTOR_REVISION, +}; +use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; use windows_sys::Win32::System::IO::IO_STATUS_BLOCK; const SHARE: FILE_SHARE_MODE = FILE_SHARE_READ | FILE_SHARE_WRITE; @@ -48,6 +50,7 @@ const DIRECTORY_BUFFER_BYTES: usize = 64 * 1024; const REPARSE_HEADER_BYTES: usize = 8; const STATUS_SUCCESS: NTSTATUS = 0; const BOOL_FALSE: BOOL = 0; +const BOOL_TRUE: BOOL = 1; const DRIVE_REMOVABLE: u32 = 2; const DRIVE_FIXED: u32 = 3; @@ -340,6 +343,16 @@ fn complete_nt( Ok(()) } +#[cfg(test)] +pub(super) fn synchronous_pending_contract_for_test() -> Result<()> { + let mut iosb = IO_STATUS_BLOCK::default(); + // Initialize the Status member even though `complete_nt` must reject the + // returned STATUS_PENDING before reading it. + iosb.Anonymous.Status = STATUS_SUCCESS; + iosb.Information = usize::MAX; + complete_nt(SafeFsOperation::ReadFile, STATUS_PENDING, &iosb) +} + #[allow(clippy::too_many_arguments)] // Mirrors the fixed NtCreateFile operation contract. fn nt_create_relative( parent: HANDLE, @@ -1319,18 +1332,496 @@ fn require_mutation(parent: &DirectoryAuthority, operation: SafeFsOperation) -> } } -fn owner_only_refusal() -> Result { - Err(SafeFsError::UnsupportedSecureFilesystem { +struct OwnerOnlySecurity { + sid: Vec, + _acl: Vec, + descriptor: Box, + ace_flags: ACE_FLAGS, +} + +impl OwnerOnlySecurity { + fn new(directory: bool) -> Result { + let operation = SafeFsOperation::VerifySecurityDescriptor; + let mut token_raw = null_mut(); + // SAFETY: the current-process pseudo-handle is valid and the output pointer is writable. + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token_raw) } == 0 { + return Err(last_win32(operation)); + } + let token = OwnedHandle::new(token_raw, operation)?; + let mut needed = 0u32; + // SAFETY: documented sizing call with a null output buffer. + let first = + unsafe { GetTokenInformation(token.raw(), TokenOwner, null_mut(), 0, &mut needed) }; + if first != 0 + || unsafe { GetLastError() } + != windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER + || needed < size_of::() as u32 + { + return Err(last_win32(operation)); + } + let mut token_words = vec![0usize; (needed as usize).div_ceil(size_of::())]; + // SAFETY: aligned storage is writable for exactly `needed` bytes. + if unsafe { + GetTokenInformation( + token.raw(), + TokenOwner, + token_words.as_mut_ptr().cast(), + needed, + &mut needed, + ) + } == 0 + { + return Err(last_win32(operation)); + } + // SAFETY: the successful TokenOwner query initialized a TOKEN_OWNER value. + let owner = unsafe { (*(token_words.as_ptr().cast::())).Owner }; + if owner.is_null() || unsafe { IsValidSid(owner) } == 0 { + return Err(last_win32(operation)); + } + // SAFETY: `owner` is a validated SID returned in the live token buffer. + let sid_len = usize::try_from(unsafe { GetLengthSid(owner) }).map_err(|_| { + SafeFsError::InvalidNativeBuffer { + operation, + reason: NativeBufferReason::SecurityDescriptorMalformed, + } + })?; + let mut sid = vec![0usize; sid_len.div_ceil(size_of::())]; + // SAFETY: destination capacity is at least sid_len and owner is a validated SID. + if unsafe { CopySid(sid_len as u32, sid.as_mut_ptr().cast(), owner) } == 0 { + return Err(last_win32(operation)); + } + drop(token_words); + drop(token); + + let acl_bytes = size_of::() + .checked_add(size_of::() - size_of::()) + .and_then(|value| value.checked_add(sid_len)) + .ok_or(SafeFsError::InvalidNativeBuffer { + operation, + reason: NativeBufferReason::LengthOverflow, + })?; + let acl_len = u32::try_from(acl_bytes).map_err(|_| SafeFsError::InvalidNativeBuffer { + operation, + reason: NativeBufferReason::LengthOverflow, + })?; + if acl_bytes > u16::MAX as usize { + return Err(SafeFsError::InvalidNativeBuffer { + operation, + reason: NativeBufferReason::LengthOverflow, + }); + } + let mut acl = vec![0usize; acl_bytes.div_ceil(size_of::())]; + let ace_flags = if directory { + OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE + } else { + 0 + }; + // SAFETY: aligned ACL storage and the copied SID remain live inside Self. + if unsafe { InitializeAcl(acl.as_mut_ptr().cast(), acl_len, ACL_REVISION) } == 0 + || unsafe { + AddAccessAllowedAceEx( + acl.as_mut_ptr().cast(), + ACL_REVISION, + ace_flags, + FILE_ALL_ACCESS, + sid.as_mut_ptr().cast(), + ) + } == 0 + { + return Err(last_win32(operation)); + } + // SAFETY: SECURITY_DESCRIPTOR is a C POD initialized immediately below. + let mut descriptor = Box::::new(unsafe { std::mem::zeroed() }); + // SAFETY: the boxed descriptor has a stable address and ACL storage remains owned by Self. + if unsafe { + InitializeSecurityDescriptor( + (&mut *descriptor as *mut SECURITY_DESCRIPTOR).cast(), + SECURITY_DESCRIPTOR_REVISION, + ) + } == 0 + || unsafe { + SetSecurityDescriptorDacl( + (&mut *descriptor as *mut SECURITY_DESCRIPTOR).cast(), + BOOL_TRUE, + acl.as_mut_ptr().cast(), + BOOL_FALSE, + ) + } == 0 + || unsafe { + SetSecurityDescriptorControl( + (&mut *descriptor as *mut SECURITY_DESCRIPTOR).cast(), + SE_DACL_PROTECTED, + SE_DACL_PROTECTED, + ) + } == 0 + { + return Err(last_win32(operation)); + } + Ok(Self { + sid, + _acl: acl, + descriptor, + ace_flags, + }) + } + + fn descriptor_ptr(&self) -> *const SECURITY_DESCRIPTOR { + &*self.descriptor + } +} + +fn malformed_security() -> SafeFsError { + SafeFsError::InvalidNativeBuffer { operation: SafeFsOperation::VerifySecurityDescriptor, - reason: SecureFilesystemReason::UnsupportedTarget, - }) + reason: NativeBufferReason::SecurityDescriptorMalformed, + } } -fn require_inherited_permissions(permissions: CreatePermissions) -> Result<()> { - match permissions { - CreatePermissions::Inherit => Ok(()), - CreatePermissions::OwnerOnly => owner_only_refusal(), +fn checked_subslice( + base: usize, + length: usize, + pointer: usize, + needed: usize, +) -> Result> { + let end = base.checked_add(length).ok_or_else(malformed_security)?; + let pointer_end = pointer.checked_add(needed).ok_or_else(malformed_security)?; + if pointer < base || pointer_end > end { + return Err(malformed_security()); } + Ok(pointer - base..pointer_end - base) +} + +fn checked_sid_length(buffer: &[u8], sid: *const c_void) -> Result { + const SID_PREFIX: usize = 8; + let range = checked_subslice( + buffer.as_ptr() as usize, + buffer.len(), + sid as usize, + SID_PREFIX, + )?; + let count = usize::from(buffer[range.start + 1]); + let length = SID_PREFIX + .checked_add( + count + .checked_mul(size_of::()) + .ok_or_else(malformed_security)?, + ) + .ok_or_else(malformed_security)?; + checked_subslice(buffer.as_ptr() as usize, buffer.len(), sid as usize, length)?; + // SAFETY: the SID prefix and every declared sub-authority are inside buffer. + if unsafe { IsValidSid(sid.cast_mut()) } == 0 { + return Err(malformed_security()); + } + // SAFETY: IsValidSid accepted the fully bounded SID. + if usize::try_from(unsafe { GetLengthSid(sid.cast_mut()) }).map_err(|_| malformed_security())? + != length + { + return Err(malformed_security()); + } + Ok(length) +} + +fn verify_single_owner_ace( + descriptor_bytes: &[u8], + dacl: *mut ACL, + acl_bytes_in_use: usize, + ace: *mut c_void, + expected: &OwnerOnlySecurity, +) -> Result<()> { + let dacl_start = dacl as usize; + let dacl_range = checked_subslice( + descriptor_bytes.as_ptr() as usize, + descriptor_bytes.len(), + dacl_start, + acl_bytes_in_use.max(size_of::()), + )?; + if acl_bytes_in_use < size_of::() || dacl_range.len() != acl_bytes_in_use { + return Err(malformed_security()); + } + let ace_start = ace as usize; + checked_subslice( + dacl_start, + acl_bytes_in_use, + ace_start, + size_of::(), + )?; + // SAFETY: only the fixed ACE header bytes were bounds checked; read unaligned. + let header = + unsafe { std::ptr::read_unaligned(ace.cast::()) }; + if u32::from(header.AceType) != ACCESS_ALLOWED_ACE_TYPE { + return Err(malformed_security()); + } + let ace_size = usize::from(header.AceSize); + let sid_offset = offset_of!(ACCESS_ALLOWED_ACE, SidStart); + if ace_size < sid_offset.checked_add(8).ok_or_else(malformed_security)? { + return Err(malformed_security()); + } + checked_subslice(dacl_start, acl_bytes_in_use, ace_start, ace_size)?; + let sid_ptr = ace_start + .checked_add(sid_offset) + .ok_or_else(malformed_security)? as *const c_void; + let sid_length = checked_sid_length(descriptor_bytes, sid_ptr)?; + if sid_offset + .checked_add(sid_length) + .ok_or_else(malformed_security)? + != ace_size + { + return Err(malformed_security()); + } + // SAFETY: ACE type, size, ACL bounds and SID range were established above. + let allowed = unsafe { std::ptr::read_unaligned(ace.cast::()) }; + let expected_flags = u8::try_from(expected.ace_flags).map_err(|_| malformed_security())?; + if allowed.Header.AceFlags != expected_flags + || allowed.Mask != FILE_ALL_ACCESS + || unsafe { EqualSid(sid_ptr.cast_mut(), expected.sid.as_ptr().cast_mut().cast()) } == 0 + { + return Err(malformed_security()); + } + Ok(()) +} + +fn verify_owner_only(handle: HANDLE, expected: &OwnerOnlySecurity) -> Result<()> { + let operation = SafeFsOperation::VerifySecurityDescriptor; + let information = OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION; + let mut needed = 0u32; + // SAFETY: documented sizing call against a retained handle. + unsafe { GetKernelObjectSecurity(handle, information, null_mut(), 0, &mut needed) }; + if unsafe { GetLastError() } != windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER { + return Err(last_win32(operation)); + } + let mut words = vec![0usize; (needed as usize).div_ceil(size_of::())]; + // SAFETY: aligned storage is writable for exactly needed bytes. + if unsafe { + GetKernelObjectSecurity( + handle, + information, + words.as_mut_ptr().cast(), + needed, + &mut needed, + ) + } == 0 + { + return Err(last_win32(operation)); + } + // SAFETY: the successful query initialized exactly `needed` bytes. + let descriptor_bytes = + unsafe { std::slice::from_raw_parts_mut(words.as_mut_ptr().cast::(), needed as usize) }; + if descriptor_bytes.len() < size_of::() { + return Err(malformed_security()); + } + let descriptor = descriptor_bytes.as_mut_ptr().cast::(); + let mut control = 0u16; + let mut revision = 0u32; + let mut owner = null_mut(); + let mut owner_defaulted = BOOL_FALSE; + let mut dacl = null_mut(); + let mut present = BOOL_FALSE; + let mut defaulted = BOOL_FALSE; + // SAFETY: the kernel returned a self-relative descriptor in aligned storage. + if unsafe { GetSecurityDescriptorControl(descriptor.cast(), &mut control, &mut revision) } == 0 + || unsafe { + GetSecurityDescriptorOwner(descriptor.cast(), &mut owner, &mut owner_defaulted) + } == 0 + || unsafe { + GetSecurityDescriptorDacl(descriptor.cast(), &mut present, &mut dacl, &mut defaulted) + } == 0 + || control & SE_DACL_PROTECTED == 0 + || owner_defaulted != BOOL_FALSE + || present == BOOL_FALSE + || defaulted != BOOL_FALSE + || dacl.is_null() + || owner.is_null() + { + return Err(malformed_security()); + } + #[cfg(test)] + let descriptor_fixture = take_owner_descriptor_fixture(); + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::NullOwner) { + owner = null_mut(); + } + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::InvalidOwner) { + owner = descriptor_bytes + .as_mut_ptr() + .wrapping_add(descriptor_bytes.len() - 1) + .cast(); + } + if owner.is_null() { + return Err(malformed_security()); + } + checked_sid_length(descriptor_bytes, owner.cast_const())?; + // SAFETY: owner SID is fully bounded and validated in descriptor_bytes. + if unsafe { EqualSid(owner, expected.sid.as_ptr().cast_mut().cast()) } == 0 { + return Err(malformed_security()); + } + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::DaclOutOfRange) { + dacl = descriptor_bytes + .as_mut_ptr() + .wrapping_add(descriptor_bytes.len() + 1) + .cast(); + } + checked_subslice( + descriptor_bytes.as_ptr() as usize, + descriptor_bytes.len(), + dacl as usize, + size_of::(), + )?; + let mut acl_info = ACL_SIZE_INFORMATION::default(); + // SAFETY: the DACL fixed header is bounded and output is writable. + if unsafe { + GetAclInformation( + dacl, + (&mut acl_info as *mut ACL_SIZE_INFORMATION).cast(), + size_of::() as u32, + AclSizeInformation, + ) + } == 0 + { + return Err(malformed_security()); + } + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::WrongAceCount) { + acl_info.AceCount = 2; + } + if acl_info.AceCount != 1 { + return Err(malformed_security()); + } + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::AclBytesOutOfRange) { + acl_info.AclBytesInUse = u32::MAX; + } + let acl_bytes_in_use = + usize::try_from(acl_info.AclBytesInUse).map_err(|_| malformed_security())?; + checked_subslice( + descriptor_bytes.as_ptr() as usize, + descriptor_bytes.len(), + dacl as usize, + acl_bytes_in_use.max(size_of::()), + )?; + let mut ace = null_mut(); + // SAFETY: the ACL and AclBytesInUse are bounded inside descriptor storage. + if unsafe { GetAce(dacl, 0, &mut ace) } == 0 || ace.is_null() { + return Err(last_win32(operation)); + } + #[cfg(test)] + if descriptor_fixture == Some(OwnerDescriptorFixture::AceOutOfRange) { + ace = descriptor_bytes + .as_mut_ptr() + .wrapping_add(descriptor_bytes.len() + 1) + .cast(); + } + #[cfg(test)] + if let Some(fixture) = descriptor_fixture { + // SAFETY: GetAce returned storage inside the already bounded single-entry + // ACL. Mutations remain inside that allocation and are consumed by the + // release bounds-first verifier before any kernel call. + unsafe { + let header = ace.cast::(); + match fixture { + OwnerDescriptorFixture::WrongAceType => (*header).AceType = 0x7f, + OwnerDescriptorFixture::UndersizedAce => { + (*header).AceSize = + size_of::() as u16; + } + OwnerDescriptorFixture::OversizedSid => { + let sid = (ace as *mut u8).add(offset_of!(ACCESS_ALLOWED_ACE, SidStart)); + *sid.add(1) = u8::MAX; + } + OwnerDescriptorFixture::InvalidSid => { + let sid = (ace as *mut u8).add(offset_of!(ACCESS_ALLOWED_ACE, SidStart)); + *sid = 0; + } + OwnerDescriptorFixture::NullOwner + | OwnerDescriptorFixture::InvalidOwner + | OwnerDescriptorFixture::DaclOutOfRange + | OwnerDescriptorFixture::AclBytesOutOfRange + | OwnerDescriptorFixture::WrongAceCount + | OwnerDescriptorFixture::AceOutOfRange => {} + } + } + } + verify_single_owner_ace(descriptor_bytes, dacl, acl_bytes_in_use, ace, expected) +} + +#[cfg(test)] +static FORCE_DACL_VERIFY_FAILURE: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +#[cfg(test)] +fn force_next_owner_verification_failure() { + FORCE_DACL_VERIFY_FAILURE.store(true, std::sync::atomic::Ordering::SeqCst); +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OwnerDescriptorFixture { + WrongAceType, + UndersizedAce, + OversizedSid, + InvalidSid, + NullOwner, + InvalidOwner, + DaclOutOfRange, + AclBytesOutOfRange, + WrongAceCount, + AceOutOfRange, +} + +#[cfg(test)] +static OWNER_DESCRIPTOR_FIXTURE: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +struct OwnerDescriptorFixtureGuard; + +#[cfg(test)] +impl Drop for OwnerDescriptorFixtureGuard { + fn drop(&mut self) { + *OWNER_DESCRIPTOR_FIXTURE + .get_or_init(Default::default) + .lock() + .expect("owner descriptor fixture mutex poisoned") = None; + } +} + +#[cfg(test)] +fn install_owner_descriptor_fixture( + fixture: OwnerDescriptorFixture, +) -> OwnerDescriptorFixtureGuard { + let mut slot = OWNER_DESCRIPTOR_FIXTURE + .get_or_init(Default::default) + .lock() + .expect("owner descriptor fixture mutex poisoned"); + assert!( + slot.is_none(), + "owner descriptor tests require --test-threads=1" + ); + *slot = Some(fixture); + OwnerDescriptorFixtureGuard +} + +#[cfg(test)] +fn take_owner_descriptor_fixture() -> Option { + OWNER_DESCRIPTOR_FIXTURE + .get_or_init(Default::default) + .lock() + .expect("owner descriptor fixture mutex poisoned") + .take() +} + +fn verify_created_owner_only(handle: HANDLE, expected: &OwnerOnlySecurity) -> Result<()> { + inject_windows_create_failure( + WindowsCreateFailurePoint::SecurityVerification, + SafeFsOperation::VerifySecurityDescriptor, + )?; + #[cfg(test)] + if FORCE_DACL_VERIFY_FAILURE.swap(false, std::sync::atomic::Ordering::SeqCst) { + return Err(malformed_security()); + } + verify_owner_only(handle, expected) } #[allow(clippy::arc_with_non_send_sync)] // Arc retains the HANDLE-bearing parent chain; it is not shared publicly. @@ -1445,7 +1936,13 @@ fn create_directory_contract( SafeFsOperation::CreateDirectory }; require_mutation(parent, operation)?; - require_inherited_permissions(permissions)?; + let security = match permissions { + CreatePermissions::OwnerOnly => Some(OwnerOnlySecurity::new(true)?), + CreatePermissions::Inherit => None, + }; + let security_descriptor = security + .as_ref() + .map_or(null(), OwnerOnlySecurity::descriptor_ptr); let handle = nt_create_relative( parent.native.node.handle.raw(), name, @@ -1454,7 +1951,7 @@ fn create_directory_contract( contract.disposition, contract.options, contract.attributes, - null(), + security_descriptor, operation, )?; let validated = @@ -1475,6 +1972,9 @@ fn create_directory_contract( kind: opened.kind, }); } + if let Some(expected) = &security { + verify_created_owner_only(handle.raw(), expected)?; + } inject_windows_create_failure(WindowsCreateFailurePoint::CaseProof, operation)?; let case_mode = query_case_mode(handle.raw())?; inject_windows_create_failure(WindowsCreateFailurePoint::SnapshotAssembly, operation)?; @@ -1638,20 +2138,6 @@ fn collect_revalidation_proof(directory: &DirectoryAuthority) -> Result(operation: SafeFsOperation) -> Result { - Err(SafeFsError::UnsupportedSecureFilesystem { - operation, - reason: SecureFilesystemReason::UnsupportedTarget, - }) -} - -fn mutation_refusal(operation: SafeFsOperation) -> Result { - Err(SafeFsError::UnsupportedAtomicPublish { - operation, - reason: AtomicPublishReason::PrimitiveUnavailable, - }) -} - #[allow(clippy::arc_with_non_send_sync)] // Arc retains the HANDLE-bearing namespace chain; it is not shared publicly. pub(super) fn capture_absolute_directory( path: &Path, @@ -1906,11 +2392,10 @@ pub(super) fn create_stage_dir_new( name: &ComponentName, permissions: CreatePermissions, ) -> Result { - require_inherited_permissions(permissions)?; let directory = create_directory_contract( parent, name, - CreatePermissions::Inherit, + permissions, DirectoryAccess::Stage, contract_for_operation(OpenOperation::CreateStage), )?; @@ -1939,7 +2424,13 @@ pub(super) fn create_file_new( permissions: CreatePermissions, ) -> Result { require_mutation(parent, SafeFsOperation::CreateFile)?; - require_inherited_permissions(permissions)?; + let security = match permissions { + CreatePermissions::OwnerOnly => Some(OwnerOnlySecurity::new(false)?), + CreatePermissions::Inherit => None, + }; + let security_descriptor = security + .as_ref() + .map_or(null(), OwnerOnlySecurity::descriptor_ptr); let contract = contract_for_operation(OpenOperation::CreateFile); let handle = nt_create_relative( parent.native.node.handle.raw(), @@ -1949,7 +2440,7 @@ pub(super) fn create_file_new( contract.disposition, contract.options, contract.attributes, - null(), + security_descriptor, SafeFsOperation::CreateFile, )?; let validated = @@ -1980,6 +2471,9 @@ pub(super) fn create_file_new( kind: opened.kind, }); } + if let Some(expected) = &security { + verify_created_owner_only(handle.raw(), expected)?; + } Ok(opened) })(); let opened = match validated { @@ -2049,35 +2543,364 @@ pub(super) fn metadata_from_file(file: &NativeFile) -> Result { ) } +fn rename_retained_noreplace( + native: &NativeDirectory, + parent: &DirectoryAuthority, + target: &ComponentName, +) -> Result<()> { + require_mutation(parent, SafeFsOperation::RenameNoReplaceSameParent)?; + if matches!( + query_child_nofollow(parent, target)?, + ChildState::Present(_) + ) { + return Err(SafeFsError::AlreadyExists { + operation: SafeFsOperation::RenameNoReplaceSameParent, + }); + } + if !native.delete_right { + return Err(raw_nt( + SafeFsOperation::RenameNoReplaceSameParent, + STATUS_ACCESS_DENIED, + )); + } + let buffer = RenameInformationBuffer::new(parent.native.node.handle.raw(), target)?; + let mut iosb = IO_STATUS_BLOCK::default(); + // SAFETY: the retained DELETE source and parent handles plus the aligned, + // initialized variable-length buffer remain live for this synchronous call. + let status = unsafe { + NtSetInformationFile( + native.node.handle.raw(), + &mut iosb, + buffer.as_ptr(), + buffer.used, + FileRenameInformation, + ) + }; + if status < STATUS_SUCCESS { + return Err(map_rename_failure( + status, + true, + native.delete_right, + query_child_nofollow(parent, target), + )); + } + complete_nt(SafeFsOperation::RenameNoReplaceSameParent, status, &iosb) +} + +fn verify_same_parent(expected: &DirectoryAuthority, actual: &DirectoryAuthority) -> Result<()> { + if expected.opened.identity == actual.opened.identity && expected.snapshot == actual.snapshot { + Ok(()) + } else { + Err(SafeFsError::NamespaceChanged { + operation: SafeFsOperation::RenameNoReplaceSameParent, + }) + } +} + pub(super) fn quarantine_stage( - _: StageCapability, - _: &DirectoryAuthority, - _: ComponentName, + stage: StageCapability, + parent: &DirectoryAuthority, + quarantine_name: ComponentName, ) -> Result { - mutation_refusal(SafeFsOperation::QuarantineNoReplace) + let StageCapability { + parent: owned_parent, + directory, + original_name, + opened, + } = stage; + verify_same_parent(&owned_parent, parent)?; + revalidate_namespace(parent)?; + rename_retained_noreplace(&directory.native, parent, &quarantine_name)?; + Ok(QuarantinedCapability { + parent: owned_parent, + directory, + original_name, + quarantine_name, + opened, + }) } pub(super) fn publish_stage_noreplace( - _: StageCapability, - _: &DirectoryAuthority, - _: ComponentName, + stage: StageCapability, + parent: &DirectoryAuthority, + destination: ComponentName, ) -> Result<()> { - mutation_refusal(SafeFsOperation::PublishNoReplace) + let StageCapability { + parent: owned_parent, + directory, + opened, + .. + } = stage; + verify_same_parent(&owned_parent, parent)?; + revalidate_namespace(parent)?; + if directory.opened.identity != opened.identity { + return Err(SafeFsError::IdentityChanged { + operation: SafeFsOperation::RenameNoReplaceSameParent, + expected: opened.identity, + actual: directory.opened.identity.clone(), + }); + } + rename_retained_noreplace(&directory.native, parent, &destination)?; + drop(directory); + Ok(()) } +#[allow(clippy::arc_with_non_send_sync)] // Arc retains the HANDLE parent chain; capabilities never cross threads. pub(super) fn open_cleanup_child_nofollow( - _: &QuarantinedCapability, - _: &ComponentName, + quarantined: &QuarantinedCapability, + name: &ComponentName, ) -> Result { - filesystem_refusal(SafeFsOperation::OpenCleanupEntry) + let parent = &quarantined.directory; + let metadata = match query_child_nofollow(parent, name)? { + ChildState::Absent => { + return Err(SafeFsError::NotFound { + operation: SafeFsOperation::OpenCleanupEntry, + }) + } + ChildState::Present(metadata) => metadata, + }; + let contract = contract_for_operation(match metadata.kind { + EntryKind::Directory => OpenOperation::CleanupDir, + EntryKind::SymlinkOrReparse => OpenOperation::CleanupReparse, + _ => OpenOperation::CleanupFile, + }); + let handle = nt_create_relative( + parent.native.node.handle.raw(), + name, + parent.case_mode, + contract.desired, + contract.disposition, + contract.options, + contract.attributes, + null(), + SafeFsOperation::OpenCleanupEntry, + )?; + let filesystem = + parent + .opened + .filesystem + .as_ref() + .ok_or(SafeFsError::UnsupportedSecureFilesystem { + operation: SafeFsOperation::ProbeFilesystem, + reason: SecureFilesystemReason::FilesystemProbeUnavailable, + })?; + let opened = query_entry_metadata(handle.raw(), filesystem, SafeFsOperation::QueryMetadata)?; + if opened.identity != metadata.identity { + return Err(SafeFsError::IdentityChanged { + operation: SafeFsOperation::QueryMetadata, + expected: metadata.identity, + actual: opened.identity, + }); + } + if opened.kind != metadata.kind { + return Err(SafeFsError::UnsupportedEntryType { + operation: SafeFsOperation::OpenCleanupEntry, + kind: opened.kind, + }); + } + if opened.kind == EntryKind::Directory { + let duplicated_parent = duplicate_directory(parent)?; + let child_case = query_case_mode(handle.raw())?; + let child_snapshot = append_snapshot(&parent.snapshot, name.clone(), &opened, child_case)?; + let node = Arc::new(DirectoryNode { + handle, + parent: Some(Arc::clone(&parent.native.node)), + name: Some(name.clone()), + case_mode: child_case, + metadata: opened.clone(), + volume: parent.native.node.volume.clone(), + }); + let directory = DirectoryAuthority { + anchor: Arc::clone(&parent.anchor), + native: NativeDirectory { + node, + access: DirectoryAccess::MutateChildren, + delete_right: true, + }, + access: DirectoryAccess::MutateChildren, + opened: opened.clone(), + case_mode: child_case, + snapshot: child_snapshot, + }; + Ok(CleanupCapability::Directory(Box::new( + QuarantinedCapability { + parent: duplicated_parent, + directory, + original_name: name.clone(), + quarantine_name: name.clone(), + opened, + }, + ))) + } else { + Ok(CleanupCapability::Entry(Box::new(CleanupEntry { + parent: duplicate_directory(parent)?, + native: NativeFile { + handle, + opened: opened.clone(), + access: FileAccess::Read, + delete_right: true, + }, + name: name.clone(), + opened, + access: CleanupAccess::Delete, + }))) + } +} + +#[cfg(test)] +type BeforeRetainedDeleteHook = + Arc Result<()> + Send + Sync>; + +#[cfg(test)] +static BEFORE_RETAINED_DELETE_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +struct BeforeRetainedDeleteHookGuard; + +#[cfg(test)] +impl Drop for BeforeRetainedDeleteHookGuard { + fn drop(&mut self) { + *BEFORE_RETAINED_DELETE_HOOK + .get_or_init(Default::default) + .lock() + .expect("retained-delete hook mutex poisoned") = None; + } +} + +#[cfg(test)] +fn install_before_retained_delete_hook( + hook: BeforeRetainedDeleteHook, +) -> BeforeRetainedDeleteHookGuard { + let mut slot = BEFORE_RETAINED_DELETE_HOOK + .get_or_init(Default::default) + .lock() + .expect("retained-delete hook mutex poisoned"); + assert!( + slot.is_none(), + "retained-delete tests require --test-threads=1" + ); + *slot = Some(hook); + BeforeRetainedDeleteHookGuard +} + +fn run_before_retained_delete_hook( + handle: HANDLE, + parent: &DirectoryAuthority, + name: &ComponentName, +) -> Result<()> { + #[cfg(test)] + { + let hook = BEFORE_RETAINED_DELETE_HOOK + .get_or_init(Default::default) + .lock() + .expect("retained-delete hook mutex poisoned") + .clone(); + if let Some(hook) = hook { + return hook(handle, parent, name); + } + } + let _ = (handle, parent, name); + Ok(()) +} + +fn dispose_retained( + mut native: NativeFile, + parent: &DirectoryAuthority, + name: &ComponentName, + expected_kind: EntryKind, + operation: SafeFsOperation, +) -> Result<()> { + if !native.delete_right { + return Err(SafeFsError::Os { + operation, + raw: RawOsError::Win32(windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED), + }); + } + if native.opened.kind != expected_kind { + return Err(SafeFsError::UnsupportedEntryType { + operation, + kind: native.opened.kind, + }); + } + run_before_retained_delete_hook(native.handle.raw(), parent, name)?; + mark_delete_handle(native.handle.raw(), operation)?; + native.delete_right = false; + drop(native); + Ok(()) } -pub(super) fn delete_quarantined_entry(_: CleanupCapability) -> Result<()> { - filesystem_refusal(SafeFsOperation::DeleteQuarantinedEntry) +pub(super) fn delete_quarantined_entry(cleanup: CleanupCapability) -> Result<()> { + match cleanup { + CleanupCapability::Entry(entry) => { + let CleanupEntry { + parent, + native, + name, + opened, + access: CleanupAccess::Delete, + } = *entry; + if native.opened.identity != opened.identity { + return Err(SafeFsError::IdentityChanged { + operation: SafeFsOperation::DeleteQuarantinedEntry, + expected: opened.identity, + actual: native.opened.identity, + }); + } + dispose_retained( + native, + &parent, + &name, + opened.kind, + SafeFsOperation::DeleteQuarantinedEntry, + ) + } + CleanupCapability::Directory(_) => Err(SafeFsError::UnsupportedEntryType { + operation: SafeFsOperation::DeleteQuarantinedEntry, + kind: EntryKind::Directory, + }), + } } -pub(super) fn delete_quarantined_empty_directory(_: QuarantinedCapability) -> Result<()> { - filesystem_refusal(SafeFsOperation::DeleteQuarantinedEmptyDirectory) +pub(super) fn delete_quarantined_empty_directory(quarantined: QuarantinedCapability) -> Result<()> { + let QuarantinedCapability { + parent, + directory, + quarantine_name, + opened, + .. + } = quarantined; + if directory.opened.identity != opened.identity || !directory.native.delete_right { + return Err(SafeFsError::IdentityChanged { + operation: SafeFsOperation::DeleteQuarantinedEmptyDirectory, + expected: opened.identity, + actual: directory.opened.identity, + }); + } + let native = NativeFile { + handle: Arc::try_unwrap(directory.native.node) + .map_err(|node| { + SafeFsError::io( + SafeFsOperation::DeleteQuarantinedEmptyDirectory, + io::Error::other(format!( + "directory handle still shared: {}", + Arc::strong_count(&node) + )), + ) + })? + .handle, + opened: directory.opened, + access: FileAccess::Read, + delete_right: true, + }; + dispose_retained( + native, + &parent, + &quarantine_name, + EntryKind::Directory, + SafeFsOperation::DeleteQuarantinedEmptyDirectory, + ) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -2088,7 +2911,6 @@ enum WindowsCreateFailurePoint { CaseProof, SnapshotAssembly, ParentDuplicate, - #[allow(dead_code)] // Task 7A test-only removes this when it constructs the variant. SecurityVerification, } @@ -2174,6 +2996,7 @@ mod tests { use super::*; use std::fs; use std::path::PathBuf; + use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; struct TestDir(PathBuf); @@ -2210,6 +3033,22 @@ mod tests { .expect("capture fixture root") } + fn present_for_test() -> ChildState { + ChildState::Present(EntryMetadata { + identity: StableIdentity::Windows { + volume_serial: 7, + file_id: [3; 16], + }, + kind: EntryKind::RegularFile, + len: 0, + link_count: 1, + filesystem: Some(LocalFilesystemSnapshot::Windows { + volume_guid: vec![1], + serial: 7, + }), + }) + } + #[test] fn remote_protocol_query_buffer_initializes_required_header() { let buffer = remote_protocol_query_buffer(SafeFsOperation::ProbeVolume).unwrap(); @@ -2268,6 +3107,364 @@ mod tests { assert_eq!(enumerate(&b).unwrap(), vec![name("data")]); } + #[test] + fn owner_only_file_directory_stage_succeed_and_rollback() { + let temp = TestDir::new("owner-only"); + let authority = root(&temp); + + let file = create_file_new(&authority, &name("file"), CreatePermissions::OwnerOnly) + .expect("owner-only file creation succeeds"); + drop(file); + let directory = create_dir_new( + &authority, + &name("directory"), + CreatePermissions::OwnerOnly, + DirectoryAccess::MutateChildren, + ) + .expect("owner-only directory creation succeeds"); + drop(directory); + let stage = create_stage_dir_new(&authority, &name("stage"), CreatePermissions::OwnerOnly) + .expect("owner-only stage creation succeeds"); + drop(stage); + for value in ["file", "directory", "stage"] { + assert!(matches!( + query_child_nofollow(&authority, &name(value)).unwrap(), + ChildState::Present(_) + )); + } + + force_next_owner_verification_failure(); + assert!(matches!( + create_file_new( + &authority, + &name("rollback-file"), + CreatePermissions::OwnerOnly + ), + Err(SafeFsError::InvalidNativeBuffer { + operation: SafeFsOperation::VerifySecurityDescriptor, + reason: NativeBufferReason::SecurityDescriptorMalformed, + }) + )); + force_next_owner_verification_failure(); + assert!(matches!( + create_dir_new( + &authority, + &name("rollback-directory"), + CreatePermissions::OwnerOnly, + DirectoryAccess::MutateChildren, + ), + Err(SafeFsError::InvalidNativeBuffer { + operation: SafeFsOperation::VerifySecurityDescriptor, + reason: NativeBufferReason::SecurityDescriptorMalformed, + }) + )); + force_next_owner_verification_failure(); + assert!(matches!( + create_stage_dir_new( + &authority, + &name("rollback-stage"), + CreatePermissions::OwnerOnly, + ), + Err(SafeFsError::InvalidNativeBuffer { + operation: SafeFsOperation::VerifySecurityDescriptor, + reason: NativeBufferReason::SecurityDescriptorMalformed, + }) + )); + for value in ["rollback-file", "rollback-directory", "rollback-stage"] { + assert!(matches!( + query_child_nofollow(&authority, &name(value)).unwrap(), + ChildState::Absent + )); + } + } + + #[test] + fn windows_post_create_security_failure_rolls_back_same_handle() { + let temp = TestDir::new("security-rollback"); + let authority = root(&temp); + let _failure = + install_windows_create_failure(WindowsCreateFailurePoint::SecurityVerification); + assert!(matches!( + create_file_new(&authority, &name("leaf"), CreatePermissions::OwnerOnly), + Err(SafeFsError::Io { + operation: SafeFsOperation::VerifySecurityDescriptor, + .. + }) + )); + assert!(matches!( + query_child_nofollow(&authority, &name("leaf")).unwrap(), + ChildState::Absent + )); + } + + fn assert_owner_descriptor_fixture_rejected(fixture: OwnerDescriptorFixture, leaf: &str) { + let temp = TestDir::new(leaf); + let authority = root(&temp); + let _fixture = install_owner_descriptor_fixture(fixture); + assert!(matches!( + create_file_new(&authority, &name(leaf), CreatePermissions::OwnerOnly), + Err(SafeFsError::InvalidNativeBuffer { + operation: SafeFsOperation::VerifySecurityDescriptor, + reason: NativeBufferReason::SecurityDescriptorMalformed, + }) + )); + assert!(matches!( + query_child_nofollow(&authority, &name(leaf)).unwrap(), + ChildState::Absent + )); + } + + #[test] + fn owner_only_dacl_rejects_wrong_ace_type() { + assert_owner_descriptor_fixture_rejected( + OwnerDescriptorFixture::WrongAceType, + "wrong-ace-type", + ); + } + + #[test] + fn owner_only_dacl_rejects_undersized_ace_and_out_of_range_acl_fields() { + for (fixture, leaf) in [ + (OwnerDescriptorFixture::UndersizedAce, "undersized-ace"), + (OwnerDescriptorFixture::DaclOutOfRange, "dacl-out-of-range"), + ( + OwnerDescriptorFixture::AclBytesOutOfRange, + "acl-bytes-out-of-range", + ), + (OwnerDescriptorFixture::WrongAceCount, "wrong-ace-count"), + (OwnerDescriptorFixture::AceOutOfRange, "ace-out-of-range"), + ] { + assert_owner_descriptor_fixture_rejected(fixture, leaf); + } + } + + #[test] + fn owner_only_dacl_rejects_oversized_sid() { + assert_owner_descriptor_fixture_rejected( + OwnerDescriptorFixture::OversizedSid, + "oversized-sid", + ); + } + + #[test] + fn owner_only_dacl_rejects_invalid_sid() { + assert_owner_descriptor_fixture_rejected(OwnerDescriptorFixture::InvalidSid, "invalid-sid"); + } + + #[test] + fn owner_only_dacl_rejects_null_or_invalid_owner() { + assert_owner_descriptor_fixture_rejected(OwnerDescriptorFixture::NullOwner, "null-owner"); + assert_owner_descriptor_fixture_rejected( + OwnerDescriptorFixture::InvalidOwner, + "invalid-owner", + ); + } + + #[test] + fn quarantine_and_publish_success_do_not_self_conflict() { + let quarantine_temp = TestDir::new("quarantine-success"); + let authority = root(&quarantine_temp); + let stage = + create_stage_dir_new(&authority, &name("stage"), CreatePermissions::Inherit).unwrap(); + let quarantined = quarantine_stage(stage, &authority, name("quarantine")) + .expect("retained quarantine rename succeeds"); + drop(quarantined); + assert!(!quarantine_temp.path().join("stage").exists()); + assert!(quarantine_temp.path().join("quarantine").is_dir()); + + let publish_temp = TestDir::new("publish-success"); + let authority = root(&publish_temp); + let stage = + create_stage_dir_new(&authority, &name("stage"), CreatePermissions::Inherit).unwrap(); + publish_stage_noreplace(stage, &authority, name("destination")) + .expect("retained publish rename succeeds"); + assert!(!publish_temp.path().join("stage").exists()); + assert!(publish_temp.path().join("destination").is_dir()); + } + + #[test] + fn rename_never_replaces_any_target_kind() { + assert!(matches!( + map_rename_failure(STATUS_ACCESS_DENIED, true, true, Ok(present_for_test())), + SafeFsError::AlreadyExists { .. } + )); + assert!(matches!( + map_rename_failure(STATUS_ACCESS_DENIED, true, true, Ok(ChildState::Absent)), + SafeFsError::Os { + raw: RawOsError::NtStatus { + status: STATUS_ACCESS_DENIED, + .. + }, + .. + } + )); + for kind in ["file", "empty-dir", "nonempty-dir", "reparse"] { + let temp = TestDir::new(kind); + let target = temp.path().join("target"); + let external = temp.path().join("external"); + match kind { + "file" => fs::write(&target, b"keep-file").unwrap(), + "empty-dir" => fs::create_dir(&target).unwrap(), + "nonempty-dir" => { + fs::create_dir(&target).unwrap(); + fs::write(target.join("keep"), b"tree").unwrap(); + } + "reparse" => { + fs::create_dir(&external).unwrap(); + fs::write(external.join("keep"), b"outside").unwrap(); + let output = Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(&target) + .arg(&external) + .output() + .unwrap(); + assert!( + output.status.success(), + "mklink failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + _ => unreachable!(), + } + let authority = root(&temp); + let before = match query_child_nofollow(&authority, &name("target")).unwrap() { + ChildState::Present(value) => value, + ChildState::Absent => panic!("collision target absent"), + }; + let stage = + create_stage_dir_new(&authority, &name("stage"), CreatePermissions::Inherit) + .unwrap(); + assert!(matches!( + publish_stage_noreplace(stage, &authority, name("target")), + Err(SafeFsError::AlreadyExists { .. }) + )); + let after = match query_child_nofollow(&authority, &name("target")).unwrap() { + ChildState::Present(value) => value, + ChildState::Absent => panic!("collision target removed"), + }; + assert_eq!(after.identity, before.identity); + match kind { + "file" => assert_eq!(fs::read(&target).unwrap(), b"keep-file"), + "nonempty-dir" => assert_eq!(fs::read(target.join("keep")).unwrap(), b"tree"), + "reparse" => assert_eq!(fs::read(external.join("keep")).unwrap(), b"outside"), + _ => assert!(target.is_dir()), + } + } + } + + #[test] + fn cleanup_quarantined_tree_deletes_nested_reparse_without_traversal() { + let temp = TestDir::new("cleanup-tree"); + let external = temp.path().join("external"); + fs::create_dir(&external).unwrap(); + fs::write(external.join("keep"), b"outside-bytes").unwrap(); + let authority = root(&temp); + let stage = + create_stage_dir_new(&authority, &name("stage"), CreatePermissions::Inherit).unwrap(); + let nested = create_dir_new( + stage.directory(), + &name("nested"), + CreatePermissions::Inherit, + DirectoryAccess::MutateChildren, + ) + .unwrap(); + let mut file = create_file_new(&nested, &name("data"), CreatePermissions::Inherit).unwrap(); + file.write_all(b"inside").unwrap(); + drop(file); + drop(nested); + let output = Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(temp.path().join("stage").join("nested").join("link")) + .arg(&external) + .output() + .unwrap(); + assert!( + output.status.success(), + "mklink failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let quarantine = quarantine_stage(stage, &authority, name("quarantine")).unwrap(); + super::super::cleanup_quarantined_tree(quarantine) + .expect("common recursive cleanup succeeds"); + assert!(matches!( + query_child_nofollow(&authority, &name("quarantine")).unwrap(), + ChildState::Absent + )); + assert_eq!(fs::read(external.join("keep")).unwrap(), b"outside-bytes"); + } + + #[test] + fn retained_delete_is_safe_when_real_name_rebinds_or_is_blocked() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let temp = TestDir::new("delete-rebound"); + let authority = root(&temp); + let stage = + create_stage_dir_new(&authority, &name("stage"), CreatePermissions::Inherit).unwrap(); + let mut file = + create_file_new(stage.directory(), &name("leaf"), CreatePermissions::Inherit).unwrap(); + file.write_all(b"original").unwrap(); + drop(file); + let quarantine = quarantine_stage(stage, &authority, name("quarantine")).unwrap(); + let cleanup = open_cleanup_child_nofollow(&quarantine, &name("leaf")).unwrap(); + let expected_source = match &cleanup { + CleanupCapability::Entry(entry) => entry.native.handle.raw() as usize, + CleanupCapability::Directory(_) => panic!("leaf opened as a directory"), + }; + let quarantine_path = temp.path().join("quarantine"); + let rebound = Arc::new(AtomicBool::new(false)); + let hook_rebound = Arc::clone(&rebound); + let _guard = install_before_retained_delete_hook(Arc::new( + move |source, parent, _old_name| { + if source as usize != expected_source { + return Ok(()); + } + let buffer = RenameInformationBuffer::new( + parent.native.node.handle.raw(), + &name("moved-original"), + )?; + let mut iosb = IO_STATUS_BLOCK::default(); + // SAFETY: source is the retained DELETE handle and all inputs + // remain live for this synchronous test-only rename. + let status = unsafe { + NtSetInformationFile( + source, + &mut iosb, + buffer.as_ptr(), + buffer.used, + FileRenameInformation, + ) + }; + if status == STATUS_SUCCESS { + complete_nt(SafeFsOperation::RenameNoReplaceSameParent, status, &iosb)?; + fs::write(quarantine_path.join("leaf"), b"replacement") + .map_err(|error| SafeFsError::io(SafeFsOperation::CreateFile, error))?; + hook_rebound.store(true, Ordering::SeqCst); + } else { + assert_eq!( + status, STATUS_SHARING_VIOLATION, + "Windows may reject the simulated same-handle rename, but no other failure is expected" + ); + } + Ok(()) + }, + )); + delete_quarantined_entry(cleanup).unwrap(); + if rebound.load(Ordering::SeqCst) { + assert_eq!( + fs::read(temp.path().join("quarantine").join("leaf")).unwrap(), + b"replacement" + ); + } else { + assert!(!temp.path().join("quarantine").join("leaf").exists()); + } + assert!(!temp + .path() + .join("quarantine") + .join("moved-original") + .exists()); + } + fn assert_file_create_failure_rolls_back(point: WindowsCreateFailurePoint, label: &str) { let temp = TestDir::new(label); let authority = root(&temp); diff --git a/crates/opentake-project/tests/archive.rs b/crates/opentake-project/tests/archive.rs index 7dfb2be6..835524cc 100644 --- a/crates/opentake-project/tests/archive.rs +++ b/crates/opentake-project/tests/archive.rs @@ -23,6 +23,8 @@ fn entry(id: &str, name: &str, kind: ClipType, source: MediaSource) -> MediaMani source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/tests/compound_roundtrip.rs b/crates/opentake-project/tests/compound_roundtrip.rs new file mode 100644 index 00000000..75c0c849 --- /dev/null +++ b/crates/opentake-project/tests/compound_roundtrip.rs @@ -0,0 +1,116 @@ +//! Persistence and compatibility boundaries for editable compound clips. + +mod common; + +use opentake_domain::{Clip, ClipType, NestedSequence, Timeline, Track}; +use opentake_project::{Project, ProjectError}; + +use common::{write_file, TempDir}; + +fn nested_timeline() -> Timeline { + let mut child = Timeline::new(); + let mut child_track = Track::new("child-track", ClipType::Video); + child_track + .clips + .push(Clip::new("child-clip", "asset-a", 2, 12)); + child.tracks.push(child_track); + + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence-a", "Scene A", child)); + let mut root_track = Track::new("root-track", ClipType::Video); + root_track + .clips + .push(Clip::new_nested("compound-a", "sequence-a", 10, 20)); + root.tracks.push(root_track); + root +} + +#[test] +fn compound_clip_roundtrips_nested_timeline() { + let temp = TempDir::new("compound-roundtrip"); + let bundle = temp.child("Compound.opentake"); + let mut project = Project::new(&bundle); + project.timeline = nested_timeline(); + + project.save().expect("save nested timeline"); + let reopened = Project::open(&bundle).expect("open nested timeline"); + + assert_eq!(reopened.timeline, project.timeline); + assert_eq!( + reopened.timeline.tracks[0].clips[0] + .nested_sequence_id + .as_deref(), + Some("sequence-a") + ); + reopened + .timeline + .validate_nested_sequences() + .expect("reopened graph stays valid"); +} + +#[test] +fn nested_future_fields_make_project_read_only() { + let temp = TempDir::new("compound-future-field"); + let bundle = temp.child("Future.opentake"); + std::fs::create_dir_all(&bundle).unwrap(); + write_file( + &bundle.join("project.json"), + br#"{ + "nestedSequences": [{ + "id": "sequence-a", + "name": "A", + "timeline": { + "tracks": [{ + "id": "child-track", + "type": "video", + "futureTrackFlag": true, + "clips": [] + }] + } + }], + "tracks": [] + }"#, + ); + + let project = Project::open(&bundle).expect("unknown field opens read-only"); + assert!(project.compatibility().is_read_only()); + assert!(project.compatibility().blockers().iter().any(|blocker| { + blocker == "project.json:nestedSequences.0.timeline.tracks.0.futureTrackFlag" + })); +} + +#[test] +fn recursive_nested_graph_fails_open_and_save() { + let temp = TempDir::new("compound-cycle"); + let bundle = temp.child("Cycle.opentake"); + let mut a = Timeline::new(); + let mut a_track = Track::new("a-track", ClipType::Video); + a_track.clips.push(Clip::new_nested("a-to-b", "b", 0, 10)); + a.tracks.push(a_track); + let mut b = Timeline::new(); + let mut b_track = Track::new("b-track", ClipType::Video); + b_track.clips.push(Clip::new_nested("b-to-a", "a", 0, 10)); + b.tracks.push(b_track); + + let mut project = Project::new(&bundle); + project.timeline.nested_sequences = vec![ + NestedSequence::new("a", "A", a), + NestedSequence::new("b", "B", b), + ]; + let error = project.save().expect_err("cycle must not be persisted"); + assert!(matches!(error, ProjectError::InvalidTimeline { .. })); + assert!( + !bundle.exists(), + "failed preflight must not create a bundle" + ); + + std::fs::create_dir_all(&bundle).unwrap(); + write_file( + &bundle.join("project.json"), + serde_json::to_string(&project.timeline).unwrap().as_bytes(), + ); + let error = Project::open(&bundle).expect_err("cycle must fail open"); + assert!(matches!(error, ProjectError::InvalidTimeline { .. })); + assert!(error.to_string().contains("a -> b -> a")); +} diff --git a/crates/opentake-project/tests/lut_storage.rs b/crates/opentake-project/tests/lut_storage.rs new file mode 100644 index 00000000..672d8a82 --- /dev/null +++ b/crates/opentake-project/tests/lut_storage.rs @@ -0,0 +1,36 @@ +#[allow(dead_code)] +mod common; + +use common::TempDir; +use opentake_project::{Project, ProjectRoot}; + +#[test] +fn managed_lut_is_bounded_nofollow_and_carried_by_complete_save_as() { + let temp = TempDir::new("lut-storage"); + let source = temp.child("Source.opentake"); + Project::new(&source).save().expect("create source bundle"); + let source_root = ProjectRoot::open(&source).expect("retain source root"); + let name = format!("{}.cube", "0123456789abcdef".repeat(4)); + let bytes = b"LUT_3D_SIZE 17\n# acceptance bytes\n"; + source_root + .write_lut_atomic(&name, bytes) + .expect("publish managed LUT"); + assert_eq!( + source_root.read_lut(&name, 4096).unwrap().as_deref(), + Some(bytes.as_slice()) + ); + assert!( + source_root.read_lut(&name, 4).is_err(), + "read cap is enforced" + ); + + let destination = temp.child("Destination.opentake"); + let destination_root = Project::new(&destination) + .publish_complete_to(&destination, Some(&source_root)) + .expect("complete Save As"); + assert_eq!( + destination_root.read_lut(&name, 4096).unwrap().as_deref(), + Some(bytes.as_slice()), + "nested media/luts asset must travel with Save As" + ); +} diff --git a/crates/opentake-project/tests/roundtrip.rs b/crates/opentake-project/tests/roundtrip.rs index 9dc25d92..4154f946 100644 --- a/crates/opentake-project/tests/roundtrip.rs +++ b/crates/opentake-project/tests/roundtrip.rs @@ -55,6 +55,8 @@ fn sample_project(bundle: &Path) -> Project { source_height: Some(2160), source_fps: Some(24.0), has_audio: Some(true), + color: None, + proxy: None, folder_id: Some("folder-1".into()), cached_remote_url: None, cached_remote_url_expires_at: None, @@ -72,6 +74,8 @@ fn sample_project(bundle: &Path) -> Project { source_height: None, source_fps: None, has_audio: None, + color: None, + proxy: None, folder_id: None, cached_remote_url: None, cached_remote_url_expires_at: None, diff --git a/crates/opentake-project/tests/schema_compat.rs b/crates/opentake-project/tests/schema_compat.rs index d71d8ff4..a6230ae5 100644 --- a/crates/opentake-project/tests/schema_compat.rs +++ b/crates/opentake-project/tests/schema_compat.rs @@ -40,6 +40,20 @@ fn write_known_bundle(bundle: &Path) { "interpolationOut": "smooth" }] }, + "loudnessNormalization": { + "targetLufs": -16.0, + "truePeakCeilingDbtp": -1.0, + "inputIntegratedLufs": -24.0, + "inputTruePeakDbtp": -12.0, + "gainDb": 8.0, + "outputIntegratedLufs": -16.0, + "outputTruePeakDbtp": -2.0 + }, + "audioDenoise": { + "mode": "voice", + "strength": 0.8, + "previewEnabled": true + }, "effects": [{"name": "blur", "params": {}, "enabled": true}], "masks": [{ "shape": {"kind": "circle", "center": {"x": 0.5, "y": 0.5}, "radius": {"x": 0.5, "y": 0.5}}, diff --git a/crates/opentake-project/tests/upstream_compat.rs b/crates/opentake-project/tests/upstream_compat.rs index 9ecd57c8..878ce2d2 100644 --- a/crates/opentake-project/tests/upstream_compat.rs +++ b/crates/opentake-project/tests/upstream_compat.rs @@ -12,7 +12,7 @@ mod common; -use opentake_domain::{ClipType, MediaSource}; +use opentake_domain::{ClipType, MediaManifest, MediaSource}; use opentake_project::{Project, ProjectError}; use serde_json::{json, Value}; @@ -241,6 +241,31 @@ fn applies_clip_defaults_for_omitted_fields() { assert_eq!(clip_a.end_frame(), 90); // source_frames_consumed = round(90 * 2.0) = 180. assert_eq!(clip_a.source_frames_consumed(), 180); + + // A missing persisted version is the legacy schema (1), even though a + // newly constructed manifest starts at the current schema (2). Explicit + // persisted versions must never be overwritten by the compatibility path. + assert_eq!(project.manifest.version, 1); + assert_eq!(MediaManifest::default().version, 2); + let explicit: MediaManifest = serde_json::from_value(json!({ + "version": 2, + "entries": [], + "folders": [] + })) + .unwrap(); + assert_eq!(explicit.version, 2); + + // Persisting the upgraded representation and opening it again must keep + // both the decoded defaults and the legacy manifest version exactly. + project.save().unwrap(); + let reopened = Project::open(&bundle).unwrap(); + assert_eq!(reopened.timeline, project.timeline); + assert_eq!(reopened.manifest, project.manifest); + let reopened_clip = &reopened.timeline.tracks[0].clips[0]; + assert_eq!(reopened_clip.trim_end_frame, 0); + assert_eq!(reopened_clip.opacity, 1.0); + assert!(reopened_clip.opacity_track.is_none()); + assert!(reopened_clip.link_group_id.is_none()); } #[test] @@ -309,7 +334,10 @@ fn parses_manifest_with_missing_version_and_tagged_sources() { fn migrates_generation_log_legacy_cost_and_version() { let (_tmp, bundle) = make_upstream_bundle("compat-genlog"); let project = Project::open(&bundle).unwrap(); - let log = project.generation_log.expect("generation log present"); + let log = project + .generation_log + .as_ref() + .expect("generation log present"); // Missing top-level version -> 1. assert_eq!(log.version, 1); @@ -330,6 +358,18 @@ fn migrates_generation_log_legacy_cost_and_version() { assert!(modern.created_at.is_none()); assert_eq!(log.total_credits(), 342); + + // The generated fallback id is synthesized only once. Saving and + // reopening must retain the migrated version, costs, and stable identity. + let expected_log = log.clone(); + project.save().unwrap(); + let reopened = Project::open(&bundle).unwrap(); + assert_eq!(reopened.generation_log.as_ref(), Some(&expected_log)); + let reopened_log = reopened.generation_log.as_ref().unwrap(); + assert_eq!(reopened_log.version, 1); + assert_eq!(reopened_log.entries[0].cost_credits, Some(42)); + assert_eq!(reopened_log.entries[1].id, modern.id); + assert_eq!(reopened_log.total_credits(), 342); } #[test] diff --git a/crates/opentake-render/Cargo.toml b/crates/opentake-render/Cargo.toml index 4da54ba1..828fab9a 100644 --- a/crates/opentake-render/Cargo.toml +++ b/crates/opentake-render/Cargo.toml @@ -17,6 +17,7 @@ thiserror = "2" wgpu = { version = "23", default-features = false, features = ["wgsl", "metal"] } # POD uniforms uploaded to the GPU (mat3x2 / crop_uv / opacity / flags). bytemuck = { version = "1", features = ["derive"] } +half = "2" # Block on wgpu's async device/queue/map calls from synchronous render code. pollster = "0.4" # Text shaping + layout + glyph rasterization for timeline text clips (upstream @@ -28,3 +29,6 @@ cosmic-text = { version = "0.12", default-features = false, features = ["std", " [dev-dependencies] # PNG read-back round-trip checks in the GPU smoke test (offline, no assets). image = { version = "0.25", default-features = false, features = ["png"] } +serde_json = { workspace = true } +opentake-media = { workspace = true } +opentake-ops = { workspace = true } diff --git a/crates/opentake-render/src/gpu/compositor.rs b/crates/opentake-render/src/gpu/compositor.rs index 7790111d..edd5de1c 100644 --- a/crates/opentake-render/src/gpu/compositor.rs +++ b/crates/opentake-render/src/gpu/compositor.rs @@ -9,39 +9,55 @@ use std::rc::Rc; use bytemuck::{Pod, Zeroable}; -use opentake_domain::{ColorGrade, LiftGammaGain, MaskShape}; +use opentake_domain::{ + validate_effect_chain, ColorGrade, LiftGammaGain, LutReference, MaskShape, MAX_EFFECTS_PER_CLIP, +}; -use crate::gpu::texture::GpuTexture; +use crate::gpu::texture::{GpuLutTexture, GpuTexture}; use crate::gpu::RenderError; use crate::plan::{FramePlan, LayerDraw, RenderSize, TextureSource}; use crate::source::DecodedFrame; +use opentake_domain::{MAX_MASKS_PER_CLIP, MAX_POLYGON_MASK_POINTS}; /// Maximum masks evaluated in-shader per draw (mirrors `MASK_CAP` in -/// `shader.wgsl`). Extra masks on a clip beyond this are ignored by the -/// compositor (the domain still stores and unit-tests all of them). -const MASK_CAP: usize = 4; +/// `shader.wgsl`). The shared edit-command validation prevents authored data +/// from exceeding this fixed uniform capacity. +const MASK_CAP: usize = MAX_MASKS_PER_CLIP; /// Flag bits packed into `canvas_op_flags[3]` (bitcast to u32 in WGSL). const FLAG_PREMULTIPLY: u32 = 1; const FLAG_GRADE: u32 = 2; const FLAG_CHROMA: u32 = 4; -/// Mask kind tags (mirror `MaskShape` / the WGSL `MASK_*` consts). Polygon masks -/// are not rendered in-shader (see shader TODO); they encode as `MASK_NOOP` which -/// the shader treats as a full-coverage circle (no clipping). +/// Mask kind tags and polygon point cap mirror the WGSL constants. const MASK_LINEAR: f32 = 0.0; const MASK_CIRCLE: f32 = 1.0; -/// A circle large enough to cover the whole canvas — used to make an unsupported -/// (polygon) mask a no-op instead of silently clipping. -const MASK_NOOP_GEO: [f32; 4] = [0.5, 0.5, 8.0, 8.0]; +const MASK_POLY: f32 = 2.0; +const POLY_POINT_CAP: usize = MAX_POLYGON_MASK_POINTS; + +/// Effect kind tags mirror the closed registry and WGSL implementation. +const EFFECT_GRAYSCALE: f32 = 0.0; +const EFFECT_SEPIA: f32 = 1.0; +const EFFECT_INVERT: f32 = 2.0; + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable, Default)] +struct EffectGpu { + // (kind, amount, pad, pad) + data: [f32; 4], +} /// One mask in the uniform (mirrors WGSL `MaskGpu`): `head = (kind, feather, -/// invert, pad)`, `geo` packs the shape geometry. +/// invert, polygon-point-count)`, `geo` packs linear/circle geometry, and +/// `points` carries a bounded pen path. #[repr(C)] #[derive(Clone, Copy, Pod, Zeroable, Default)] struct MaskGpu { head: [f32; 4], geo: [f32; 4], + transform: [f32; 4], + transform_meta: [f32; 4], + points: [[f32; 4]; POLY_POINT_CAP], } /// Uniform mirror of WGSL `struct U` (SPEC §3.2), extended with the A-tier color @@ -50,59 +66,99 @@ struct MaskGpu { #[repr(C)] #[derive(Clone, Copy, Pod, Zeroable)] struct Uniforms { - affine0: [f32; 4], // a, b, c, d - crop_uv: [f32; 4], // u0, v0, u1, v1 - affine1_nat: [f32; 4], // tx, ty, natW, natH - canvas_op_flags: [f32; 4], // canvasW, canvasH, opacity, flags-as-f32 - grade_exp_wb: [f32; 4], // exposure, wb_r, wb_g, wb_b - grade_lift: [f32; 4], // lift_r, lift_g, lift_b, contrast - grade_gamma: [f32; 4], // gamma_r, gamma_g, gamma_b, saturation - grade_gain: [f32; 4], // gain_r, gain_g, gain_b, pad - chroma0: [f32; 4], // key_r, key_g, key_b, similarity - chroma1: [f32; 4], // smoothness, spill, pad, pad - mask_meta: [f32; 4], // mask_count, pad, pad, pad + affine0: [f32; 4], // a, b, c, d + crop_uv: [f32; 4], // u0, v0, u1, v1 + affine1_nat: [f32; 4], // tx, ty, natW, natH + canvas_op_flags: [f32; 4], // canvasW, canvasH, opacity, flags-as-f32 + grade_exp_wb: [f32; 4], // exposure, wb_r, wb_g, wb_b + grade_lift: [f32; 4], // lift_r, lift_g, lift_b, contrast + grade_gamma: [f32; 4], // gamma_r, gamma_g, gamma_b, saturation + grade_gain: [f32; 4], // gain_r, gain_g, gain_b, pad + hsl_secondary_meta: [f32; 4], // enabled, hue center, full width, feather + hsl_secondary_adjust: [f32; 4], // hue shift, saturation, lightness, pad + lut_meta: [f32; 4], // enabled, intensity, table size, pad + lut_domain_min: [f32; 4], // min r/g/b, pad + lut_domain_scale: [f32; 4], // reciprocal domain span r/g/b, pad + chroma0: [f32; 4], // key_r, key_g, key_b, similarity + chroma1: [f32; 4], // smoothness, spill, pad, pad + mask_meta: [f32; 4], // mask_count, pad, pad, pad masks: [MaskGpu; MASK_CAP], + effect_meta: [f32; 4], // effect_count, pad, pad, pad + effects: [EffectGpu; MAX_EFFECTS_PER_CLIP], +} + +#[derive(Clone, Copy)] +struct GradeBlocks { + exp_wb: [f32; 4], + lift: [f32; 4], + gamma: [f32; 4], + gain: [f32; 4], + hsl_meta: [f32; 4], + hsl_adjust: [f32; 4], } /// Identity color-grade uniform block (exposure 0, wb/gain 1, lift 0, gamma 1, /// contrast 0, saturation 1). Used when a draw has no grade. -fn identity_grade_blocks() -> ([f32; 4], [f32; 4], [f32; 4], [f32; 4]) { - ( - [0.0, 1.0, 1.0, 1.0], // exposure, wb - [0.0, 0.0, 0.0, 0.0], // lift, contrast - [1.0, 1.0, 1.0, 1.0], // gamma, saturation - [1.0, 1.0, 1.0, 0.0], // gain, pad - ) +fn identity_grade_blocks() -> GradeBlocks { + GradeBlocks { + exp_wb: [0.0, 1.0, 1.0, 1.0], // exposure, wb + lift: [0.0, 0.0, 0.0, 0.0], // lift, contrast + gamma: [1.0, 1.0, 1.0, 1.0], // gamma, saturation + gain: [1.0, 1.0, 1.0, 0.0], // gain, pad + hsl_meta: [0.0, 0.0, 1.0, 0.0], // disabled, center, width, feather + hsl_adjust: [0.0; 4], // hue shift, saturation, lightness, pad + } } -/// Pack a [`ColorGrade`] into the four uniform vec4 blocks the shader reads. The +/// Pack a [`ColorGrade`] into the six uniform vec4 blocks the shader reads. The /// white balance is resolved to per-channel gain CPU-side (the shader multiplies /// it directly), keeping the WGSL mirror of `ColorGrade::apply_linear` simple. -fn grade_blocks(g: &ColorGrade) -> ([f32; 4], [f32; 4], [f32; 4], [f32; 4]) { +fn grade_blocks(g: &ColorGrade) -> GradeBlocks { let wb = g.white_balance_gain(); let LiftGammaGain { lift, gamma, gain } = g.lift_gamma_gain; - ( - [g.exposure as f32, wb.r as f32, wb.g as f32, wb.b as f32], - [ + let (hsl_meta, hsl_adjust) = + g.hsl_secondary + .map_or(([0.0, 0.0, 1.0, 0.0], [0.0; 4]), |secondary| { + ( + [ + 1.0, + secondary.hue_center as f32, + secondary.hue_width as f32, + secondary.feather as f32, + ], + [ + secondary.hue_shift as f32, + secondary.saturation as f32, + secondary.lightness as f32, + 0.0, + ], + ) + }); + GradeBlocks { + exp_wb: [g.exposure as f32, wb.r as f32, wb.g as f32, wb.b as f32], + lift: [ lift.r as f32, lift.g as f32, lift.b as f32, g.contrast as f32, ], - [ + gamma: [ gamma.r as f32, gamma.g as f32, gamma.b as f32, g.saturation as f32, ], - [gain.r as f32, gain.g as f32, gain.b as f32, 0.0], - ) + gain: [gain.r as f32, gain.g as f32, gain.b as f32, 0.0], + hsl_meta, + hsl_adjust, + } } /// Pack a draw's masks into the fixed-capacity uniform array, returning the count -/// the shader should evaluate. Linear + circle masks encode directly; polygon -/// masks (unsupported in-shader) encode as a full-coverage no-op so they neither -/// clip nor crash. Masks beyond [`MASK_CAP`] are dropped. +/// the shader should evaluate. Polygon paths are bounded to [`POLY_POINT_CAP`] +/// points. The shared edit-command validation prevents authored data from +/// exceeding either fixed GPU capacity; the `min`/`break` here is a deterministic +/// defensive fallback for an in-memory timeline that bypassed that boundary. fn pack_masks(draw: &LayerDraw<'_>) -> ([MaskGpu; MASK_CAP], f32) { let mut out = [MaskGpu::default(); MASK_CAP]; let mut n = 0usize; @@ -111,7 +167,8 @@ fn pack_masks(draw: &LayerDraw<'_>) -> ([MaskGpu; MASK_CAP], f32) { break; } let invert = if mask.invert { 1.0 } else { 0.0 }; - let (kind, geo) = match &mask.shape { + let mut points = [[0.0; 4]; POLY_POINT_CAP]; + let (kind, geo, point_count) = match &mask.shape { MaskShape::Linear { point, normal } => ( MASK_LINEAR, [ @@ -120,6 +177,7 @@ fn pack_masks(draw: &LayerDraw<'_>) -> ([MaskGpu; MASK_CAP], f32) { normal.x as f32, normal.y as f32, ], + 0, ), MaskShape::Circle { center, radius } => ( MASK_CIRCLE, @@ -129,31 +187,160 @@ fn pack_masks(draw: &LayerDraw<'_>) -> ([MaskGpu; MASK_CAP], f32) { radius.x as f32, radius.y as f32, ], + 0, ), - // Polygon masks are unsupported in-shader (TODO: storage buffer for - // points). Encode as a full-canvas circle so they are a visual no-op - // rather than silently clipping. - MaskShape::Poly { .. } => (MASK_CIRCLE, MASK_NOOP_GEO), + MaskShape::Poly { points: path } => { + let point_count = path.len().min(POLY_POINT_CAP); + for (target, point) in points.iter_mut().zip(path).take(point_count) { + *target = [point.x as f32, point.y as f32, 0.0, 0.0]; + } + (MASK_POLY, [0.0; 4], point_count) + } }; out[n] = MaskGpu { - head: [kind, mask.feather as f32, invert, 0.0], + head: [kind, mask.feather as f32, invert, point_count as f32], geo, + transform: [ + mask.transform.offset.x as f32, + mask.transform.offset.y as f32, + mask.transform.scale.x as f32, + mask.transform.scale.y as f32, + ], + transform_meta: [ + mask.transform.rotation_degrees.to_radians() as f32, + 0.0, + 0.0, + 0.0, + ], + points, }; n += 1; } (out, n as f32) } +fn pack_effects( + draw: &LayerDraw<'_>, +) -> Result<([EffectGpu; MAX_EFFECTS_PER_CLIP], f32), RenderError> { + validate_effect_chain(draw.effects)?; + let mut out = [EffectGpu::default(); MAX_EFFECTS_PER_CLIP]; + let mut count = 0usize; + for effect in draw.effects.iter().filter(|effect| effect.enabled) { + let kind = match effect.name.as_str() { + "grayscale" => EFFECT_GRAYSCALE, + "sepia" => EFFECT_SEPIA, + "invert" => EFFECT_INVERT, + _ => unreachable!("validate_effect_chain accepts only registered effects"), + }; + out[count] = EffectGpu { + data: [kind, effect.registered_param("amount")? as f32, 0.0, 0.0], + }; + count += 1; + } + Ok((out, count as f32)) +} + /// Working color format. The PoC composites in the sRGB non-linear domain /// (SPEC §3.7): an `Rgba8Unorm` target stores raw encoded bytes and blends them /// directly, matching AVFoundation most closely. Read-back returns those bytes. const RT_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm; +/// Frame reconstruction requested from the media resolver when source and +/// project rates differ. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TextureInterpolationMode { + Nearest, + Blend, + OpticalFlow, +} + +/// Deterministic recovery policy when the requested optical-flow backend is +/// unavailable for a resolver/device. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TextureInterpolationFallback { + Nearest, + Blend, + Error, +} + +/// Explicit source/target-rate contract shared by preview and export. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TextureInterpolationConfig { + pub source_fps: f64, + pub target_fps: f64, + pub mode: TextureInterpolationMode, + pub fallback: TextureInterpolationFallback, +} + +impl TextureInterpolationConfig { + pub fn new( + source_fps: f64, + target_fps: f64, + mode: TextureInterpolationMode, + fallback: TextureInterpolationFallback, + ) -> Result { + if !source_fps.is_finite() || source_fps <= 0.0 { + return Err("source_fps must be finite and greater than zero"); + } + if !target_fps.is_finite() || target_fps <= 0.0 { + return Err("target_fps must be finite and greater than zero"); + } + Ok(Self { + source_fps, + target_fps, + mode, + fallback, + }) + } + + /// Backward-compatible resolver behavior for callers that have not selected + /// a rate-conversion mode. + pub const fn passthrough() -> Self { + Self { + source_fps: 1.0, + target_fps: 1.0, + mode: TextureInterpolationMode::Nearest, + fallback: TextureInterpolationFallback::Nearest, + } + } +} + +/// Complete per-layer texture request. Keeping the interpolation contract on +/// the request prevents preview/export adapters from silently selecting +/// different reconstruction modes. +#[derive(Clone, Copy, Debug)] +pub struct TextureResolveRequest<'a> { + pub source: &'a TextureSource, + pub source_frame: i64, + pub interpolation: TextureInterpolationConfig, +} + /// Resolves a draw's [`TextureSource`] + source frame to a GPU texture. The /// compositor is decode-agnostic; the integrating layer (or a test) supplies /// pixels (e.g. via [`crate::source::FrameProvider`] + a cache). pub trait TextureResolver { fn resolve(&mut self, source: &TextureSource, source_frame: i64) -> Option>; + + /// Resolve through an explicit rate-conversion contract. Existing + /// resolvers remain nearest-frame compatible; optical-flow-aware resolvers + /// override this method and apply the requested fallback policy before GPU + /// upload. + fn resolve_with_interpolation( + &mut self, + request: TextureResolveRequest<'_>, + ) -> Option> { + self.resolve(request.source, request.source_frame) + } + + /// Resolve a validated project-managed LUT reference. The default keeps + /// source-only resolvers source-compatible; the compositor still fails a + /// draw carrying a LUT when no asset is returned. + fn resolve_lut( + &mut self, + _reference: &LutReference, + ) -> Result>, RenderError> { + Ok(None) + } } /// A textured-quad compositor bound to one device. @@ -161,6 +348,7 @@ pub struct Compositor { pipeline: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, + fallback_lut: GpuLutTexture, } impl Compositor { @@ -200,6 +388,22 @@ impl Compositor { ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D3, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, ], }); @@ -264,10 +468,38 @@ impl Compositor { ..Default::default() }); + // A bound texture is required even when a draw has no active LUT. The + // shader never samples this uninitialized 1x1 fallback when disabled. + let fallback_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("opentake-render inactive LUT binding"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + let fallback_view = fallback_texture.create_view(&wgpu::TextureViewDescriptor { + dimension: Some(wgpu::TextureViewDimension::D3), + ..Default::default() + }); + Compositor { pipeline, bind_group_layout, sampler, + fallback_lut: GpuLutTexture { + texture: fallback_texture, + view: fallback_view, + size: 1, + domain_min: [0.0; 3], + domain_max: [1.0; 3], + }, } } @@ -284,6 +516,28 @@ impl Compositor { size: RenderSize, frame_plan: &FramePlan<'_>, resolver: &mut dyn TextureResolver, + ) -> Result { + self.render_to_rgba_with_interpolation( + device, + queue, + size, + frame_plan, + resolver, + TextureInterpolationConfig::passthrough(), + ) + } + + /// Render with an explicit source/target-rate interpolation policy. Preview + /// and export pass the same value here so their resolver behavior cannot + /// drift independently. + pub fn render_to_rgba_with_interpolation( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + size: RenderSize, + frame_plan: &FramePlan<'_>, + resolver: &mut dyn TextureResolver, + interpolation: TextureInterpolationConfig, ) -> Result { let rt = device.create_texture(&wgpu::TextureDescriptor { label: Some("opentake-render target"), @@ -306,11 +560,26 @@ impl Compositor { struct Prepared { bind_group: wgpu::BindGroup, _tex: Rc, + _lut: Option>, } let mut prepared: Vec = Vec::with_capacity(frame_plan.draws.len()); for draw in &frame_plan.draws { - let Some(tex) = resolver.resolve(draw.source, draw.source_frame) else { + // Reject invalid persisted data even when the source is offline; + // an unknown effect or malformed grade must never degrade into an + // unchanged frame or reach the GPU as NaN/Inf uniforms. + let (effects, effect_count) = pack_effects(draw)?; + if let Some(grade) = draw.color_grade { + grade.validate()?; + } + if let Some(reference) = draw.lut { + reference.validate()?; + } + let Some(tex) = resolver.resolve_with_interpolation(TextureResolveRequest { + source: draw.source, + source_frame: draw.source_frame, + interpolation, + }) else { continue; }; // Assemble flags + the A-tier parameter blocks for this draw. @@ -319,7 +588,7 @@ impl Compositor { } else { 0 }; - let (grade_exp_wb, grade_lift, grade_gamma, grade_gain) = match draw.color_grade { + let grade = match draw.color_grade { Some(g) if !g.is_identity() => { flags |= FLAG_GRADE; grade_blocks(g) @@ -342,6 +611,33 @@ impl Compositor { None => ([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]), }; let (masks, mask_count) = pack_masks(draw); + let resolved_lut = match draw.lut { + Some(reference) => Some( + resolver + .resolve_lut(reference)? + .ok_or_else(|| RenderError::MissingLut(reference.id.clone()))?, + ), + None => None, + }; + let (lut_meta, lut_domain_min, lut_domain_scale) = match draw.lut { + Some(reference) => { + let parsed = resolved_lut.as_ref().expect("resolved above"); + let domain_scale: [f32; 3] = std::array::from_fn(|channel| { + 1.0 / (parsed.domain_max[channel] - parsed.domain_min[channel]) + }); + ( + [1.0, reference.intensity as f32, parsed.size as f32, 0.0], + [ + parsed.domain_min[0], + parsed.domain_min[1], + parsed.domain_min[2], + 0.0, + ], + [domain_scale[0], domain_scale[1], domain_scale[2], 0.0], + ) + } + None => ([0.0; 4], [0.0; 4], [1.0, 1.0, 1.0, 0.0]), + }; let u = Uniforms { affine0: [ draw.affine[0] as f32, @@ -374,14 +670,21 @@ impl Compositor { draw.opacity as f32, f32::from_bits(flags), ], - grade_exp_wb, - grade_lift, - grade_gamma, - grade_gain, + grade_exp_wb: grade.exp_wb, + grade_lift: grade.lift, + grade_gamma: grade.gamma, + grade_gain: grade.gain, + hsl_secondary_meta: grade.hsl_meta, + hsl_secondary_adjust: grade.hsl_adjust, + lut_meta, + lut_domain_min, + lut_domain_scale, chroma0, chroma1, mask_meta: [mask_count, 0.0, 0.0, 0.0], masks, + effect_meta: [effect_count, 0.0, 0.0, 0.0], + effects, }; let ubuf = device.create_buffer(&wgpu::BufferDescriptor { label: Some("opentake-render uniform"), @@ -391,6 +694,10 @@ impl Compositor { }); queue.write_buffer(&ubuf, 0, bytemuck::bytes_of(&u)); + let lut_view = resolved_lut + .as_ref() + .map_or(&self.fallback_lut.view, |lut| &lut.view); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("opentake-render bind group"), layout: &self.bind_group_layout, @@ -407,11 +714,20 @@ impl Compositor { binding: 2, resource: wgpu::BindingResource::Sampler(&self.sampler), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(lut_view), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, ], }); prepared.push(Prepared { bind_group, _tex: tex, + _lut: resolved_lut, }); } diff --git a/crates/opentake-render/src/gpu/mod.rs b/crates/opentake-render/src/gpu/mod.rs index 4b4b11b5..9115a0bd 100644 --- a/crates/opentake-render/src/gpu/mod.rs +++ b/crates/opentake-render/src/gpu/mod.rs @@ -17,7 +17,7 @@ pub use compositor::{Compositor, TextureResolver}; pub use device::RenderDevice; pub use text_engine::CosmicTextRasterizer; pub use text_raster::{NullTextRasterizer, TextRasterRequest, TextRasterizer}; -pub use texture::{upload_rgba, GpuTexture, TextureCache}; +pub use texture::{upload_lut_3d, upload_rgba, GpuLutTexture, GpuTexture, TextureCache}; /// Errors from GPU device acquisition and frame compositing. #[derive(Debug, thiserror::Error)] @@ -29,4 +29,14 @@ pub enum RenderError { DeviceRequest(String), #[error("frame read-back failed: {0}")] Readback(String), + #[error("invalid effect chain: {0}")] + InvalidEffect(#[from] opentake_domain::EffectValidationError), + #[error("invalid color grade: {0}")] + InvalidColorGrade(#[from] opentake_domain::ColorGradeValidationError), + #[error("invalid LUT reference: {0}")] + InvalidLutReference(#[from] opentake_domain::LutReferenceValidationError), + #[error("LUT asset could not be resolved: {0}")] + MissingLut(String), + #[error("invalid LUT asset: {0}")] + InvalidLut(String), } diff --git a/crates/opentake-render/src/gpu/shader.wgsl b/crates/opentake-render/src/gpu/shader.wgsl index 5a8d2157..6f2b0e89 100644 --- a/crates/opentake-render/src/gpu/shader.wgsl +++ b/crates/opentake-render/src/gpu/shader.wgsl @@ -26,28 +26,39 @@ // on the (sampled) color directly, matching the domain reference which is // space-agnostic for those stages. // -// MASK CAP: up to MASK_CAP masks are evaluated in-shader (linear + circle SDF). -// Polygon masks are carried in the domain/plan and fully unit-tested there, but -// their variable-length point list does not fit this fixed uniform; wiring -// polygon points through a storage buffer is a documented render-side TODO. +// MASK CAP: up to MASK_CAP masks and POLY_POINT_CAP pen points per mask are +// evaluated in-shader. The fixed point cap keeps the uniform layout portable. const MASK_CAP: u32 = 4u; +const POLY_POINT_CAP: u32 = 16u; +const EFFECT_CAP: u32 = 8u; // Flag bits packed into U.canvas_op_flags.w (bitcast to u32). const FLAG_PREMULTIPLY: u32 = 1u; // straight-alpha source needs premultiply const FLAG_GRADE: u32 = 2u; // color grade active const FLAG_CHROMA: u32 = 4u; // chroma key active -// Mask kind tags (mirror MaskShape; poly is not evaluated in-shader). +// Mask kind tags (mirror MaskShape). const MASK_LINEAR: u32 = 0u; const MASK_CIRCLE: u32 = 1u; +const MASK_POLY: u32 = 2u; struct MaskGpu { - // (kind-as-f32, feather, invert-as-f32, pad) + // (kind-as-f32, feather, invert-as-f32, polygon-point-count) head: vec4, // linear: (point.x, point.y, normal.x, normal.y) // circle: (center.x, center.y, radius.x, radius.y) geo: vec4, + // (offset.x, offset.y, scale.x, scale.y) + transform: vec4, + // (rotation radians, pad, pad, pad) + transform_meta: vec4, + points: array, POLY_POINT_CAP>, +}; + +struct EffectGpu { + // (kind, amount, pad, pad) + data: vec4, }; // Laid out as vec4s so every field is 16-byte aligned (no implicit WGSL padding) @@ -62,17 +73,43 @@ struct U { grade_lift: vec4, // lift_r, lift_g, lift_b, contrast grade_gamma: vec4, // gamma_r, gamma_g, gamma_b, saturation grade_gain: vec4, // gain_r, gain_g, gain_b, pad + hsl_secondary_meta: vec4, // enabled, hue center, full width, feather + hsl_secondary_adjust: vec4, // hue shift, saturation, lightness, pad + lut_meta: vec4, // enabled, intensity, table size, pad + lut_domain_min: vec4, // min r/g/b, pad + lut_domain_scale: vec4, // reciprocal domain span r/g/b, pad // Chroma key. chroma0: vec4, // key_r, key_g, key_b, similarity chroma1: vec4, // smoothness, spill, pad, pad // Mask count (x) + padding. mask_meta: vec4, // mask_count, pad, pad, pad masks: array, + effect_meta: vec4, // effect_count, pad, pad, pad + effects: array, }; @group(0) @binding(0) var u: U; @group(0) @binding(1) var t_color: texture_2d; @group(0) @binding(2) var s_color: sampler; +@group(0) @binding(3) var t_lut: texture_3d; +@group(0) @binding(4) var s_lut: sampler; + +fn apply_lut(rgb: vec3) -> vec3 { + if (u.lut_meta.x < 0.5 || u.lut_meta.y <= 0.0) { + return rgb; + } + let normalized = clamp( + (rgb - u.lut_domain_min.xyz) * u.lut_domain_scale.xyz, + vec3(0.0), + vec3(1.0), + ); + let table_size = u.lut_meta.z; + // Align authored grid points i/(N-1) with texel centers before hardware + // trilinear filtering. + let coordinate = (normalized * (table_size - 1.0) + vec3(0.5)) / table_size; + let transformed = textureSample(t_lut, s_lut, coordinate).rgb; + return mix(rgb, transformed, clamp(u.lut_meta.y, 0.0, 1.0)); +} struct VsOut { @builtin(position) pos: vec4, @@ -112,15 +149,84 @@ fn smoothstep01(edge0: f32, edge1: f32, x: f32) -> f32 { const CONTRAST_PIVOT: f32 = 0.18; fn apply_channel_lgg(x: f32, lift: f32, gamma: f32, gain: f32) -> f32 { - let v = gain * (x + lift); + let shaped = x + lift * (1.0 - x); if (abs(gamma - 1.0) > 1e-6 && gamma > 0.0) { - return pow(max(v, 0.0), 1.0 / gamma); + return gain * pow(max(shaped, 0.0), 1.0 / gamma); + } + return gain * shaped; +} + +fn rgb_to_hsl(c: vec3) -> vec3 { + let maximum = max(c.r, max(c.g, c.b)); + let minimum = min(c.r, min(c.g, c.b)); + let delta = maximum - minimum; + let lightness = (maximum + minimum) * 0.5; + if (delta <= 1e-7) { + return vec3(0.0, 0.0, lightness); + } + let saturation = delta / max(1.0 - abs(2.0 * lightness - 1.0), 1e-7); + var sector = 0.0; + if (maximum == c.r) { + sector = ((c.g - c.b) / delta) % 6.0; + if (sector < 0.0) { + sector = sector + 6.0; + } + } else if (maximum == c.g) { + sector = (c.b - c.r) / delta + 2.0; + } else { + sector = (c.r - c.g) / delta + 4.0; } - return v; + return vec3(sector / 6.0, saturation, lightness); +} + +fn hsl_to_rgb(hsl: vec3) -> vec3 { + let chroma = (1.0 - abs(2.0 * hsl.z - 1.0)) * hsl.y; + let sector = fract(hsl.x) * 6.0; + let x = chroma * (1.0 - abs((sector % 2.0) - 1.0)); + var rgb = vec3(0.0); + if (sector < 1.0) { + rgb = vec3(chroma, x, 0.0); + } else if (sector < 2.0) { + rgb = vec3(x, chroma, 0.0); + } else if (sector < 3.0) { + rgb = vec3(0.0, chroma, x); + } else if (sector < 4.0) { + rgb = vec3(0.0, x, chroma); + } else if (sector < 5.0) { + rgb = vec3(x, 0.0, chroma); + } else { + rgb = vec3(chroma, 0.0, x); + } + return rgb + vec3(hsl.z - chroma * 0.5); +} + +fn apply_hsl_secondary(c: vec3) -> vec3 { + if (u.hsl_secondary_meta.x < 0.5) { + return c; + } + var hsl = rgb_to_hsl(c); + if (hsl.y <= 1e-7) { + return c; + } + let delta = abs(fract(hsl.x - u.hsl_secondary_meta.y + 0.5) - 0.5); + let outer = u.hsl_secondary_meta.z * 0.5; + if (delta > outer) { + return c; + } + let feather = u.hsl_secondary_meta.w; + var weight = 1.0; + if (feather > 1e-7) { + weight = 1.0 - smoothstep01(max(outer - feather, 0.0), outer, delta); + } + hsl.x = fract(hsl.x + u.hsl_secondary_adjust.x * weight + 1.0); + hsl.y = clamp(hsl.y * (1.0 + u.hsl_secondary_adjust.y * weight), 0.0, 1.0); + hsl.z = clamp(hsl.z + u.hsl_secondary_adjust.z * weight, 0.0, 1.0); + return hsl_to_rgb(hsl); } // Applies the grade to a LINEAR-rgb triple, returning clamped linear rgb. Mirror -// of ColorGrade::apply_linear (exposure -> wb -> lgg -> contrast -> saturation). +// of ColorGrade::apply_linear +// (exposure -> wb -> lgg -> contrast -> saturation -> HSL secondary). fn apply_grade_linear(rgb_in: vec3) -> vec3 { var c = rgb_in; @@ -151,6 +257,9 @@ fn apply_grade_linear(rgb_in: vec3) -> vec3 { let l = luma709(c); c = vec3(l) + (c - vec3(l)) * saturation; + // 6. Feathered HSL secondary qualifier. + c = apply_hsl_secondary(c); + return clamp(c, vec3(0.0), vec3(1.0)); } @@ -193,10 +302,60 @@ fn suppress_spill(c: vec3) -> vec3 { return vec3(nr, c.g, c.b); } -// ---- Masks (mirror of Mask::coverage; linear + circle in-shader) ------------ +// ---- Masks (mirror of Mask::coverage) --------------------------------------- + +fn mask_local_point(m: MaskGpu, p: vec2) -> vec2 { + let scale = max(abs(m.transform.zw), vec2(1e-6)); + let radians = m.transform_meta.x; + let c = cos(radians); + let s = sin(radians); + let delta = p - vec2(0.5) - m.transform.xy; + let unrotated = vec2( + c * delta.x + s * delta.y, + -s * delta.x + c * delta.y, + ); + return unrotated / scale + vec2(0.5); +} + +fn point_segment_dist2(p: vec2, a: vec2, b: vec2) -> f32 { + let ab = b - a; + let denom = dot(ab, ab); + var t = 0.0; + if (denom > 1e-12) { + t = clamp(dot(p - a, ab) / denom, 0.0, 1.0); + } + let delta = p - (a + ab * t); + return dot(delta, delta); +} + +fn polygon_signed_distance(m: MaskGpu, p: vec2) -> f32 { + let count = min(u32(m.head.w + 0.5), POLY_POINT_CAP); + if (count < 3u) { + return 1e6; + } + var inside = false; + var min_d2 = 1e12; + var j = count - 1u; + for (var i: u32 = 0u; i < count; i = i + 1u) { + let a = m.points[i].xy; + let b = m.points[j].xy; + let crosses_y = (a.y > p.y) != (b.y > p.y); + if (crosses_y) { + let edge_x = (b.x - a.x) * (p.y - a.y) / (b.y - a.y) + a.x; + if (p.x < edge_x) { + inside = !inside; + } + } + min_d2 = min(min_d2, point_segment_dist2(p, a, b)); + j = i; + } + let distance = sqrt(min_d2); + return select(distance, -distance, inside); +} fn mask_signed_distance(m: MaskGpu, p: vec2) -> f32 { let kind = u32(m.head.x + 0.5); + let local = mask_local_point(m, p); if (kind == MASK_LINEAR) { let point = m.geo.xy; let normal = m.geo.zw; @@ -205,12 +364,15 @@ fn mask_signed_distance(m: MaskGpu, p: vec2) -> f32 { return 0.0; } let n = normal / nlen; - return -dot(p - point, n); + return -dot(local - point, n); + } + if (kind == MASK_POLY) { + return polygon_signed_distance(m, local); } // Circle (default for any other tag). let center = m.geo.xy; let radius = max(m.geo.zw, vec2(1e-6)); - let d = length((p - center) / radius); + let d = length((local - center) / radius); return (d - 1.0) * min(radius.x, radius.y); } @@ -237,6 +399,39 @@ fn masks_coverage(p: vec2) -> f32 { return cov; } +// ---- Closed generic effect chain ------------------------------------------ + +const EFFECT_GRAYSCALE: u32 = 0u; +const EFFECT_SEPIA: u32 = 1u; +const EFFECT_INVERT: u32 = 2u; + +fn apply_effect(effect: EffectGpu, input: vec3) -> vec3 { + let kind = u32(effect.data.x + 0.5); + let amount = clamp(effect.data.y, 0.0, 1.0); + var transformed = input; + if (kind == EFFECT_GRAYSCALE) { + transformed = vec3(luma709(input)); + } else if (kind == EFFECT_SEPIA) { + transformed = vec3( + dot(input, vec3(0.393, 0.769, 0.189)), + dot(input, vec3(0.349, 0.686, 0.168)), + dot(input, vec3(0.272, 0.534, 0.131)), + ); + } else if (kind == EFFECT_INVERT) { + transformed = vec3(1.0) - input; + } + return clamp(mix(input, transformed, amount), vec3(0.0), vec3(1.0)); +} + +fn apply_effect_chain(input: vec3) -> vec3 { + let count = min(u32(u.effect_meta.x + 0.5), EFFECT_CAP); + var result = input; + for (var i: u32 = 0u; i < count; i = i + 1u) { + result = apply_effect(u.effects[i], result); + } + return result; +} + @vertex fn vs(@builtin(vertex_index) vi: u32) -> VsOut { // Triangle-strip quad: (0,0) (1,0) (0,1) (1,1). @@ -333,7 +528,13 @@ fn fs(in: VsOut) -> @location(0) vec4 { rgb = linear_to_srgb(graded); } - // 3. Masks (intersected coverage) scale alpha. + // 3. Project-managed 3D LUT in display-encoded RGB. + rgb = apply_lut(rgb); + + // 4. Ordered, schema-validated generic effects. + rgb = apply_effect_chain(rgb); + + // 5. Masks (intersected coverage) scale alpha. alpha = alpha * masks_coverage(in.canvas_uv); // Premultiply once (the compositor blends premultiplied over), then apply the diff --git a/crates/opentake-render/src/gpu/text_engine.rs b/crates/opentake-render/src/gpu/text_engine.rs index d87668d8..d22e4353 100644 --- a/crates/opentake-render/src/gpu/text_engine.rs +++ b/crates/opentake-render/src/gpu/text_engine.rs @@ -16,16 +16,17 @@ //! - **Rich text** (multi-span styles): cosmic-text `Buffer::set_rich_text` takes //! an iterator of `(text, Attrs)` spans; the current path uses `set_text` //! (single style per clip) — switching needs a `TextRasterRequest` shape change. -//! - **Emoji / CJK fallback**: cosmic-text 0.12 `Attrs` has no `family_emoji` / -//! `family_asian` (added in 0.14+); fallback relies on fontdb's default -//! sans-serif. TODO when the crate is bumped. +//! - **Emoji / CJK selection hints**: cosmic-text 0.12 `Attrs` has no +//! `family_emoji` / `family_asian` (added in 0.14+), so fallback uses its +//! script-aware fontdb search rather than an explicit preferred family. //! - **Vertical text**: unsupported (upstream doesn't expose it via `TextStyle`); //! v1 leaves it horizontal. use std::cell::RefCell; use cosmic_text::{ - Align, Attrs, Buffer, Color, Family, FontSystem, Metrics, Shaping, Style, SwashCache, Weight, + fontdb, Align, Attrs, Buffer, Color, Family, FontSystem, Metrics, Shaping, Style, SwashCache, + Weight, }; use opentake_domain::{Rgba, TextAlignment}; @@ -57,9 +58,23 @@ impl CosmicTextRasterizer { /// directories once and is mildly expensive (~tens of ms); construct it once /// and reuse. pub fn new() -> Self { + Self::from_font_system(FontSystem::new()) + } + + /// Build the deterministic no-font fallback used by headless runtimes and + /// tests. Text requests still yield a correctly sized premultiplied frame; + /// glyph coverage is empty while background and border styles remain live. + pub fn without_system_fonts() -> Self { + Self::from_font_system(FontSystem::new_with_locale_and_db( + "en-US".to_string(), + fontdb::Database::new(), + )) + } + + fn from_font_system(font_system: FontSystem) -> Self { CosmicTextRasterizer { inner: RefCell::new(Inner { - font_system: FontSystem::new(), + font_system, swash_cache: SwashCache::new(), }), } @@ -180,6 +195,22 @@ fn rasterize_box(inner: &mut Inner, req: &TextRasterRequest<'_>) -> Option) -> Option, +) -> GpuLutTexture { + upload_lut_table_3d( + device, + queue, + lut.size(), + lut.domain_min(), + lut.domain_max(), + lut.table(), + label, + ) +} + +pub(crate) fn upload_lut_table_3d( + device: &wgpu::Device, + queue: &wgpu::Queue, + lut_size: u32, + domain_min: [f32; 3], + domain_max: [f32; 3], + table: &[[f32; 3]], + label: Option<&str>, +) -> GpuLutTexture { + let extent = wgpu::Extent3d { + width: lut_size, + height: lut_size, + depth_or_array_layers: lut_size, + }; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label, + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let mut rgba = Vec::::with_capacity(table.len() * 4); + for value in table { + rgba.extend(value.map(|channel| half::f16::from_f32(channel).to_bits())); + rgba.push(half::f16::ONE.to_bits()); + } + queue.write_texture( + wgpu::ImageCopyTexture { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + bytemuck::cast_slice(&rgba), + wgpu::ImageDataLayout { + offset: 0, + bytes_per_row: Some(lut_size * 8), + rows_per_image: Some(lut_size), + }, + extent, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor { + dimension: Some(wgpu::TextureViewDimension::D3), + ..Default::default() + }); + GpuLutTexture { + texture, + view, + size: lut_size, + domain_min, + domain_max, + } +} + /// Upload a [`DecodedFrame`] as an RGBA8 texture. /// /// `srgb` selects the texture format: `Rgba8UnormSrgb` makes the sampler return diff --git a/crates/opentake-render/src/lib.rs b/crates/opentake-render/src/lib.rs index 893896a4..639a82f9 100644 --- a/crates/opentake-render/src/lib.rs +++ b/crates/opentake-render/src/lib.rs @@ -14,13 +14,14 @@ pub mod source; pub use wgpu; pub use plan::{ - affine_transform, build_render_plan, compose, crop_to_uv, source_frame_index, ClipPlan, - FramePlan, LayerDraw, RenderPlan, RenderSize, TextureSource, + affine_transform, build_render_plan, compose, crop_to_uv, source_frame_index, + try_build_render_plan, AudioClipPlan, ClipPlan, CompoundAncestor, FramePlan, LayerDraw, + RenderPlan, RenderSize, TextureSource, }; pub use size::{even, export_render_size, ExportResolution}; pub use source::{DecodedFrame, FrameProvider, SourceMetrics}; pub use gpu::{ - Compositor, CosmicTextRasterizer, GpuTexture, NullTextRasterizer, RenderDevice, RenderError, - TextRasterRequest, TextRasterizer, TextureCache, TextureResolver, + Compositor, CosmicTextRasterizer, GpuLutTexture, GpuTexture, NullTextRasterizer, RenderDevice, + RenderError, TextRasterRequest, TextRasterizer, TextureCache, TextureResolver, }; diff --git a/crates/opentake-render/src/plan/build.rs b/crates/opentake-render/src/plan/build.rs index 2e84da15..e64a7564 100644 --- a/crates/opentake-render/src/plan/build.rs +++ b/crates/opentake-render/src/plan/build.rs @@ -6,10 +6,15 @@ //! keyframe / fade / dB sample goes through the domain `*_at` methods (SPEC §0 //! iron rule); this module only adds geometry projection + frame scheduling. -use opentake_domain::{Clip, ClipType, Timeline}; +use std::collections::{HashMap, HashSet}; + +use opentake_domain::{Clip, ClipType, NestedSequence, Timeline, TransitionKind}; use super::affine::{affine_transform, compose, crop_to_uv}; -use super::types::{ClipPlan, FramePlan, LayerDraw, RenderPlan, RenderSize, TextureSource}; +use super::types::{ + AudioClipPlan, ClipPlan, CompoundAncestor, FramePlan, LayerDraw, RenderPlan, RenderSize, + TextureSource, +}; use crate::source::SourceMetrics; /// Half-away-from-zero round, matching the domain convention (`clip.rs` L7). @@ -65,11 +70,42 @@ fn texture_source_for(clip: &Clip) -> TextureSource { } } +/// Compose authored transform animation with the clip-relative stabilization +/// track. The same helper is used by ordinary frames, transitions, preview, and +/// export, preventing separate stabilization math from drifting by surface. +fn evaluated_transform( + clip: &Clip, + frame: i32, + render_size: RenderSize, +) -> opentake_domain::Transform { + let mut transform = if clip.has_transform_animation() { + clip.transform_at(frame) + } else { + clip.transform + }; + if let Some(stabilization) = &clip.stabilization { + let correction = stabilization.sample(frame - clip.start_frame); + let scale = stabilization.crop_scale(render_size.width_f() / render_size.height_f()); + transform.center_x += correction.translation_x; + transform.center_y += correction.translation_y; + transform.width *= scale; + transform.height *= scale; + transform.rotation += correction.rotation_degrees; + } + transform +} + /// Build a [`ClipPlan`] for one selected clip. +#[allow(clippy::too_many_arguments)] fn make_clip_plan( clip: &Clip, track_index: usize, clip_index: usize, + blend_path: Vec, + compound_ancestors: Vec, + visible_start: i32, + visible_end: i32, + effective_trim_start: i32, sources: &dyn SourceMetrics, render_size: RenderSize, ) -> ClipPlan { @@ -111,24 +147,28 @@ fn make_clip_plan( }; ClipPlan { + clip: clip.clone(), clip_id: clip.id.clone(), track_index, clip_index, + blend_path, + compound_ancestors, source: texture_source_for(clip), - start_frame: clip.start_frame, - end_frame: clip.end_frame(), + start_frame: visible_start, + end_frame: visible_end, nat_size, preferred_transform, needs_premultiply, speed: clip.speed, reversed: clip.reversed, - trim_start_frame: clip.trim_start_frame, + trim_start_frame: effective_trim_start, media_type: clip.media_type, lottie_frame_count, // Advanced pixel-effect inputs, copied verbatim from the clip (frame- // independent this round). Drop a color grade that is the identity so the // compositor can skip it cheaply. color_grade: clip.color_grade.filter(|g| !g.is_identity()), + lut: clip.lut.clone(), chroma_key: clip.chroma_key, masks: clip.masks.clone(), effects: clip.effects.clone(), @@ -151,10 +191,84 @@ pub fn build_render_plan( render_size: RenderSize, sources: &dyn SourceMetrics, ) -> RenderPlan { + try_build_render_plan(timeline, render_size, sources) + .expect("timeline graph must be valid before building a render plan") +} + +/// Fail-closed render-plan construction used by preview, export, and agents. +pub fn try_build_render_plan( + timeline: &Timeline, + render_size: RenderSize, + sources: &dyn SourceMetrics, +) -> Result { + timeline.validate_nested_sequences()?; + validate_nested_render_constraints(timeline)?; let total_frames = timeline.total_frames(); let mut clip_plans: Vec = Vec::new(); let mut text_plans: Vec = Vec::new(); + let mut audio_clips: Vec = Vec::new(); + + let registry: HashMap<&str, &NestedSequence> = timeline + .nested_sequences + .iter() + .map(|sequence| (sequence.id.as_str(), sequence)) + .collect(); + collect_timeline_plans( + timeline, + ®istry, + 0, + None, + None, + &[], + &[], + render_size, + sources, + &mut clip_plans, + &mut text_plans, + ); + collect_nested_audio( + timeline, + ®istry, + 0, + None, + None, + false, + &[], + &mut audio_clips, + ); + + // Final blend order: bottom-to-top. Upstream keeps visual track 0 topmost, + // so higher track indexes draw first and lower indexes draw last. + clip_plans.sort_by(|a, b| { + b.blend_path + .cmp(&a.blend_path) + .then(a.start_frame.cmp(&b.start_frame)) + }); + + Ok(RenderPlan { + fps: timeline.fps, + render_size, + total_frames, + clip_plans, + text_plans, + audio_clips, + }) +} +#[allow(clippy::too_many_arguments)] +fn collect_timeline_plans( + timeline: &Timeline, + registry: &HashMap<&str, &NestedSequence>, + frame_offset: i32, + parent_start: Option, + parent_end: Option, + parent_blend_path: &[usize], + compound_ancestors: &[CompoundAncestor], + render_size: RenderSize, + sources: &dyn SourceMetrics, + clip_plans: &mut Vec, + text_plans: &mut Vec, +) { for (track_index, track) in timeline.tracks.iter().enumerate() { if track.hidden { continue; @@ -167,17 +281,79 @@ pub fn build_render_plan( order.sort_by_key(|&i| track.clips[i].start_frame); let mut prev_end_frame = i32::MIN; + let mut blend_path = parent_blend_path.to_vec(); + blend_path.push(track_index); for &clip_index in &order { let clip = &track.clips[clip_index]; + let unclamped_start = frame_offset.saturating_add(clip.start_frame); + let absolute_start = unclamped_start.max(parent_start.unwrap_or(i32::MIN)); + let absolute_end = frame_offset + .saturating_add(clip.end_frame()) + .min(parent_end.unwrap_or(i32::MAX)); + if absolute_end <= absolute_start { + continue; + } + + let clipped_left = absolute_start - unclamped_start; + let effective_trim_start = clip + .trim_start_frame + .saturating_add((clipped_left as f64 * clip.speed).round() as i32); + let mut mapped_clip = clip.clone(); + mapped_clip.start_frame = unclamped_start; + + // Apply the same overlap policy to compound and ordinary visual + // clips before recursively expanding the selected compound. + if clip.media_type != ClipType::Text { + if clip.duration_frames <= 0 || clip.start_frame < prev_end_frame { + continue; + } + prev_end_frame = clip.end_frame(); + } + + if let Some(sequence_id) = clip.nested_sequence_id.as_deref() { + let sequence = registry + .get(sequence_id) + .expect("validated nested reference must exist"); + let mut child_ancestors = compound_ancestors.to_vec(); + child_ancestors.push(CompoundAncestor { + clip: mapped_clip, + // Flattened leaves are already projected into the output + // canvas' normalized coordinate space. Compound transforms + // therefore operate on that output canvas as their source; + // using the authored child pixel size here would scale the + // same normalization a second time. + canvas_size: (render_size.width_f(), render_size.height_f()), + }); + collect_timeline_plans( + &sequence.timeline, + registry, + absolute_start.saturating_sub(effective_trim_start), + Some(absolute_start), + Some(absolute_end), + &blend_path, + &child_ancestors, + render_size, + sources, + clip_plans, + text_plans, + ); + continue; + } + if clip.media_type == ClipType::Text { // Text: no overlap skip, no audio gate; each text clip stands // alone (SPEC §4.2). Defensive: require a positive span. if clip.duration_frames > 0 { text_plans.push(make_clip_plan( - clip, + &mapped_clip, track_index, clip_index, + blend_path.clone(), + compound_ancestors.to_vec(), + absolute_start, + absolute_end, + effective_trim_start, sources, render_size, )); @@ -191,34 +367,143 @@ pub fn build_render_plan( } // Video-track de-dup (upstream L152 / L424). - if clip.duration_frames <= 0 || clip.start_frame < prev_end_frame { - continue; - } clip_plans.push(make_clip_plan( - clip, + &mapped_clip, track_index, clip_index, + blend_path.clone(), + compound_ancestors.to_vec(), + absolute_start, + absolute_end, + effective_trim_start, sources, render_size, )); - prev_end_frame = clip.end_frame(); } } +} - // Final blend order: bottom-to-top. Upstream keeps visual track 0 topmost, - // so higher track indexes draw first and lower indexes draw last. - clip_plans.sort_by(|a, b| { - b.track_index - .cmp(&a.track_index) - .then(a.start_frame.cmp(&b.start_frame)) - }); +fn validate_nested_render_constraints(timeline: &Timeline) -> Result<(), String> { + let mut timelines = Vec::with_capacity(timeline.nested_sequences.len() + 1); + timelines.push(timeline); + timelines.extend( + timeline + .nested_sequences + .iter() + .map(|sequence| &sequence.timeline), + ); + let compound_ids = timelines + .iter() + .flat_map(|candidate| candidate.tracks.iter()) + .flat_map(|track| &track.clips) + .filter(|clip| clip.nested_sequence_id.is_some()) + .map(|clip| clip.id.as_str()) + .collect::>(); + for candidate in timelines { + for clip in candidate.tracks.iter().flat_map(|track| &track.clips) { + if clip + .transition_out + .as_ref() + .is_some_and(|transition| compound_ids.contains(transition.to_clip_id.as_str())) + { + return Err(format!( + "transition into compound clip {} requires offscreen nesting", + clip.transition_out + .as_ref() + .expect("transition was matched") + .to_clip_id + )); + } + } + for clip in candidate + .tracks + .iter() + .flat_map(|track| track.clips.iter()) + .filter(|clip| clip.nested_sequence_id.is_some()) + { + if (clip.speed - 1.0).abs() > f64::EPSILON || clip.reversed { + return Err(format!( + "compound clip {} must use forward 1x playback", + clip.id + )); + } + if clip.crop != Default::default() + || clip.crop_track.is_some() + || clip.color_grade.is_some() + || clip.chroma_key.is_some() + || !clip.masks.is_empty() + || !clip.effects.is_empty() + || clip.transition_out.is_some() + { + return Err(format!( + "compound clip {} uses effects that require offscreen nesting", + clip.id + )); + } + } + } + Ok(()) +} - RenderPlan { - fps: timeline.fps, - render_size, - total_frames, - clip_plans, - text_plans, +#[allow(clippy::too_many_arguments)] +fn collect_nested_audio( + timeline: &Timeline, + registry: &HashMap<&str, &NestedSequence>, + frame_offset: i32, + parent_start: Option, + parent_end: Option, + parent_muted: bool, + compound_ancestors: &[Clip], + audio_clips: &mut Vec, +) { + for track in &timeline.tracks { + let muted = parent_muted || track.muted; + for clip in &track.clips { + let unclamped_start = frame_offset.saturating_add(clip.start_frame); + let absolute_start = unclamped_start.max(parent_start.unwrap_or(i32::MIN)); + let absolute_end = frame_offset + .saturating_add(clip.end_frame()) + .min(parent_end.unwrap_or(i32::MAX)); + if absolute_end <= absolute_start { + continue; + } + let clipped_left = absolute_start - unclamped_start; + let effective_trim_start = clip + .trim_start_frame + .saturating_add((clipped_left as f64 * clip.speed).round() as i32); + let mut mapped_gain_clip = clip.clone(); + mapped_gain_clip.start_frame = unclamped_start; + if let Some(sequence_id) = clip.nested_sequence_id.as_deref() { + let sequence = registry + .get(sequence_id) + .expect("validated nested reference must exist"); + let mut child_ancestors = compound_ancestors.to_vec(); + child_ancestors.push(mapped_gain_clip); + collect_nested_audio( + &sequence.timeline, + registry, + absolute_start.saturating_sub(effective_trim_start), + Some(absolute_start), + Some(absolute_end), + muted, + &child_ancestors, + audio_clips, + ); + continue; + } + if muted || !matches!(clip.media_type, ClipType::Audio | ClipType::Video) { + continue; + } + let mut flattened = clip.clone(); + flattened.start_frame = absolute_start; + flattened.duration_frames = absolute_end - absolute_start; + flattened.trim_start_frame = effective_trim_start; + audio_clips.push(AudioClipPlan { + clip: flattened, + gain_clip: mapped_gain_clip, + compound_ancestors: compound_ancestors.to_vec(), + }); + } } } @@ -276,7 +561,10 @@ fn eval_layer<'a>( if f < plan.start_frame || f >= plan.end_frame { return None; } - let opacity = clip.opacity_at(f); + let mut opacity = clip.opacity_at(f); + for ancestor in plan.compound_ancestors.iter().rev() { + opacity *= ancestor.clip.opacity_at(f); + } if opacity <= 0.0 { return None; // behavior-equivalent skip (SPEC §2.4 step 3). } @@ -285,15 +573,18 @@ fn eval_layer<'a>( // path uses `clip.transformAt(frame)` (which rebuilds top-left/size/rotation // and intentionally drops flip — matching domain `transform_at`). Replicate // that split so flip behaves exactly as upstream. - let transform = if clip.has_transform_animation() { - clip.transform_at(f) - } else { - clip.transform - }; - let affine = compose( + let transform = evaluated_transform(clip, f, render_size); + let mut affine = compose( plan.preferred_transform, affine_transform(&transform, plan.nat_size, render_size), ); + for ancestor in plan.compound_ancestors.iter().rev() { + let transform = evaluated_transform(&ancestor.clip, f, render_size); + affine = compose( + affine, + affine_transform(&transform, ancestor.canvas_size, render_size), + ); + } let crop_uv = crop_to_uv(clip.crop_at(f)); let source_frame = source_frame_index(plan, f); @@ -310,6 +601,54 @@ fn eval_layer<'a>( needs_premultiply: plan.needs_premultiply, clip_id: &plan.clip_id, color_grade: plan.color_grade.as_ref(), + lut: plan.lut.as_ref(), + chroma_key: plan.chroma_key.as_ref(), + masks: &plan.masks, + effects: &plan.effects, + }) +} + +/// Evaluate the incoming side of a cross dissolve before its nominal timeline +/// start. The first source frame is held during the dissolve, then regular +/// playback begins at the cut; this avoids reading outside the clip's source +/// window while preserving timeline duration and adjacency. +fn eval_transition_incoming<'a>( + plan: &'a ClipPlan, + clip: &Clip, + progress: f64, + render_size: RenderSize, +) -> Option> { + let sample_frame = plan.start_frame; + let mut opacity = clip.raw_opacity_at(sample_frame) * progress.clamp(0.0, 1.0); + for ancestor in plan.compound_ancestors.iter().rev() { + opacity *= ancestor.clip.opacity_at(sample_frame); + } + if opacity <= 0.0 { + return None; + } + let transform = evaluated_transform(clip, sample_frame, render_size); + let mut affine = compose( + plan.preferred_transform, + affine_transform(&transform, plan.nat_size, render_size), + ); + for ancestor in plan.compound_ancestors.iter().rev() { + let transform = evaluated_transform(&ancestor.clip, sample_frame, render_size); + affine = compose( + affine, + affine_transform(&transform, ancestor.canvas_size, render_size), + ); + } + Some(LayerDraw { + source: &plan.source, + source_frame: source_frame_index(plan, sample_frame), + affine, + nat_size: plan.nat_size, + crop_uv: crop_to_uv(clip.crop_at(sample_frame)), + opacity, + needs_premultiply: plan.needs_premultiply, + clip_id: &plan.clip_id, + color_grade: plan.color_grade.as_ref(), + lut: plan.lut.as_ref(), chroma_key: plan.chroma_key.as_ref(), masks: &plan.masks, effects: &plan.effects, @@ -322,21 +661,51 @@ impl RenderPlan { /// `timeline` must be the same one the plan was built from (they share clip /// indices). Video clips composite first; text clips composite last (on /// top), matching upstream's text-over-video layering (SPEC §4.2). - pub fn frame<'a>(&'a self, timeline: &'a Timeline, f: i32) -> FramePlan<'a> { + pub fn frame<'a>(&'a self, _timeline: &'a Timeline, f: i32) -> FramePlan<'a> { let mut draws: Vec> = Vec::new(); - for plan in &self.clip_plans { - let Some(clip) = clip_for(timeline, plan) else { - continue; - }; + for (index, plan) in self.clip_plans.iter().enumerate() { + let clip = &plan.clip; + let transition = clip.transition_out.as_ref().and_then(|transition| { + let incoming_plan = self.clip_plans.get(index + 1)?; + if transition.kind != TransitionKind::CrossDissolve + || (!transition.from_clip_id.is_empty() + && transition.from_clip_id != plan.clip_id) + || incoming_plan.track_index != plan.track_index + || incoming_plan.clip_id != transition.to_clip_id + || incoming_plan.start_frame != plan.end_frame + { + return None; + } + let incoming = &incoming_plan.clip; + let duration = transition + .duration_frames + .max(1) + .min(clip.duration_frames.max(1)) + .min(incoming.duration_frames.max(1)); + let start = plan.end_frame - duration; + if f < start || f >= plan.end_frame { + return None; + } + let progress = (f - start) as f64 / duration as f64; + Some((incoming_plan, incoming, progress)) + }); + if let Some(d) = eval_layer(plan, clip, f, self.render_size) { - draws.push(d); + if d.opacity > 0.0 { + draws.push(d); + } + } + if let Some((incoming_plan, incoming, progress)) = transition { + if let Some(d) = + eval_transition_incoming(incoming_plan, incoming, progress, self.render_size) + { + draws.push(d); + } } } for plan in &self.text_plans { - let Some(clip) = clip_for(timeline, plan) else { - continue; - }; + let clip = &plan.clip; if let Some(d) = eval_layer(plan, clip, f, self.render_size) { draws.push(d); } @@ -348,19 +717,3 @@ impl RenderPlan { } } } - -/// Resolve the `&Clip` for a plan via its stored indices, falling back to an id -/// search if the indices no longer line up (defensive; the indexed path is the -/// fast one per SPEC §2.4). -fn clip_for<'a>(timeline: &'a Timeline, plan: &ClipPlan) -> Option<&'a Clip> { - if let Some(track) = timeline.tracks.get(plan.track_index) { - if let Some(clip) = track.clips.get(plan.clip_index) { - if clip.id == plan.clip_id { - return Some(clip); - } - } - // Indices drifted: fall back to id lookup within the track. - return track.clips.iter().find(|c| c.id == plan.clip_id); - } - None -} diff --git a/crates/opentake-render/src/plan/mod.rs b/crates/opentake-render/src/plan/mod.rs index e5a2cb68..f0c7aedf 100644 --- a/crates/opentake-render/src/plan/mod.rs +++ b/crates/opentake-render/src/plan/mod.rs @@ -10,5 +10,8 @@ pub mod types; mod tests; pub use affine::{affine_transform, compose, crop_to_uv}; -pub use build::{build_render_plan, source_frame_index}; -pub use types::{ClipPlan, FramePlan, LayerDraw, RenderPlan, RenderSize, TextureSource}; +pub use build::{build_render_plan, source_frame_index, try_build_render_plan}; +pub use types::{ + AudioClipPlan, ClipPlan, CompoundAncestor, FramePlan, LayerDraw, RenderPlan, RenderSize, + TextureSource, +}; diff --git a/crates/opentake-render/src/plan/tests.rs b/crates/opentake-render/src/plan/tests.rs index d37984a1..d17d079b 100644 --- a/crates/opentake-render/src/plan/tests.rs +++ b/crates/opentake-render/src/plan/tests.rs @@ -3,7 +3,7 @@ use opentake_domain::{ AnimPair, Clip, ClipType, Crop, Interpolation, Keyframe, KeyframeTrack, Point, Timeline, Track, - Transform, + Transform, Transition, TransitionKind, }; use super::affine::affine_transform; @@ -76,6 +76,36 @@ const RS: RenderSize = RenderSize { height: 1080, }; +#[test] +fn cross_dissolve_emits_two_weighted_layers_before_the_cut() { + let mut a = video_clip("a", 0, 30); + a.transition_out = Some(Transition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: TransitionKind::CrossDissolve, + duration_frames: 10, + }); + let b = video_clip("b", 30, 30); + let mut tl = Timeline::new(); + let mut track = Track::new("v", ClipType::Video); + track.clips = vec![a, b]; + tl.tracks.push(track); + let plan = build_render_plan(&tl, RS, &TestMetrics::default()); + + let midpoint = plan.frame(&tl, 25); + assert_eq!(midpoint.draws.len(), 2); + assert_eq!(midpoint.draws[0].clip_id, "a"); + assert_eq!(midpoint.draws[1].clip_id, "b"); + approx(midpoint.draws[0].opacity, 1.0); + approx(midpoint.draws[1].opacity, 0.5); + assert_eq!(midpoint.draws[1].source_frame, 0); + + let after_cut = plan.frame(&tl, 30); + assert_eq!(after_cut.draws.len(), 1); + assert_eq!(after_cut.draws[0].clip_id, "b"); + approx(after_cut.draws[0].opacity, 1.0); +} + // --- Single clip, no transform: full-canvas identity-ish affine --- #[test] diff --git a/crates/opentake-render/src/plan/types.rs b/crates/opentake-render/src/plan/types.rs index a4635e87..3d78a36a 100644 --- a/crates/opentake-render/src/plan/types.rs +++ b/crates/opentake-render/src/plan/types.rs @@ -13,7 +13,7 @@ //! The black background is NOT a clip here — it is the compositor clear color //! `(0,0,0,1)` (SPEC §3.5). -use opentake_domain::{ChromaKey, ClipType, ColorGrade, Effect, Mask}; +use opentake_domain::{ChromaKey, Clip, ClipType, ColorGrade, Effect, LutReference, Mask}; /// Canvas pixel size (already even-ized; see [`crate::size`]). #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -51,21 +51,57 @@ pub enum TextureSource { Image { media_ref: String }, /// Lottie: a texture per "Lottie internal frame" (content-hash cached). Lottie { media_ref: String }, - /// Text: rasterized for this clip at the canvas size (content-hash cached, - /// key = style + content + canvas). + /// Text clip identity: rasterized at the canvas size. The resolver reads + /// content/style from the same timeline snapshot and includes style, + /// content, and canvas in its cache key, so an edited clip cannot reuse + /// stale pixels based on id alone. Text { clip_id: String }, } +/// One compound clip surrounding a flattened leaf, plus the child canvas size +/// that clip transforms as its source. +#[derive(Clone, PartialEq, Debug)] +pub struct CompoundAncestor { + pub clip: Clip, + pub canvas_size: (f64, f64), +} + +/// One audio-bearing leaf projected into root timeline coordinates. `clip` +/// owns the visible source window used for decode/placement, while `gain_clip` +/// and `compound_ancestors` retain unclipped timing for frame-accurate volume +/// keyframe and fade sampling. +#[derive(Clone, PartialEq, Debug)] +pub struct AudioClipPlan { + pub clip: Clip, + pub gain_clip: Clip, + pub compound_ancestors: Vec, +} + +impl AudioClipPlan { + pub fn volume_at(&self, frame: i32) -> f64 { + self.compound_ancestors + .iter() + .fold(self.gain_clip.volume_at(frame), |gain, ancestor| { + gain * ancestor.volume_at(frame) + }) + } +} + /// Static (frame-independent) render description for one clip. #[derive(Clone, PartialEq, Debug)] pub struct ClipPlan { + /// Immutable clip snapshot owned by this plan. This lets recursively + /// flattened nested clips use the exact same plan in preview and export + /// without indexing back into a different timeline tree. + pub clip: Clip, pub clip_id: String, - /// Index of the timeline track this clip belongs to (blend order; SPEC §1.5). + /// Flattened track index retained for diagnostics and compatibility. pub track_index: usize, - /// Index of the clip inside `timeline.tracks[track_index].clips`, so - /// [`RenderPlan::frame`](crate::plan::RenderPlan::frame) can fetch the `&Clip` - /// without a string search (SPEC §2.4 note). + /// Index of the source clip inside its immediate timeline track. pub clip_index: usize, + /// Root-to-leaf track path used for deterministic nested blend order. + pub blend_path: Vec, + pub compound_ancestors: Vec, pub source: TextureSource, pub start_frame: i32, /// Half-open end (`start_frame + duration_frames`). @@ -96,6 +132,8 @@ pub struct ClipPlan { // shader; the pure pixel math lives in `opentake_domain::grade`. /// Linear-light color grade, or `None` when the clip has no grade. pub color_grade: Option, + /// Project-managed 3D LUT applied after the primary grade. + pub lut: Option, /// Chroma key, or `None` when the clip has no keying. pub chroma_key: Option, /// Vector masks (intersected coverage). Empty = no masking. @@ -121,6 +159,10 @@ pub struct RenderPlan { /// ExportService L237-248; SPEC §4.2). Ordered by appearance, NOT deduped /// per track (upstream collects text clips without the per-track skip rule). pub text_plans: Vec, + /// Audio-bearing leaf clips flattened through the same nested timing map. + /// Export consumes this list so compound preview/video and audio share + /// identical in/out boundaries. + pub audio_clips: Vec, } /// One draw after evaluating a single frame (instantaneous). @@ -153,6 +195,8 @@ pub struct LayerDraw<'a> { /// Color grade applied in-shader (linear-light chain), borrowed from the /// [`ClipPlan`]. `None` = no grade. pub color_grade: Option<&'a ColorGrade>, + /// Project-managed 3D LUT applied after the primary grade. + pub lut: Option<&'a LutReference>, /// Chroma key applied in-shader, borrowed from the [`ClipPlan`]. `None` = none. pub chroma_key: Option<&'a ChromaKey>, /// Masks applied in-shader (intersected coverage), borrowed from the diff --git a/crates/opentake-render/tests/composite_acceptance.rs b/crates/opentake-render/tests/composite_acceptance.rs new file mode 100644 index 00000000..a9361659 --- /dev/null +++ b/crates/opentake-render/tests/composite_acceptance.rs @@ -0,0 +1,163 @@ +const EVIDENCE: &str = include_str!( + "../../../docs/audit/2026-07-14/runtime-artifacts/automated/hdr-proxy-account-real-device-2026-08-01.md" +); +const GPU_CHILDREN: &str = include_str!("gpu_effects.rs"); +const COMPOSITOR: &str = include_str!("../src/gpu/compositor.rs"); +const RENDER_OVERVIEW: &str = include_str!("../../../docs/modules/opentake-render/OVERVIEW.md"); +const MEDIA_PRINCIPLES: &str = include_str!("../../../docs/specs/media/0-principles.md"); +const MEDIA_DOMAIN_CONTRACT: &str = include_str!("../../../docs/specs/media/9-domain-contract.md"); +const MEDIA_FFMPEG_CHILDREN: &str = + include_str!("../../opentake-media/tests/ffmpeg_integration.rs"); +const MEDIA_ENGINE_SOURCE: &str = include_str!("../../opentake-media/src/lib.rs"); +const MEDIA_INDEX_SOURCE: &str = include_str!("../../opentake-media/src/index_coordinator.rs"); + +use opentake_domain::{ + Clip, ClipType, Effect, Mask, MaskShape, Point2, Timeline, Track, MAX_MASKS_PER_CLIP, + MAX_POLYGON_MASK_POINTS, +}; +use opentake_ops::{apply, EditCommand, EditError, EditorState, SeqIdGen}; + +#[test] +fn hdr_proxy_account_children_close_one_composite_acceptance() { + let evidence = EVIDENCE.replace("\r\n", "\n"); + for child in [ + "HDR child result: **PASS**", + "Proxy child result: **PASS**", + "Account child result: **PASS**", + ] { + assert!(evidence.contains(child), "missing child evidence: {child}"); + } + assert!(evidence.contains("`HDR child PASS + proxy child PASS + account child PASS`")); + assert!(evidence.contains("closes one composite\nacceptance")); + assert!(evidence.contains("codesign --verify --deep --strict")); + assert!(evidence.contains("This is not\nan HDR-passthrough claim.")); + assert!(evidence.contains("Export therefore used the original source, not the enabled proxy.")); + assert!(evidence.contains("Local editing remains the default")); +} + +fn state_with_visual_clip() -> EditorState { + let mut timeline = Timeline::new(); + let mut track = Track::new("video", ClipType::Video); + track.clips.push(Clip::new("clip", "asset", 0, 30)); + timeline.tracks.push(track); + EditorState::from_timeline(timeline) +} + +#[test] +fn mask_and_effect_records_have_separate_child_owners() { + // The mixed audit record is closed by two executable pixel owners, rather + // than by duplicating either renderer in this aggregation test. + for owner in [ + "linear_circle_and_polygon_masks_match_cpu_reference_in_preview_and_export", + "advertised_effect_registry_has_preview_export_golden_fixtures", + ] { + assert!( + GPU_CHILDREN.contains(owner), + "missing executable child owner: {owner}" + ); + } + for boundary in [ + "fn pack_masks", + "MAX_POLYGON_MASK_POINTS", + "fn pack_effects", + "validate_effect_chain(draw.effects)", + ] { + assert!( + COMPOSITOR.contains(boundary), + "missing compositor boundary: {boundary}" + ); + } + + // Authored overflow and unknown registry entries fail before mutation. + let ids = SeqIdGen::default(); + let mut state = state_with_visual_clip(); + let original = state.timeline.clone(); + let too_many = vec![Mask::default(); MAX_MASKS_PER_CLIP + 1]; + let error = apply( + &mut state, + EditCommand::SetMasks { + clip_ids: vec!["clip".into()], + masks: too_many, + }, + &ids, + ) + .unwrap_err(); + assert!(matches!(error, EditError::Invalid(message) if message.contains("at most"))); + assert_eq!(state.timeline, original); + assert_eq!(state.undo_depth(), 0); + + let polygon = Mask { + shape: MaskShape::Poly { + points: vec![Point2::new(0.5, 0.5); MAX_POLYGON_MASK_POINTS + 1], + }, + ..Mask::default() + }; + let error = apply( + &mut state, + EditCommand::SetMasks { + clip_ids: vec!["clip".into()], + masks: vec![polygon], + }, + &ids, + ) + .unwrap_err(); + assert!(matches!(error, EditError::Invalid(message) if message.contains("polygon"))); + assert_eq!(state.timeline, original); + + let error = apply( + &mut state, + EditCommand::SetEffects { + clip_ids: vec!["clip".into()], + effects: vec![Effect::new("unadvertised")], + }, + &ids, + ) + .unwrap_err(); + assert!(matches!(error, EditError::Invalid(message) if message.contains("unknown effect"))); + assert_eq!(state.timeline, original); + + assert!(RENDER_OVERVIEW.contains("多边形(钢笔)蒙版与通用 Effect 链已落地")); + assert!(!RENDER_OVERVIEW.contains("编码为全画布 no-op")); +} + +#[test] +fn media_principles_headings_reference_exact_child_capabilities() { + assert!(MEDIA_PRINCIPLES.starts_with("# 设计原则与移植铁律(本 crate 必须遵守)")); + assert!(MEDIA_DOMAIN_CONTRACT.starts_with("# 跨平台与合规要点")); + for document in [MEDIA_PRINCIPLES, MEDIA_DOMAIN_CONTRACT] { + assert!(document.contains("可执行子能力集合")); + for child in [ + "probe_reports_dimensions_fps_and_audio", + "decode_frame_returns_rgba_of_expected_size", + "extract_pcm_yields_16k_mono", + "waveform_has_expected_bucket_count", + "encode_roundtrip_produces_playable_video", + "export_pause_ref_counts", + ] { + assert!(document.contains(child), "missing child reference: {child}"); + } + } + + for child in [ + "fn probe_reports_dimensions_fps_and_audio", + "fn decode_frame_returns_rgba_of_expected_size", + "fn extract_pcm_yields_16k_mono", + "fn waveform_has_expected_bucket_count", + "fn encode_roundtrip_produces_playable_video", + ] { + assert!( + MEDIA_FFMPEG_CHILDREN.contains(child), + "missing executable media child: {child}" + ); + } + assert!(MEDIA_ENGINE_SOURCE.contains("seconds (f64) at every IO")); + assert!(MEDIA_ENGINE_SOURCE.contains("pub struct MediaEngine")); + assert!(MEDIA_INDEX_SOURCE.contains("fn export_pause_ref_counts")); + + // The compliance collection must describe the actual subprocess-sidecar + // architecture and preserve the known release blocker instead of claiming + // dynamic linking or a completed public distribution review. + assert!(MEDIA_DOMAIN_CONTRACT.contains("FFmpeg 子进程 sidecar")); + assert!(MEDIA_DOMAIN_CONTRACT.contains("Beta 发布阻塞")); + assert!(!MEDIA_DOMAIN_CONTRACT.contains("动态链接 + NOTICE")); +} diff --git a/crates/opentake-render/tests/compound_render.rs b/crates/opentake-render/tests/compound_render.rs new file mode 100644 index 00000000..440ea7a0 --- /dev/null +++ b/crates/opentake-render/tests/compound_render.rs @@ -0,0 +1,128 @@ +use opentake_domain::{Clip, ClipType, NestedSequence, Timeline, Track}; +use opentake_render::{try_build_render_plan, RenderSize, SourceMetrics}; + +struct Metrics; + +impl SourceMetrics for Metrics { + fn natural_size(&self, _media_ref: &str) -> Option<(u32, u32)> { + Some((64, 64)) + } +} + +fn video_track(id: &str, clips: Vec) -> Track { + let mut track = Track::new(id, ClipType::Video); + track.clips = clips; + track +} + +#[test] +fn compound_clip_preview_export_frames_match() { + let mut leaf = Clip::new("leaf", "asset-a", 3, 10); + leaf.trim_start_frame = 2; + let mut sequence_b = Timeline::new(); + sequence_b.tracks = vec![video_track("b-track", vec![leaf])]; + + let mut nested_b = Clip::new_nested("nested-b", "sequence-b", 5, 8); + nested_b.trim_start_frame = 3; + let mut sequence_a = Timeline::new(); + sequence_a.tracks = vec![video_track("a-track", vec![nested_b])]; + + let mut root = Timeline::new(); + root.nested_sequences = vec![ + NestedSequence::new("sequence-a", "A", sequence_a), + NestedSequence::new("sequence-b", "B", sequence_b), + ]; + let mut compound = Clip::new_nested("compound", "sequence-a", 20, 6); + compound.trim_start_frame = 5; + compound.opacity = 0.5; + compound.transform.width = 0.5; + compound.transform.height = 0.5; + root.tracks = vec![video_track("root-track", vec![compound])]; + + let preview = try_build_render_plan(&root, RenderSize::new(64, 64), &Metrics).unwrap(); + let export = try_build_render_plan(&root, RenderSize::new(64, 64), &Metrics).unwrap(); + for frame in 19..=27 { + assert_eq!(preview.frame(&root, frame), export.frame(&root, frame)); + } + assert!(preview.frame(&root, 19).draws.is_empty()); + assert_eq!(preview.frame(&root, 20).draws[0].source_frame, 2); + assert_eq!(preview.frame(&root, 20).draws[0].opacity, 0.5); + assert_eq!( + preview.frame(&root, 20).draws[0].affine, + [0.5, 0.0, 0.0, 0.5, 16.0, 16.0] + ); + assert_eq!(preview.frame(&root, 25).draws[0].source_frame, 7); + assert!(preview.frame(&root, 26).draws.is_empty()); +} + +#[test] +fn compound_trim_preserves_inner_fade_sampling_offset() { + let mut leaf = Clip::new("leaf", "asset-a", 0, 20); + leaf.fade_in_frames = 20; + let mut child = Timeline::new(); + child.tracks = vec![video_track("child-track", vec![leaf])]; + + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence", "Sequence", child)); + let mut compound = Clip::new_nested("compound", "sequence", 100, 5); + compound.trim_start_frame = 10; + root.tracks = vec![video_track("root-track", vec![compound])]; + + let plan = try_build_render_plan(&root, RenderSize::new(64, 64), &Metrics).unwrap(); + let first = &plan.frame(&root, 100).draws[0]; + assert_eq!(first.source_frame, 10); + assert_eq!(first.opacity, 0.5); +} + +#[test] +fn unsupported_compound_retime_fails_before_preview_or_export() { + let mut child = Timeline::new(); + child.tracks = vec![video_track( + "child-track", + vec![Clip::new("leaf", "asset-a", 0, 10)], + )]; + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence", "Sequence", child)); + let mut compound = Clip::new_nested("compound", "sequence", 0, 10); + compound.speed = 2.0; + root.tracks = vec![video_track("root-track", vec![compound])]; + + assert_eq!( + try_build_render_plan(&root, RenderSize::new(64, 64), &Metrics).unwrap_err(), + "compound clip compound must use forward 1x playback" + ); +} + +#[test] +fn compound_audio_uses_the_same_trimmed_root_span() { + let mut audio = Clip::new("audio", "asset-a", 4, 10); + audio.media_type = ClipType::Audio; + audio.source_clip_type = ClipType::Audio; + audio.trim_start_frame = 2; + audio.fade_in_frames = 10; + let mut child = Timeline::new(); + child.tracks = vec![video_track("unused", vec![])]; + let mut audio_track = Track::new("audio-track", ClipType::Audio); + audio_track.clips.push(audio); + child.tracks.push(audio_track); + + let mut root = Timeline::new(); + root.nested_sequences + .push(NestedSequence::new("sequence", "Sequence", child)); + let mut compound = Clip::new_nested("compound", "sequence", 20, 5); + compound.trim_start_frame = 6; + compound.volume = 0.5; + compound.fade_in_frames = 5; + root.tracks = vec![video_track("root-track", vec![compound])]; + + let plan = try_build_render_plan(&root, RenderSize::new(64, 64), &Metrics).unwrap(); + assert_eq!(plan.audio_clips.len(), 1); + let flattened = &plan.audio_clips[0]; + assert_eq!(flattened.clip.start_frame, 20); + assert_eq!(flattened.clip.duration_frames, 5); + assert_eq!(flattened.clip.trim_start_frame, 4); + assert_eq!(flattened.volume_at(20), 0.0); + assert!((flattened.volume_at(22) - 0.08).abs() < 1e-12); +} diff --git a/crates/opentake-render/tests/gpu_effects.rs b/crates/opentake-render/tests/gpu_effects.rs index 5ec32236..91d72380 100644 --- a/crates/opentake-render/tests/gpu_effects.rs +++ b/crates/opentake-render/tests/gpu_effects.rs @@ -10,15 +10,16 @@ use std::rc::Rc; use opentake_domain::{ - ChromaKey, Clip, ClipType, ColorGrade, Mask, MaskShape, Point, Point2, Rgb, Timeline, Track, - Transform, + effect_registry, ChromaKey, Clip, ClipType, ColorGrade, Effect, EffectValidationError, + HslSecondary, LiftGammaGain, Mask, MaskShape, MaskTransform, Point, Point2, Rgb, Timeline, + Track, Transform, }; use opentake_render::gpu::texture::upload_rgba; use opentake_render::source::DecodedFrame; use opentake_render::wgpu; use opentake_render::{ - build_render_plan, Compositor, GpuTexture, RenderDevice, RenderSize, SourceMetrics, - TextureResolver, TextureSource, + build_render_plan, Compositor, GpuTexture, RenderDevice, RenderError, RenderSize, + SourceMetrics, TextureResolver, TextureSource, }; const RS: RenderSize = RenderSize { @@ -43,6 +44,39 @@ struct SolidResolver<'d> { cached: Option>, } +/// Four equal-width chart bars: red, orange, green and blue. This lets the HSL +/// qualifier test an in-range hue, its feather boundary and two isolated hues +/// in one real compositor submission. +struct ColorChartResolver<'d> { + device: &'d wgpu::Device, + queue: &'d wgpu::Queue, + cached: Option>, +} + +impl TextureResolver for ColorChartResolver<'_> { + fn resolve(&mut self, _source: &TextureSource, _frame: i64) -> Option> { + if self.cached.is_none() { + let colors = [ + [255, 0, 0, 255], + [255, 200, 0, 255], + [0, 255, 0, 255], + [0, 0, 255, 255], + ]; + let mut buf = vec![0u8; 16 * 16 * 4]; + for y in 0..16 { + for x in 0..16 { + let i = (y * 16 + x) * 4; + buf[i..i + 4].copy_from_slice(&colors[x / 4]); + } + } + let frame = DecodedFrame::new(16, 16, buf, true); + let tex = upload_rgba(self.device, self.queue, &frame, false, Some("hsl-chart")); + self.cached = Some(Rc::new(tex)); + } + self.cached.clone() + } +} + impl TextureResolver for SolidResolver<'_> { fn resolve(&mut self, _source: &TextureSource, _frame: i64) -> Option> { if self.cached.is_none() { @@ -118,6 +152,115 @@ fn render(dev: &RenderDevice, tl: &Timeline, rgba: [u8; 4]) -> DecodedFrame { .expect("render") } +fn render_color_chart(dev: &RenderDevice, tl: &Timeline) -> DecodedFrame { + let plan = build_render_plan(tl, RS, &Metrics); + let fp = plan.frame(tl, 0); + let compositor = Compositor::new(&dev.device); + let mut resolver = ColorChartResolver { + device: &dev.device, + queue: &dev.queue, + cached: None, + }; + compositor + .render_to_rgba(&dev.device, &dev.queue, RS, &fp, &mut resolver) + .expect("render color chart") +} + +#[test] +fn advertised_effect_registry_has_preview_export_golden_fixtures() { + let Some(dev) = device_or_skip("advertised_effect_registry_has_preview_export_golden_fixtures") + else { + return; + }; + + let registry = effect_registry(); + assert_eq!( + registry + .iter() + .map(|effect| effect.name) + .collect::>(), + ["grayscale", "sepia", "invert"] + ); + + // Golden center pixels for a fixed opaque source. Each advertised effect is + // exercised at its persisted default and a non-default amount. Preview and + // export deliberately render through fresh resolver state and must agree + // byte-for-byte. + let fixtures = [ + ("grayscale", None, [81, 81, 81, 255]), + ("grayscale", Some(0.4), [164, 50, 140, 255]), + ("sepia", None, [144, 128, 99, 255]), + ("sepia", Some(0.4), [189, 69, 148, 255]), + ("invert", None, [35, 225, 75, 255]), + ("invert", Some(0.4), [146, 108, 138, 255]), + ]; + for (name, amount, golden) in fixtures { + let mut timeline = full_canvas_timeline(); + let effect = amount.map_or_else( + || Effect::new(name), + |value| Effect::new(name).with_param("amount", value), + ); + effect.validate().expect("advertised effect validates"); + timeline.tracks[0].clips[0].effects = vec![effect]; + + let preview = render(&dev, &timeline, [220, 30, 180, 255]); + let export = render(&dev, &timeline, [220, 30, 180, 255]); + assert_eq!(preview.rgba, export.rgba, "preview/export drift for {name}"); + let actual = center_pixel(&preview); + for channel in 0..4 { + assert!( + (actual[channel] as i32 - golden[channel]).abs() <= 3, + "{name} amount={amount:?}: expected {golden:?}, got {actual:?}" + ); + } + } + + // The sequence itself is authored state: changing order must change pixels. + let mut first = full_canvas_timeline(); + first.tracks[0].clips[0].effects = vec![Effect::new("sepia"), Effect::new("invert")]; + let mut second = full_canvas_timeline(); + second.tracks[0].clips[0].effects = vec![Effect::new("invert"), Effect::new("sepia")]; + assert_ne!( + center_pixel(&render(&dev, &first, [220, 30, 180, 255])), + center_pixel(&render(&dev, &second, [220, 30, 180, 255])), + "effect order must be rendered, not stored as inert metadata" + ); + + // Disabled registered effects remain persisted but are skipped by the + // render chain in both preview and export. + let source = [220, 30, 180, 255]; + let mut disabled = full_canvas_timeline(); + disabled.tracks[0].clips[0].effects = vec![Effect { + enabled: false, + ..Effect::new("invert") + }]; + let baseline = render(&dev, &full_canvas_timeline(), source); + let disabled_preview = render(&dev, &disabled, source); + let disabled_export = render(&dev, &disabled, source); + assert_eq!(disabled_preview.rgba, baseline.rgba); + assert_eq!(disabled_export.rgba, baseline.rgba); + + let mut invalid = full_canvas_timeline(); + invalid.tracks[0].clips[0].effects = vec![Effect::new("unadvertised")]; + let plan = build_render_plan(&invalid, RS, &Metrics); + let frame_plan = plan.frame(&invalid, 0); + let compositor = Compositor::new(&dev.device); + let mut resolver = SolidResolver { + device: &dev.device, + queue: &dev.queue, + rgba: [220, 30, 180, 255], + cached: None, + }; + let error = compositor + .render_to_rgba(&dev.device, &dev.queue, RS, &frame_plan, &mut resolver) + .expect_err("unknown effects must fail instead of rendering unchanged"); + assert!(matches!( + error, + RenderError::InvalidEffect(EffectValidationError::UnknownEffect { ref name }) + if name == "unadvertised" + )); +} + #[test] fn color_grade_zero_saturation_greyscales() { let Some(dev) = device_or_skip("color_grade_zero_saturation_greyscales") else { @@ -188,6 +331,150 @@ fn color_grade_identity_is_passthrough() { } } +#[test] +fn lift_gamma_gain_matches_cpu_reference() { + let grade = ColorGrade { + lift_gamma_gain: LiftGammaGain { + lift: Rgb::new(0.08, -0.03, 0.12), + gamma: Rgb::new(1.8, 0.75, 1.25), + gain: Rgb::new(0.82, 1.15, 0.93), + }, + ..Default::default() + }; + let source = [96_u8, 128, 192, 255]; + let linear = source[..3] + .iter() + .map(|channel| opentake_render::gpu::srgb_to_linear(f64::from(*channel) / 255.0)) + .collect::>(); + + let source_formula = |x: f64, lift: f64, gamma: f64, gain: f64| { + gain * (x + lift * (1.0 - x)).max(0.0).powf(1.0 / gamma) + }; + let expected_linear = [ + source_formula(linear[0], 0.08, 1.8, 0.82), + source_formula(linear[1], -0.03, 0.75, 1.15), + source_formula(linear[2], 0.12, 1.25, 0.93), + ]; + let cpu = grade.apply_linear(linear[0], linear[1], linear[2]); + for (actual, expected) in [cpu.0, cpu.1, cpu.2].into_iter().zip(expected_linear) { + assert!( + (actual - expected).abs() < 1e-9, + "CPU color-wheel reference drift: expected {expected}, got {actual}" + ); + } + + let Some(dev) = device_or_skip("lift_gamma_gain_matches_cpu_reference") else { + return; + }; + let mut timeline = full_canvas_timeline(); + timeline.tracks[0].clips[0].color_grade = Some(grade); + let preview = render(&dev, &timeline, source); + let export = render(&dev, &timeline, source); + assert_eq!(preview.rgba, export.rgba, "preview/export LGG drift"); + + let expected = expected_linear.map(|channel| { + (opentake_render::gpu::linear_to_srgb(channel.clamp(0.0, 1.0)) * 255.0).round() as u8 + }); + let actual = center_pixel(&preview); + for channel in 0..3 { + assert!( + (i16::from(actual[channel]) - i16::from(expected[channel])).abs() <= 2, + "GPU LGG channel {channel}: expected {expected:?}, got {actual:?}" + ); + } + assert_eq!(actual[3], 255); + + // A malformed persisted grade is rejected before source resolution or any + // GPU submission, so preview/export cannot silently diverge or render an + // unchanged frame. + let mut invalid = full_canvas_timeline(); + invalid.tracks[0].clips[0].color_grade = Some(ColorGrade { + lift_gamma_gain: LiftGammaGain { + gamma: Rgb::new(0.0, 1.0, 1.0), + ..Default::default() + }, + ..Default::default() + }); + let invalid_plan = build_render_plan(&invalid, RS, &Metrics); + let invalid_frame = invalid_plan.frame(&invalid, 0); + let compositor = Compositor::new(&dev.device); + let mut resolver = SolidResolver { + device: &dev.device, + queue: &dev.queue, + rgba: source, + cached: None, + }; + let error = compositor + .render_to_rgba(&dev.device, &dev.queue, RS, &invalid_frame, &mut resolver) + .expect_err("zero gamma must be rejected before source resolution"); + assert!(matches!( + error, + RenderError::InvalidColorGrade(ref invalid) + if invalid.to_string() + == "liftGammaGain.gamma.r must be finite and within (0, 4]" + )); + assert!( + resolver.cached.is_none(), + "invalid grade resolved source data" + ); +} + +#[test] +fn hsl_secondary_hue_boundary_feather_and_isolation() { + let grade = ColorGrade { + hsl_secondary: Some(HslSecondary { + hue_center: 0.0, + hue_width: 0.24, + feather: 0.08, + hue_shift: 0.20, + saturation: -0.25, + lightness: 0.10, + }), + ..Default::default() + }; + grade.validate().expect("bounded HSL secondary validates"); + + // Persisted authored state must survive a save/reopen boundary exactly. + let json = serde_json::to_string(&grade).expect("serialize HSL secondary"); + let reopened: ColorGrade = serde_json::from_str(&json).expect("reopen HSL secondary"); + assert_eq!(reopened, grade); + + let Some(dev) = device_or_skip("hsl_secondary_hue_boundary_feather_and_isolation") else { + return; + }; + let plain = render_color_chart(&dev, &full_canvas_timeline()); + let mut timeline = full_canvas_timeline(); + timeline.tracks[0].clips[0].color_grade = Some(grade); + let preview = render_color_chart(&dev, &timeline); + let export = render_color_chart(&dev, &timeline); + assert_eq!(preview.rgba, export.rgba, "preview/export HSL drift"); + + let delta = |x: u32| { + let before = pixel_at(&plain, x, 8); + let after = pixel_at(&preview, x, 8); + before[..3] + .iter() + .zip(&after[..3]) + .map(|(a, b)| (i16::from(*a) - i16::from(*b)).unsigned_abs()) + .max() + .unwrap() + }; + let selected_red = delta(2); + let feathered_orange = delta(6); + let isolated_green = delta(10); + let isolated_blue = delta(14); + assert!( + selected_red > 20, + "selected red did not change: {selected_red}" + ); + assert!( + feathered_orange > 2 && feathered_orange < selected_red, + "orange must receive only the feathered adjustment: red={selected_red}, orange={feathered_orange}" + ); + assert!(isolated_green <= 2, "green leaked by {isolated_green}"); + assert!(isolated_blue <= 2, "blue leaked by {isolated_blue}"); +} + #[test] fn chroma_key_removes_green() { let Some(dev) = device_or_skip("chroma_key_removes_green") else { @@ -238,6 +525,7 @@ fn circle_mask_clips_to_center() { }, feather: 0.0, invert: false, + ..Mask::default() }]; // White source, masked to a small centered circle over black. let frame = render(&dev, &tl, [255, 255, 255, 255]); @@ -266,6 +554,7 @@ fn inverted_mask_clips_out_center() { }, feather: 0.0, invert: true, + ..Mask::default() }]; let frame = render(&dev, &tl, [255, 255, 255, 255]); // Inverted: center is now masked OUT -> black. @@ -281,3 +570,116 @@ fn inverted_mask_clips_out_center() { "corner kept by inverted mask, got {corner:?}" ); } + +#[test] +fn linear_circle_and_polygon_masks_match_cpu_reference_in_preview_and_export() { + let Some(dev) = + device_or_skip("linear_circle_and_polygon_masks_match_cpu_reference_in_preview_and_export") + else { + return; + }; + let shapes = [ + MaskShape::Linear { + point: Point2::new(0.45, 0.55), + normal: Point2::new(0.8, -0.3), + }, + MaskShape::Circle { + center: Point2::new(0.55, 0.45), + radius: Point2::new(0.31, 0.22), + }, + MaskShape::Poly { + points: vec![ + Point2::new(0.2, 0.2), + Point2::new(0.82, 0.28), + Point2::new(0.68, 0.82), + Point2::new(0.28, 0.72), + ], + }, + ]; + + for shape in shapes { + for feather in [0.0, 0.18] { + let mask = Mask { + shape: shape.clone(), + feather, + invert: feather > 0.0, + transform: if matches!(&shape, MaskShape::Poly { .. }) { + MaskTransform { + offset: Point2::new(0.07, -0.04), + scale: Point2::new(0.82, 1.13), + rotation_degrees: 17.0, + } + } else { + MaskTransform::default() + }, + }; + let mut timeline = full_canvas_timeline(); + timeline.tracks[0].clips[0].masks = vec![mask.clone()]; + + // Paused preview and export both consume the same FramePlan and + // compositor boundary. Render twice with fresh resolvers to guard + // against path-local state and compare both against the CPU mirror. + let preview = render(&dev, &timeline, [255, 255, 255, 255]); + let export = render(&dev, &timeline, [255, 255, 255, 255]); + assert_eq!(preview.rgba, export.rgba); + + for y in 0..RS.height { + for x in 0..RS.width { + let expected = (mask.coverage( + (x as f64 + 0.5) / RS.width as f64, + (y as f64 + 0.5) / RS.height as f64, + ) * 255.0) + .round() as i32; + let actual = pixel_at(&preview, x, y)[0] as i32; + assert!( + (actual - expected).abs() <= 3, + "shape={shape:?} feather={feather} pixel=({x},{y}) expected={expected} actual={actual}" + ); + } + } + } + } + + // Multiple masks intersect in authored order. Compare the real GPU result + // against the product of both CPU coverage functions at every pixel. + let masks = vec![ + Mask { + shape: MaskShape::Circle { + center: Point2::new(0.38, 0.5), + radius: Point2::new(0.34, 0.3), + }, + feather: 0.08, + ..Mask::default() + }, + Mask { + shape: MaskShape::Circle { + center: Point2::new(0.62, 0.5), + radius: Point2::new(0.34, 0.3), + }, + feather: 0.08, + ..Mask::default() + }, + ]; + let mut timeline = full_canvas_timeline(); + timeline.tracks[0].clips[0].masks = masks.clone(); + let preview = render(&dev, &timeline, [255, 255, 255, 255]); + let export = render(&dev, &timeline, [255, 255, 255, 255]); + assert_eq!(preview.rgba, export.rgba); + for y in 0..RS.height { + for x in 0..RS.width { + let px = (x as f64 + 0.5) / RS.width as f64; + let py = (y as f64 + 0.5) / RS.height as f64; + let expected = (masks + .iter() + .map(|mask| mask.coverage(px, py)) + .product::() + * 255.0) + .round() as i32; + let actual = pixel_at(&preview, x, y)[0] as i32; + assert!( + (actual - expected).abs() <= 3, + "multiple masks pixel=({x},{y}) expected={expected} actual={actual}" + ); + } + } +} diff --git a/crates/opentake-render/tests/gpu_text.rs b/crates/opentake-render/tests/gpu_text.rs index ce367318..3b0b811e 100644 --- a/crates/opentake-render/tests/gpu_text.rs +++ b/crates/opentake-render/tests/gpu_text.rs @@ -192,6 +192,33 @@ fn y_span(frame: &DecodedFrame) -> u32 { } } +fn alpha_bounds(frame: &DecodedFrame) -> Option<(u32, u32, u32, u32)> { + let mut x0 = frame.width; + let mut y0 = frame.height; + let mut x1 = 0; + let mut y1 = 0; + let mut found = false; + for y in 0..frame.height { + for x in 0..frame.width { + if frame.rgba[((y * frame.width + x) * 4 + 3) as usize] > 0 { + found = true; + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x); + y1 = y1.max(y); + } + } + } + found.then_some((x0, y0, x1, y1)) +} + +fn left_alpha_run(frame: &DecodedFrame) -> u32 { + let y = frame.height / 2; + (0..frame.width) + .take_while(|&x| frame.rgba[((y * frame.width + x) * 4 + 3) as usize] > 0) + .count() as u32 +} + #[test] fn font_size_scales_with_canvas_height() { let r = CosmicTextRasterizer::new(); @@ -377,3 +404,123 @@ fn natural_size_shadow_padding_matches_upstream() { assert_eq!(TextLayout::SHADOW_PADDING, 12.0); assert_eq!(TextLayout::REFERENCE_CANVAS_HEIGHT, 1080.0); } + +#[test] +fn fallback_font_no_font_scaled_stroke_and_structural_golden_matrix() { + // Pinned structural values from the upstream CATextLayer/TextLayout path. + // Glyph edge antialiasing is renderer-specific, so the golden compares + // geometry and ordering rather than exact CoreText coverage bytes. + const UPSTREAM_REFERENCE_HEIGHT: f64 = 1080.0; + const UPSTREAM_BORDER_AT_REFERENCE: u32 = 2; + const UPSTREAM_SHADOW_PADDING_EACH_SIDE: f64 = 12.0; + + assert_eq!( + TextLayout::REFERENCE_CANVAS_HEIGHT, + UPSTREAM_REFERENCE_HEIGHT + ); + assert_eq!( + TextLayout::SHADOW_PADDING, + UPSTREAM_SHADOW_PADDING_EACH_SIDE + ); + + // A truly empty font database models headless CI. The request still + // returns a correctly sized transparent premultiplied frame and never + // panics or invents replacement glyphs. + let headless = CosmicTextRasterizer::without_system_fonts(); + assert!(!headless.has_fonts()); + let mut plain = TextStyle { + font_size: 96.0, + ..TextStyle::default() + }; + plain.shadow.enabled = false; + let no_font = headless + .rasterize(&TextRasterRequest { + clip_id: "headless", + content: "终验 Headless", + style: &plain, + box_norm: (0.0, 0.0, 1.0, 1.0), + canvas: (640, 360), + }) + .expect("headless frame"); + assert_eq!((no_font.width, no_font.height), (640, 360)); + assert!(no_font.premultiplied); + assert!(alpha_bounds(&no_font).is_none()); + + let rasterizer = CosmicTextRasterizer::new(); + if !rasterizer.has_fonts() { + eprintln!("[skip] no system fonts for structural golden matrix"); + return; + } + + // A missing requested family must fall back through cosmic-text/fontdb. + let mut fallback = plain.clone(); + fallback.font_name = "OpenTake-Definitely-Missing-Bold".into(); + fallback.alignment = TextAlignment::Center; + let mixed = rasterizer + .rasterize(&TextRasterRequest { + clip_id: "fallback", + content: "终验 FINAL CHECK", + style: &fallback, + box_norm: (0.0, 0.0, 1.0, 1.0), + canvas: (960, 270), + }) + .expect("fallback frame"); + let mixed_bounds = alpha_bounds(&mixed).expect("fallback must paint glyphs"); + assert!(mixed_bounds.0 < mixed.width / 2); + assert!(mixed_bounds.2 > mixed.width / 2); + + // Pinned left/center/right structural matrix for the same Latin/CJK run. + let centroid = |alignment| { + let mut style = fallback.clone(); + style.alignment = alignment; + let frame = rasterizer + .rasterize(&TextRasterRequest { + clip_id: "alignment", + content: "OpenTake 终验", + style: &style, + box_norm: (0.0, 0.0, 1.0, 1.0), + canvas: (960, 270), + }) + .expect("alignment frame"); + x_centroid(&frame) / frame.width as f64 + }; + let left = centroid(TextAlignment::Left); + let center = centroid(TextAlignment::Center); + let right = centroid(TextAlignment::Right); + assert!(left < 0.35, "left centroid {left}"); + assert!((0.4..0.6).contains(¢er), "center centroid {center}"); + assert!(right > 0.65, "right centroid {right}"); + assert!(left < center && center < right); + + // A narrow box must wrap the mixed-language fixture into more vertical + // structure than a short line. + let render = |content: &str| { + rasterizer + .rasterize(&TextRasterRequest { + clip_id: "wrap", + content, + style: &fallback, + box_norm: (0.0, 0.0, 0.34, 1.0), + canvas: (960, 540), + }) + .expect("wrap frame") + }; + assert!(y_span(&render("OpenTake 终验 wraps across multiple lines")) > y_span(&render("终验"))); + + // The upstream box-border stroke is 2 px at 1080p and scales with canvas + // height, with a one-pixel floor for small canvases. + let mut border = plain; + border.border.enabled = true; + for (height, expected) in [(540, 1), (1080, UPSTREAM_BORDER_AT_REFERENCE), (2160, 4)] { + let frame = rasterizer + .rasterize(&TextRasterRequest { + clip_id: "border", + content: "I", + style: &border, + box_norm: (0.0, 0.0, 1.0, 1.0), + canvas: (height, height), + }) + .expect("border frame"); + assert_eq!(left_alpha_run(&frame), expected, "canvas height {height}"); + } +} diff --git a/crates/opentake-render/tests/gpu_y_orientation.rs b/crates/opentake-render/tests/gpu_y_orientation.rs index f45e2933..cb334686 100644 --- a/crates/opentake-render/tests/gpu_y_orientation.rs +++ b/crates/opentake-render/tests/gpu_y_orientation.rs @@ -243,6 +243,7 @@ fn off_center_mask_clips_to_authored_screen_region_not_mirrored() { }, feather: 0.0, invert: false, + ..Mask::default() }]; let mut track = Track::new("t0", ClipType::Video); track.clips.push(clip); diff --git a/crates/opentake-render/tests/lut.rs b/crates/opentake-render/tests/lut.rs new file mode 100644 index 00000000..6190a3bf --- /dev/null +++ b/crates/opentake-render/tests/lut.rs @@ -0,0 +1,165 @@ +//! Real GPU acceptance for project-managed 3D `.cube` LUTs. + +use std::rc::Rc; + +use opentake_domain::{Clip, ClipType, CubeLut, LutReference, Point, Timeline, Track, Transform}; +use opentake_render::gpu::texture::{upload_lut_3d, upload_rgba}; +use opentake_render::source::DecodedFrame; +use opentake_render::wgpu; +use opentake_render::{ + build_render_plan, Compositor, GpuLutTexture, GpuTexture, RenderDevice, RenderError, + RenderSize, SourceMetrics, TextureResolver, TextureSource, +}; + +const RS: RenderSize = RenderSize { + width: 16, + height: 16, +}; + +struct Metrics; +impl SourceMetrics for Metrics { + fn natural_size(&self, _media_ref: &str) -> Option<(u32, u32)> { + Some((16, 16)) + } +} + +struct LutResolver<'d> { + device: &'d wgpu::Device, + queue: &'d wgpu::Queue, + source: Option>, + lut: CubeLut, + uploaded_lut: Option>, +} + +impl TextureResolver for LutResolver<'_> { + fn resolve(&mut self, _source: &TextureSource, _frame: i64) -> Option> { + if self.source.is_none() { + let frame = DecodedFrame::new(16, 16, [64, 128, 192, 255].repeat(16 * 16), true); + self.source = Some(Rc::new(upload_rgba( + self.device, + self.queue, + &frame, + false, + Some("lut-source"), + ))); + } + self.source.clone() + } + + fn resolve_lut( + &mut self, + _reference: &LutReference, + ) -> Result>, RenderError> { + if self.uploaded_lut.is_none() { + self.uploaded_lut = Some(Rc::new(upload_lut_3d( + self.device, + self.queue, + &self.lut, + Some("known-transform-lut"), + ))); + } + Ok(self.uploaded_lut.clone()) + } +} + +fn cube(size: usize, transform: impl Fn(f32, f32, f32) -> [f32; 3]) -> Vec { + let mut text = + format!("TITLE \"acceptance\"\nLUT_3D_SIZE {size}\nDOMAIN_MIN 0 0 0\nDOMAIN_MAX 1 1 1\n"); + let last = (size - 1) as f32; + for b in 0..size { + for g in 0..size { + for r in 0..size { + let [r, g, b] = transform(r as f32 / last, g as f32 / last, b as f32 / last); + text.push_str(&format!("{r:.7} {g:.7} {b:.7}\n")); + } + } + } + text.into_bytes() +} + +fn timeline(reference: LutReference) -> Timeline { + let mut timeline = Timeline::new(); + timeline.width = 16; + timeline.height = 16; + timeline.fps = 30; + let mut clip = Clip::new("clip", "asset", 0, 30); + clip.transform = Transform::from_top_left(Point { x: 0.0, y: 0.0 }, 1.0, 1.0); + clip.lut = Some(reference); + let mut track = Track::new("track", ClipType::Video); + track.clips.push(clip); + timeline.tracks.push(track); + timeline +} + +fn render(dev: &RenderDevice, timeline: &Timeline, lut: CubeLut) -> DecodedFrame { + let plan = build_render_plan(timeline, RS, &Metrics); + let frame = plan.frame(timeline, 0); + let compositor = Compositor::new(&dev.device); + let mut resolver = LutResolver { + device: &dev.device, + queue: &dev.queue, + source: None, + lut, + uploaded_lut: None, + }; + compositor + .render_to_rgba(&dev.device, &dev.queue, RS, &frame, &mut resolver) + .expect("render with a valid LUT") +} + +#[test] +fn malformed_and_oversized_luts_fail_closed_and_valid_lut_matches_preview_export() { + let malformed = b"LUT_3D_SIZE 17\n0 0 0\n"; + assert!(CubeLut::parse(malformed).is_err(), "short table must fail"); + assert!( + CubeLut::parse(&vec![b' '; CubeLut::MAX_BYTES + 1]).is_err(), + "oversized input must fail before parsing" + ); + assert!( + CubeLut::parse(&cube(16, |r, g, b| [r, g, b])).is_err(), + "only planned 17- and 33-point tables are accepted" + ); + + let identity = CubeLut::parse(&cube(17, |r, g, b| [r, g, b])).expect("17-point identity"); + let transform = + CubeLut::parse(&cube(33, |r, g, b| [b, g * 0.5, r])).expect("33-point transform"); + assert_eq!(identity.size(), 17); + assert_eq!(transform.size(), 33); + + let reference = LutReference::new( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "Known Transform", + 0.75, + ) + .expect("managed reference"); + assert_eq!( + reference.relative_path(), + "media/luts/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef.cube" + ); + + let authored = timeline(reference.clone()); + let json = serde_json::to_vec(&authored).expect("save project timeline"); + let reopened: Timeline = serde_json::from_slice(&json).expect("reopen project timeline"); + assert_eq!(reopened.tracks[0].clips[0].lut.as_ref(), Some(&reference)); + + let Ok(dev) = RenderDevice::try_new() else { + eprintln!("[skip] LUT GPU acceptance: no GPU device"); + return; + }; + let preview = render(&dev, &reopened, transform.clone()); + let export = render(&dev, &reopened, transform); + assert_eq!(preview.rgba, export.rgba, "preview/export LUT drift"); + + let center = (8 * 16 + 8) * 4; + let pixel = &preview.rgba[center..center + 4]; + assert!(pixel[0] > 140, "blue-to-red transform missing: {pixel:?}"); + assert!(pixel[1] < 100, "green attenuation missing: {pixel:?}"); + assert!(pixel[2] < 100, "red-to-blue transform missing: {pixel:?}"); + + let identity_timeline = timeline(LutReference::new(reference.id, "Identity", 1.0).unwrap()); + let identity_frame = render(&dev, &identity_timeline, identity); + let identity_pixel = &identity_frame.rgba[center..center + 4]; + for (actual, expected) in identity_pixel.iter().zip([64_u8, 128, 192, 255]) { + assert!((i16::from(*actual) - i16::from(expected)).abs() <= 2); + } +} diff --git a/crates/opentake-render/tests/nested_timeline.rs b/crates/opentake-render/tests/nested_timeline.rs new file mode 100644 index 00000000..6306c35c --- /dev/null +++ b/crates/opentake-render/tests/nested_timeline.rs @@ -0,0 +1,92 @@ +use opentake_domain::{Clip, ClipType, NestedSequence, Timeline, Track}; +use opentake_render::{build_render_plan, RenderSize, SourceMetrics}; + +struct Metrics; + +impl SourceMetrics for Metrics { + fn natural_size(&self, _media_ref: &str) -> Option<(u32, u32)> { + Some((64, 64)) + } +} + +fn video_clip(id: &str, media_ref: &str, start: i32, duration: i32) -> Clip { + let mut clip = Clip::new(id, media_ref, start, duration); + clip.media_type = ClipType::Video; + clip.source_clip_type = ClipType::Video; + clip +} + +fn sequence_clip(id: &str, sequence_id: &str, start: i32, duration: i32) -> Clip { + Clip::new_nested(id, sequence_id, start, duration) +} + +#[test] +fn nested_edits_preview_and_export_same_frames() { + let mut child_timeline = Timeline::new(); + child_timeline.settings_configured = true; + let mut child_track = Track::new("child-v1", ClipType::Video); + child_track + .clips + .push(video_clip("child-clip", "child-source-a", 2, 8)); + child_timeline.tracks.push(child_track); + + let mut root = Timeline::new(); + root.settings_configured = true; + root.nested_sequences + .push(NestedSequence::new("sequence-a", "Scene A", child_timeline)); + let mut root_track = Track::new("root-v1", ClipType::Video); + root_track + .clips + .push(sequence_clip("compound", "sequence-a", 10, 20)); + root.tracks.push(root_track); + + root.validate_nested_sequences() + .expect("valid nested graph"); + let encoded = serde_json::to_vec(&root).expect("serialize nested project"); + let mut reopened: Timeline = serde_json::from_slice(&encoded).expect("reopen nested project"); + assert_eq!(reopened.nested_sequences[0].name, "Scene A"); + + let preview_plan = build_render_plan(&reopened, RenderSize::new(64, 64), &Metrics); + let preview = preview_plan.frame(&reopened, 12); + assert_eq!(preview.draws.len(), 1); + assert_eq!(preview.draws[0].clip_id, "child-clip"); + assert_eq!(preview.draws[0].source_frame, 0); + + reopened.nested_sequences[0].timeline.tracks[0].clips[0].media_ref = + "child-source-b".to_string(); + reopened.nested_sequences[0].timeline.tracks[0].clips[0].trim_start_frame = 3; + + let preview_after_edit = build_render_plan(&reopened, RenderSize::new(64, 64), &Metrics); + let export_after_edit = build_render_plan(&reopened, RenderSize::new(64, 64), &Metrics); + for frame in 12..20 { + assert_eq!( + preview_after_edit.frame(&reopened, frame), + export_after_edit.frame(&reopened, frame), + "preview/export diverged at frame {frame}", + ); + } + assert_eq!( + preview_after_edit.frame(&reopened, 12).draws[0].source_frame, + 3 + ); + + let mut cycle = Timeline::new(); + let mut a = Timeline::new(); + let mut a_track = Track::new("a-track", ClipType::Video); + a_track.clips.push(sequence_clip("a-to-b", "b", 0, 10)); + a.tracks.push(a_track); + let mut b = Timeline::new(); + let mut b_track = Track::new("b-track", ClipType::Video); + b_track.clips.push(sequence_clip("b-to-a", "a", 0, 10)); + b.tracks.push(b_track); + cycle + .nested_sequences + .push(NestedSequence::new("a", "A", a)); + cycle + .nested_sequences + .push(NestedSequence::new("b", "B", b)); + let error = cycle + .validate_nested_sequences() + .expect_err("cycle must be rejected"); + assert!(error.contains("a -> b -> a"), "unexpected error: {error}"); +} diff --git a/crates/opentake-render/tests/optical_flow.rs b/crates/opentake-render/tests/optical_flow.rs new file mode 100644 index 00000000..63a1d0da --- /dev/null +++ b/crates/opentake-render/tests/optical_flow.rs @@ -0,0 +1,189 @@ +use std::rc::Rc; + +use opentake_media::{ + convert_frame_rate, interpolate_frame_pair, FrameInterpolationFallback, FrameInterpolationMode, + RgbaFrame, +}; +use opentake_render::gpu::compositor::{ + TextureInterpolationConfig, TextureInterpolationFallback, TextureInterpolationMode, + TextureResolveRequest, +}; +use opentake_render::{GpuTexture, TextureResolver, TextureSource}; + +fn moving_square(x: u32) -> RgbaFrame { + let mut frame = RgbaFrame::black(8, 4); + for y in 1..=2 { + for px_x in x..x + 2 { + let offset = ((y * frame.width + px_x) * 4) as usize; + frame.rgba[offset..offset + 4].copy_from_slice(&[255, 255, 255, 255]); + } + } + frame +} + +fn opposing_squares(left_x: u32, right_x: u32) -> RgbaFrame { + let mut frame = RgbaFrame::black(64, 32); + for y in 12..20 { + for x in left_x..left_x + 8 { + let offset = ((y * frame.width + x) * 4) as usize; + frame.rgba[offset..offset + 4].copy_from_slice(&[255, 255, 255, 255]); + } + for x in right_x..right_x + 8 { + let offset = ((y * frame.width + x) * 4) as usize; + frame.rgba[offset..offset + 4].copy_from_slice(&[255, 255, 255, 255]); + } + } + frame +} + +fn is_lit(frame: &RgbaFrame, x: u32, y: u32) -> bool { + frame.rgba[((y * frame.width + x) * 4) as usize] > 200 +} + +fn light_centroid_x(frame: &RgbaFrame) -> f64 { + let mut weighted_x = 0.0; + let mut weight = 0.0; + for y in 0..frame.height { + for x in 0..frame.width { + let offset = ((y * frame.width + x) * 4) as usize; + let value = frame.rgba[offset] as f64; + weighted_x += x as f64 * value; + weight += value; + } + } + weighted_x / weight +} + +#[derive(Default)] +struct RecordingResolver { + requests: Vec, +} + +impl TextureResolver for RecordingResolver { + fn resolve(&mut self, _source: &TextureSource, _source_frame: i64) -> Option> { + None + } + + fn resolve_with_interpolation( + &mut self, + request: TextureResolveRequest<'_>, + ) -> Option> { + self.requests.push(request.interpolation); + None + } +} + +#[test] +fn two_frame_fixture_is_deterministic_and_matches_preview_export() { + let first = moving_square(1); + let last = moving_square(5); + let conversion = convert_frame_rate(2, 24.0, 60.0).expect("valid 24 to 60 conversion"); + + assert_eq!(conversion.len(), 4); + assert_eq!(conversion.first().unwrap().timestamp_secs, 0.0); + assert_eq!(conversion.last().unwrap().timestamp_secs, 1.0 / 24.0); + assert_eq!(conversion[1].source_frame, 0); + assert_eq!(conversion[1].next_source_frame, 1); + assert!((conversion[1].source_alpha - 0.4).abs() < 1e-12); + assert!((conversion[2].source_alpha - 0.8).abs() < 1e-12); + + let render = |optical_flow_available| { + let source = [&first, &last]; + conversion + .iter() + .map(|sample| { + interpolate_frame_pair( + source[sample.source_frame as usize], + source[sample.next_source_frame as usize], + sample.source_alpha, + FrameInterpolationMode::OpticalFlow, + FrameInterpolationFallback::Blend, + optical_flow_available, + ) + .expect("configured blend fallback is infallible") + }) + .collect::>() + }; + let preview = render(true); + let export = render(true); + + assert_eq!(preview, export); + assert_eq!(preview.first().unwrap().frame, first); + assert_eq!(preview.last().unwrap().frame, last); + let centroids = preview + .iter() + .map(|result| light_centroid_x(&result.frame)) + .collect::>(); + assert!(centroids.windows(2).all(|pair| pair[0] < pair[1])); + assert!(preview + .iter() + .all(|result| result.mode_used == FrameInterpolationMode::OpticalFlow)); + + let fallback = render(false); + assert!(fallback + .iter() + .all(|result| result.mode_used == FrameInterpolationMode::Blend)); + assert_ne!(fallback[1].frame, preview[1].frame); + assert!(interpolate_frame_pair( + &first, + &last, + 0.5, + FrameInterpolationMode::OpticalFlow, + FrameInterpolationFallback::Error, + false, + ) + .is_err()); + + let interpolation = TextureInterpolationConfig::new( + 24.0, + 60.0, + TextureInterpolationMode::OpticalFlow, + TextureInterpolationFallback::Blend, + ) + .expect("valid render interpolation config"); + assert!(TextureInterpolationConfig::new( + 0.0, + 60.0, + TextureInterpolationMode::OpticalFlow, + TextureInterpolationFallback::Blend, + ) + .is_err()); + let source = TextureSource::Decoded { + media_ref: "motion-24fps".to_string(), + }; + let mut preview_resolver = RecordingResolver::default(); + let mut export_resolver = RecordingResolver::default(); + let request = TextureResolveRequest { + source: &source, + source_frame: 1, + interpolation, + }; + preview_resolver.resolve_with_interpolation(request); + export_resolver.resolve_with_interpolation(request); + + assert_eq!(preview_resolver.requests, export_resolver.requests); + assert_eq!(preview_resolver.requests, vec![interpolation]); +} + +#[test] +fn optical_flow_tracks_opposing_local_motion_without_global_frame_shift() { + let first = opposing_squares(4, 52); + let last = opposing_squares(12, 44); + let result = interpolate_frame_pair( + &first, + &last, + 0.5, + FrameInterpolationMode::OpticalFlow, + FrameInterpolationFallback::Error, + true, + ) + .expect("local optical flow should be available"); + + // Both objects move toward the center. A single global translation cannot + // satisfy these two regions simultaneously; the local field places both at + // their respective half-way locations. + assert!(is_lit(&result.frame, 10, 15)); + assert!(is_lit(&result.frame, 50, 15)); + assert!(!is_lit(&result.frame, 4, 15)); + assert!(!is_lit(&result.frame, 59, 15)); +} diff --git a/crates/opentake-render/tests/stabilization.rs b/crates/opentake-render/tests/stabilization.rs new file mode 100644 index 00000000..60171f68 --- /dev/null +++ b/crates/opentake-render/tests/stabilization.rs @@ -0,0 +1,149 @@ +use opentake_domain::{Clip, ClipType, Timeline, Track}; +use opentake_media::analysis::{ + analyze_stabilization, StabilizationConfig, StabilizationMotionSample, +}; +use opentake_media::MediaCancelToken; +use opentake_ops::{apply, EditCommand, EditorState, SeqIdGen}; +use opentake_render::{build_render_plan, RenderSize, SourceMetrics}; + +struct FullHdSource; + +impl SourceMetrics for FullHdSource { + fn natural_size(&self, _media_ref: &str) -> Option<(u32, u32)> { + Some((1920, 1080)) + } +} + +fn displacement(path: &[(f64, f64)]) -> f64 { + path.windows(2) + .map(|pair| (pair[1].0 - pair[0].0).hypot(pair[1].1 - pair[0].1)) + .sum() +} + +#[test] +fn synthetic_shake_produces_editable_undoable_preview_export_solution() { + let observed = [ + (0.000, 0.000), + (0.040, -0.018), + (-0.032, 0.022), + (0.047, -0.026), + (-0.038, 0.019), + (0.034, -0.015), + (0.000, 0.000), + ]; + let samples = observed + .iter() + .enumerate() + .map(|(frame, &(x, y))| StabilizationMotionSample { + frame: frame as i32, + translation_x: x, + translation_y: y, + rotation_degrees: 0.0, + }) + .collect::>(); + let cancel = MediaCancelToken::new(); + let solution = analyze_stabilization( + &samples, + "asset-shake", + StabilizationConfig::default(), + &cancel, + ) + .expect("synthetic stabilization analysis"); + + assert_eq!(solution.model, "opentake.motion-smoothing"); + assert_eq!(solution.model_version, 1); + assert_eq!(solution.source_identity, "asset-shake"); + let stabilized = samples + .iter() + .map(|sample| { + let correction = solution.sample(sample.frame); + ( + sample.translation_x + correction.translation_x, + sample.translation_y + correction.translation_y, + ) + }) + .collect::>(); + assert!(displacement(&stabilized) < displacement(&observed)); + assert!(solution.guarantees_coverage(16.0 / 9.0)); + + let mut timeline = Timeline::new(); + let mut track = Track::new("video-track", ClipType::Video); + track.clips.push(Clip::new( + "clip-shake", + "asset-shake", + 0, + samples.len() as i32, + )); + timeline.tracks.push(track); + let mut state = EditorState::from_timeline(timeline); + let ids = SeqIdGen::new("stabilization"); + + apply( + &mut state, + EditCommand::ApplyStabilization { + clip_id: "clip-shake".into(), + solution: solution.clone(), + }, + &ids, + ) + .expect("apply stabilization"); + assert_eq!(state.timeline.tracks[0].clips[0].media_ref, "asset-shake"); + assert_eq!(state.undo_depth(), 1); + + apply( + &mut state, + EditCommand::AdjustStabilization { + clip_id: "clip-shake".into(), + strength: Some(0.65), + crop_margin: Some(0.03), + }, + &ids, + ) + .expect("adjust stabilization"); + let edited = state.timeline.tracks[0].clips[0] + .stabilization + .as_ref() + .expect("persisted editable stabilization track"); + assert_eq!(edited.strength, 0.65); + assert_eq!(edited.crop_margin, 0.03); + + let preview = build_render_plan(&state.timeline, RenderSize::new(1920, 1080), &FullHdSource); + let export = build_render_plan(&state.timeline, RenderSize::new(1920, 1080), &FullHdSource); + for frame in 0..samples.len() as i32 { + let preview_draw = &preview.frame(&state.timeline, frame).draws[0]; + let export_draw = &export.frame(&state.timeline, frame).draws[0]; + assert_eq!(preview_draw.affine, export_draw.affine); + assert_eq!(preview_draw.crop_uv, export_draw.crop_uv); + } + + apply(&mut state, EditCommand::Undo, &ids).expect("undo adjustment"); + assert_eq!( + state.timeline.tracks[0].clips[0] + .stabilization + .as_ref() + .expect("analysis remains after undo") + .strength, + 1.0 + ); + apply( + &mut state, + EditCommand::ResetStabilization { + clip_id: "clip-shake".into(), + }, + &ids, + ) + .expect("reset stabilization"); + assert!(state.timeline.tracks[0].clips[0].stabilization.is_none()); + apply(&mut state, EditCommand::Undo, &ids).expect("undo reset"); + assert!(state.timeline.tracks[0].clips[0].stabilization.is_some()); + + let cancelled = MediaCancelToken::new(); + cancelled.cancel(); + assert!(analyze_stabilization( + &samples, + "asset-shake", + StabilizationConfig::default(), + &cancelled, + ) + .is_err()); +} diff --git a/crates/opentake-render/tests/transitions.rs b/crates/opentake-render/tests/transitions.rs new file mode 100644 index 00000000..51f59b2f --- /dev/null +++ b/crates/opentake-render/tests/transitions.rs @@ -0,0 +1,237 @@ +//! Owning acceptance test for the first advertised editable transition. + +use std::collections::HashMap; +use std::rc::Rc; + +use opentake_domain::{Clip, ClipType, Timeline, Track, Transform, TransitionKind}; +use opentake_ops::{apply, EditCommand, EditorState, SeqIdGen}; +use opentake_render::gpu::texture::upload_rgba; +use opentake_render::source::DecodedFrame; +use opentake_render::wgpu; +use opentake_render::{ + build_render_plan, Compositor, GpuTexture, RenderDevice, RenderSize, SourceMetrics, + TextureResolver, TextureSource, +}; + +const SIZE: RenderSize = RenderSize { + width: 16, + height: 16, +}; + +struct Metrics; + +impl SourceMetrics for Metrics { + fn natural_size(&self, _media_ref: &str) -> Option<(u32, u32)> { + Some((16, 16)) + } +} + +struct PairResolver<'d> { + device: &'d wgpu::Device, + queue: &'d wgpu::Queue, + cache: HashMap>, +} + +impl TextureResolver for PairResolver<'_> { + fn resolve(&mut self, source: &TextureSource, _frame: i64) -> Option> { + let media_ref = match source { + TextureSource::Decoded { media_ref } + | TextureSource::Image { media_ref } + | TextureSource::Lottie { media_ref } => media_ref, + TextureSource::Text { .. } => return None, + }; + if let Some(texture) = self.cache.get(media_ref) { + return Some(texture.clone()); + } + let color = match media_ref.as_str() { + "red" => [255, 0, 0, 255], + "blue" => [0, 0, 255, 255], + other => panic!("unexpected transition source {other}"), + }; + let mut rgba = vec![0; 16 * 16 * 4]; + for pixel in rgba.chunks_exact_mut(4) { + pixel.copy_from_slice(&color); + } + let frame = DecodedFrame::new(16, 16, rgba, true); + let texture = Rc::new(upload_rgba( + self.device, + self.queue, + &frame, + false, + Some("transition fixture"), + )); + self.cache.insert(media_ref.clone(), texture.clone()); + Some(texture) + } +} + +fn transition_timeline() -> Timeline { + let mut timeline = Timeline::new(); + timeline.fps = 30; + timeline.width = 16; + timeline.height = 16; + let mut outgoing = Clip::new("a", "red", 0, 12); + outgoing.transform = Transform::default(); + let mut incoming = Clip::new("b", "blue", 12, 12); + incoming.transform = Transform::default(); + let mut track = Track::new("v", ClipType::Video); + track.clips = vec![outgoing, incoming]; + timeline.tracks.push(track); + timeline +} + +fn render_frame(device: &RenderDevice, timeline: &Timeline, frame: i32) -> DecodedFrame { + let plan = build_render_plan(timeline, SIZE, &Metrics); + let frame_plan = plan.frame(timeline, frame); + let compositor = Compositor::new(&device.device); + let mut resolver = PairResolver { + device: &device.device, + queue: &device.queue, + cache: HashMap::new(), + }; + compositor + .render_to_rgba( + &device.device, + &device.queue, + SIZE, + &frame_plan, + &mut resolver, + ) + .expect("render transition frame") +} + +fn center_pixel(frame: &DecodedFrame) -> [u8; 4] { + let index = ((frame.height / 2 * frame.width + frame.width / 2) * 4) as usize; + frame.rgba[index..index + 4].try_into().unwrap() +} + +fn assert_pixel_near(actual: [u8; 4], expected: [u8; 4]) { + for channel in 0..4 { + assert!( + (actual[channel] as i16 - expected[channel] as i16).abs() <= 3, + "expected {expected:?}, got {actual:?}" + ); + } +} + +#[test] +fn adjacent_clip_transition_is_editable_undoable_and_matches_preview_export() { + let ids = SeqIdGen::default(); + let mut state = EditorState::from_timeline(transition_timeline()); + apply( + &mut state, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 4, + }, + &ids, + ) + .expect("add advertised transition"); + + let transition = state.timeline.tracks[0].clips[0] + .transition_out + .as_ref() + .expect("transition persisted on outgoing clip"); + assert_eq!(transition.to_clip_id, "b"); + assert_eq!(transition.kind, TransitionKind::CrossDissolve); + assert_eq!(transition.duration_frames, 4); + + // Pair identity must be explicit in the saved object, not inferred solely + // from whichever clip happens to contain it after reopening. + let saved = serde_json::to_string(&state.timeline).expect("save timeline JSON"); + assert!(saved.contains(r#""fromClipId":"a""#)); + assert!(saved.contains(r#""toClipId":"b""#)); + let reopened: Timeline = serde_json::from_str(&saved).expect("reopen timeline JSON"); + assert_eq!(reopened, state.timeline); + + apply(&mut state, EditCommand::Undo, &ids).expect("undo transition"); + assert!(state.timeline.tracks[0].clips[0].transition_out.is_none()); + apply(&mut state, EditCommand::Redo, &ids).expect("redo transition"); + assert_eq!(state.timeline, reopened); + + apply( + &mut state, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 3, + }, + &ids, + ) + .expect("change transition duration"); + assert_eq!( + state.timeline.tracks[0].clips[0] + .transition_out + .as_ref() + .unwrap() + .duration_frames, + 3 + ); + apply( + &mut state, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: None, + duration_frames: 3, + }, + &ids, + ) + .expect("remove transition"); + assert!(state.timeline.tracks[0].clips[0].transition_out.is_none()); + apply(&mut state, EditCommand::Undo, &ids).expect("undo transition removal"); + assert_eq!( + state.timeline.tracks[0].clips[0] + .transition_out + .as_ref() + .unwrap() + .duration_frames, + 3 + ); + + // A 12-frame pair exposes a six-frame centered transition handle. An + // oversized request is an explicit refusal and must not mutate history. + let mut invalid = EditorState::from_timeline(transition_timeline()); + apply( + &mut invalid, + EditCommand::SetTransition { + from_clip_id: "a".into(), + to_clip_id: "b".into(), + kind: Some(TransitionKind::CrossDissolve), + duration_frames: 7, + }, + &ids, + ) + .expect_err("transition longer than either available handle must be rejected"); + assert_eq!(invalid.version(), 0); + assert!(!invalid.can_undo()); + + let device = match RenderDevice::try_new() { + Ok(device) => device, + Err(error) => { + eprintln!("[skip] transition pixel fixture: no GPU device ({error})"); + return; + } + }; + + // Duration four covers frames 8..12. Exercise the first transition frame, + // midpoint, last blended frame, and the exact cut/end frame. Preview and + // export are represented by fresh compositor/resolver executions. + for (frame, golden) in [ + (8, [255, 0, 0, 255]), + (10, [128, 0, 128, 255]), + (11, [64, 0, 191, 255]), + (12, [0, 0, 255, 255]), + ] { + let preview = render_frame(&device, &reopened, frame); + let export = render_frame(&device, &reopened, frame); + assert_eq!( + preview.rgba, export.rgba, + "preview/export drift at frame {frame}" + ); + assert_pixel_near(center_pixel(&preview), golden); + } +} diff --git a/docs/architecture/ADVANCED-FEATURES.md b/docs/architecture/ADVANCED-FEATURES.md index eb4cd7e1..b71dd571 100644 --- a/docs/architecture/ADVANCED-FEATURES.md +++ b/docs/architecture/ADVANCED-FEATURES.md @@ -1,5 +1,19 @@ # OpenTake 进阶能力设计(对标剪映模块 1–5 的差距深化) +> **2026-08-01 Beta 对账附录(覆盖下表“现状”列)**:本文的 `missing` / `partial` +> 是实现前的历史快照,继续保留作为设计来源,不再表示当前代码状态。首个 Beta 已闭环: +> 通用特效、交叉溶解、线性光调色、Lift/Gamma/Gain、HSL、3D LUT、绿幕、圆形/线性/ +> 钢笔蒙版、参考色彩匹配、RVM 抠像、防抖、运动追踪、光流补帧、智能擦除、响度、降噪、 +> 声部分离、嵌套时间线、字幕样式同步与 SRT/VTT、口播清理、图文成片、字幕翻译、数字人和 +> 音色克隆。代码与实机证据以 +> `docs/audit/2026-07-14/runtime-artifacts/automated/` 及 +> `docs/releases/1.0.0-beta.1.md` 为准。 +> +> 后续 Beta 明确保留的扩展是:Bezier/Spring 物理 easing、RGB 多维曲线、wipe/slide/zoom +> 等通用双源转场、本地神经超分、任意混音的神经语义分轨、高阶曲线变速、多机位自动对齐、 +> 任意 Motion Canvas TSX 与透明 frame sequence。这些是产品路线,不属于 +> `1.0.0-beta.1` 可用性门禁;当前 UI 不把它们伪装成已可用能力。 + > 来源:`docs/CAPCUT-GAP.md`(5 个子 Agent 对照剪映模块 1–5 与上游源码的逐特性差距分析)。 > 范围:**不含**剪映模块 6(自然语言交互/语音助手 —— OpenTake 已有 Agent)与模块 7(云生态/企业协作)。 > 现状:33 项中 已有 2 / 部分 7 / 缺失 24。上游 Palmier Pro 自述「尚无:特效/转场/调色/蒙版/图形」,源码核对完全坐实。 @@ -76,7 +90,7 @@ OpenTake 补齐这些进阶能力,**几乎不需要新建基础设施**,全部 | 复合片段嵌套 nested clip | missing | high | p2 | **过渡方案先做**:`saveTimelineRange` 用 FFmpeg/wgpu 重写为「打组烧成内部媒体 + content-hash 缓存」满足「精简图层」;**完整方案后做**:domain 新增 `MediaSource::Nested(child_timeline_id)`,RenderPlan 递归展开或子序列离屏渲染成单层 | | 多机位自动对齐 multicam | missing | medium | p2 | 纯本地:各机位音轨 → PCM → rustfft **互相关**求最佳时移 → ops 整体平移到同一时基;多角度切换面板作后续 UI | | 字幕样式全局批量同步 | partial(共享样式在,批量算子缺) | low | p1 | 新增「改一处 → 批量回写整 captionGroup」编辑命令 | -| 导出 .srt 字幕文件 | missing | low | p1 | 从 caption 模型按时码序列化 SubRip;顺带支持 .vtt | +| 导出 .srt 字幕文件 | **已有** | low | p1 | caption 模型按时码序列化 SubRip/WebVTT;TitleBar 原生保存对话框接 `export_subtitles`,SRT/VTT 均有 Rust 与 UI 路由测试 | --- @@ -88,8 +102,8 @@ OpenTake 补齐这些进阶能力,**几乎不需要新建基础设施**,全部 |---|---|---|---|---| | 智能剪口播(剔除停顿/语气词) | partial(已规划) | medium | p1 | **本地为主**:词级 `get_transcript` + 静音检测 → Rust 内一次算好 ripple 区间(高阶工具 `remove_filler_words`/`tighten_silences`,避免把帧算术外包给 LLM) | | 图文成片 script-to-video | partial(地基在) | high | p1 | agent 编排既有工具:脚本 → `generate_image`→`generate_video`→`generate_audio`(配音)→`add_clips`/`add_texts`/`set_transition`;素材匹配用 SigLIP2 搜索 + import_media 接 stock | -| 音色克隆 voice cloning | missing | high | p2 | 外部 API(ElevenLabs 等)经 opentake-gen,扩展 audio 生成参数支持自定义音色 | -| 虚拟数字人 digital avatar | missing | high | p3 | 外部 API(HeyGen/fal 等)经 opentake-gen,新增 catalog kind | +| 音色克隆 voice cloning | **has** | high | p2 | ElevenLabs IVC 注册/TTS/永久撤销生产桥;参考音频、同意记录、请求哈希和 provider voice id 持久化,生成音频原子导入落轨并可试听/撤销 | +| 虚拟数字人 digital avatar | **has** | high | p3 | fal Sync Lipsync v3 image-to-video 生产桥;人像+驱动音频、同意与成本确认、结果探测、原子导入落轨和预览/撤销完整接入 | | 多语种翻译(字幕) | partial(靠 agent) | medium | p2 | 一等公民:离线 MT 或外部 API + LLM 兜底,翻译后保持时码 | --- diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index af78cb65..a3914380 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -77,7 +77,7 @@ OpenTake/ │ ├── opentake-motion/ # 原生 motion fallback:RGBA frame cache / sandbox / StubRenderer / 后续 alpha source │ └── opentake-core/ # 组装:EditorState(持有 timeline+manifest)、command 路由、事件总线 ├── plugins/ -│ └── motion-canvas-studio/ # 待新增:Motion Canvas(MIT) fork/plugin,渲染 mp4 后导入落轨 +│ └── motion-canvas-studio/ # Motion Canvas 3.17.2 MIT wrapper,确定性渲染 mp4 后导入落轨 ├── src-tauri/ # Tauri 2 app:#[tauri::command] 薄封装 + 窗口/菜单/生命周期 ├── web/ # React + TS 前端(Vite) ├── services/ @@ -87,7 +87,7 @@ OpenTake/ 依赖法则(经上游验证):`domain` 零依赖叶子;`ops` 只依赖 `domain`;`command` 是唯一编辑入口;UI/Agent/MCP 是命令层三个对等客户端。 -> Motion / AI Video 主线改为 `plugins/motion-canvas-studio/`:fork Motion Canvas(MIT),渲染 materialized mp4 后由 OpenTake 当普通媒体导入并落轨。`crates/opentake-motion/` 已有 scaffold 保留为后续透明 alpha / PNG sequence / HTML-CSS fallback,不作为 v1 主渲染器 blocker。 +> Motion / AI Video v1 已由 `plugins/motion-canvas-studio/` 落地:锁定 Motion Canvas 3.17.2(MIT),渲染 materialized mp4 后由 OpenTake 当普通媒体原子导入并落轨。`crates/opentake-motion/` 同时提供离线 Chromium 宿主与 HTML/CSS fallback;透明 alpha / PNG sequence 留给后续版本。 ## 4. 领域模型(可直接复刻,见 MODULE-PORT-MAP.md「Models」) diff --git a/docs/architecture/BUGS.md b/docs/architecture/BUGS.md index 4dda349b..194184b9 100644 --- a/docs/architecture/BUGS.md +++ b/docs/architecture/BUGS.md @@ -17,7 +17,7 @@ --- -### B2. 前端帧数学截断不一致(高) +### B2. 前端帧数学截断不一致(已修复) | 属性 | 值 | |---|---| @@ -25,6 +25,7 @@ | **描述** | Rust 端 `seconds_to_frame()` 使用截断 `(seconds * fps) as i32`(对应上游 `Int(s*fps)`),前端使用 `Math.round`(四舍五入)。当 `seconds * fps` 小数部分 ≥0.5 时,双方结果差 1 帧,导致同一媒体的计算时长不一致 | | **影响** | 媒体导入时的帧数计算偏差可能导致 clip 时长 off-by-1 | | **修复方案** | 将前端的 `Math.round(seconds * fps)` 改为 `Math.floor(seconds * fps)` | +| **验证** | 媒体时长、搜索片段 trim 起点/时长、ripple insert 与默认文本时长统一使用向零截断;`seconds_to_frame_truncates_fractional_boundaries` 覆盖 24/30 fps 的半帧、帧边界和无效输入。仅播放头吸附等明确的 nearest-frame UI 交互继续使用 `Math.round`。 | --- @@ -59,12 +60,12 @@ | **描述** | GPU 合成的 infrastructure 已就绪(`composite_frame` Tauri 命令、`useTimelineFrame` hook 均存在),但 `Preview.tsx` 仍然使用 DOM `
); diff --git a/web/src/components/agent/AgentPanel.tsx b/web/src/components/agent/AgentPanel.tsx index 97c5fd53..f47de367 100644 --- a/web/src/components/agent/AgentPanel.tsx +++ b/web/src/components/agent/AgentPanel.tsx @@ -1,7 +1,16 @@ -import { useEffect, useRef, useState, type CSSProperties, type KeyboardEvent } from "react"; +import { + useEffect, + useRef, + useState, + type CSSProperties, + type KeyboardEvent, + type ReactNode, +} from "react"; import { ChevronDown, ChevronRight, + Clapperboard, + MessageSquare, Plus, Send, Settings as SettingsIcon, @@ -25,6 +34,7 @@ import { useSettingsStore } from "../../store/settingsStore"; import { mintSessionId, useChatStore } from "../../store/chatStore"; import { useEditorUiStore } from "../../store/uiStore"; import { useProjectStore } from "../../store/projectStore"; +import { MotionPanel } from "./MotionPanel"; const NO_KEY_HINT = /Settings|设置|API key/i; @@ -45,13 +55,22 @@ export function AgentPanel() { const finalize = useChatStore((state) => state.finalize); const setMessages = useChatStore((state) => state.setMessages); const reset = useChatStore((state) => state.reset); + const composerDraft = useChatStore((state) => state.composerDraft); + const setComposerDraft = useChatStore((state) => state.setComposerDraft); const [input, setInput] = useState(""); + const [panelMode, setPanelMode] = useState<"chat" | "motion">("chat"); const [sessions, setSessions] = useState([]); const sessionsRef = useRef([]); const tabMutationRef = useRef>(Promise.resolve()); const scrollRef = useRef(null); + useEffect(() => { + if (composerDraft === null) return; + setInput(composerDraft); + setComposerDraft(null); + }, [composerDraft, setComposerDraft]); + function commitSessions(next: ChatSession[]) { sessionsRef.current = next; setSessions(next); @@ -342,9 +361,9 @@ export function AgentPanel() { }} > - {t("agent.title")} + {panelMode === "chat" ? t("agent.title") : t("motion.heading")} - + } +
+ } + onClick={() => setPanelMode("chat")} + /> + } + onClick={() => setPanelMode("motion")} + /> +
+ + {panelMode === "chat" ? <>
)}
+ : } ); } +function PanelModeButton({ + active, + label, + icon, + onClick, +}: { + active: boolean; + label: string; + icon: ReactNode; + onClick: () => void; +}) { + return ( + + ); +} + function sessionTitle(session: ChatSession, fallback: string): string { const firstUserMessage = session.messages.find( (message) => message.role === "user" && message.content.trim().length > 0, diff --git a/web/src/components/agent/MotionPanel.test.tsx b/web/src/components/agent/MotionPanel.test.tsx new file mode 100644 index 00000000..2cde5b56 --- /dev/null +++ b/web/src/components/agent/MotionPanel.test.tsx @@ -0,0 +1,85 @@ +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const api = vi.hoisted(() => ({ + addMotion: vi.fn(), + cancelMotion: vi.fn(), + motionCapability: vi.fn(), + onMotionProgress: vi.fn(), +})); +const sync = vi.hoisted(() => ({ forceRefresh: vi.fn() })); + +vi.mock("../../lib/api", () => ({ + ...api, + isTauri: true, +})); +vi.mock("../../store/sync", () => sync); + +import { useI18nStore } from "../../i18n"; +import { useEditorUiStore } from "../../store/uiStore"; +import { useProjectStore } from "../../store/projectStore"; +import { MotionPanel } from "./MotionPanel"; + +describe("MotionPanel", () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(async () => { + vi.clearAllMocks(); + useI18nStore.setState({ locale: "en" }); + useProjectStore.setState({ + projectPath: "/tmp/demo.opentake", + timeline: { fps: 24, width: 1920, height: 1080, settingsConfigured: true, tracks: [] }, + }); + useEditorUiStore.setState({ activeFrame: 48, selectedClipIds: new Set() }); + api.motionCapability.mockResolvedValue(true); + api.onMotionProgress.mockResolvedValue(() => {}); + api.addMotion.mockResolvedValue({ + clipId: "motion-clip", + assetId: "motion-asset", + contentHash: "hash", + actionName: "Add Motion Graphic", + output: { + renderer: "motion-canvas", + rendererVersion: "3.17.2", + outputFile: "output.mp4", + fps: 24, + width: 1920, + height: 1080, + durationFrames: 72, + durationSeconds: 3, + contentHash: "hash", + }, + }); + sync.forceRefresh.mockResolvedValue(undefined); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + await act(async () => root.render()); + }); + + afterEach(async () => { + if (root) await act(async () => root.unmount()); + container?.remove(); + }); + + it("submits a frame-exact template at the playhead and selects the committed clip", async () => { + const add = Array.from(container.querySelectorAll("button")).find((button) => + button.textContent?.includes("Add at playhead"), + ); + expect(add).toBeDefined(); + await act(async () => add!.click()); + + expect(api.addMotion).toHaveBeenCalledWith( + expect.objectContaining({ + templateId: "title-card", + startFrame: 48, + durationFrames: 72, + }), + ); + expect(sync.forceRefresh).toHaveBeenCalledOnce(); + expect(useEditorUiStore.getState().selectedClipIds).toEqual(new Set(["motion-clip"])); + }); +}); +// @vitest-environment happy-dom diff --git a/web/src/components/agent/MotionPanel.tsx b/web/src/components/agent/MotionPanel.tsx new file mode 100644 index 00000000..dee1a746 --- /dev/null +++ b/web/src/components/agent/MotionPanel.tsx @@ -0,0 +1,257 @@ +import { useEffect, useMemo, useState } from "react"; +import { Clapperboard, Square } from "lucide-react"; +import { + addMotion, + cancelMotion, + isTauri, + motionCapability, + onMotionProgress, + type MotionProgressPhase, +} from "../../lib/api"; +import { useT } from "../../i18n"; +import { useEditorUiStore } from "../../store/uiStore"; +import { useProjectStore } from "../../store/projectStore"; +import { forceRefresh } from "../../store/sync"; + +const PHASE_KEYS: Record = { + validating: "motion.phaseValidating", + rendering: "motion.phaseRendering", + encoding: "motion.phaseEncoding", + committing: "motion.phaseCommitting", + complete: "motion.phaseComplete", +}; + +export function MotionPanel() { + const t = useT(); + const projectPath = useProjectStore((state) => state.projectPath); + const fps = useProjectStore((state) => state.timeline.fps); + const activeFrame = useEditorUiStore((state) => state.activeFrame); + const selectClips = useEditorUiStore((state) => state.selectClips); + const [available, setAvailable] = useState(null); + const [templateId, setTemplateId] = useState<"title-card" | "lower-third.glass">( + "title-card", + ); + const [title, setTitle] = useState("OpenTake"); + const [subtitle, setSubtitle] = useState(""); + const [accent, setAccent] = useState("#7C5CFF"); + const [durationSeconds, setDurationSeconds] = useState(3); + const [phase, setPhase] = useState(null); + const [error, setError] = useState(null); + const rendering = phase !== null && phase !== "complete"; + const durationFrames = useMemo( + () => Math.max(1, Math.round(durationSeconds * Math.max(1, fps))), + [durationSeconds, fps], + ); + + useEffect(() => { + let disposed = false; + void motionCapability().then((result) => { + if (!disposed) setAvailable(result); + }); + return () => { + disposed = true; + }; + }, []); + + useEffect(() => { + let disposed = false; + let unlisten: () => void = () => {}; + void onMotionProgress(setPhase).then((dispose) => { + if (disposed) dispose(); + else unlisten = dispose; + }); + return () => { + disposed = true; + unlisten(); + }; + }, []); + + async function add() { + if (!projectPath || !available || rendering || !title.trim()) return; + setError(null); + setPhase("validating"); + try { + const commit = await addMotion({ + templateId, + params: { + title: title.trim(), + subtitle: subtitle.trim(), + accent, + }, + startFrame: Math.max(0, Math.round(activeFrame)), + durationFrames, + }); + await forceRefresh(); + selectClips(new Set([commit.clipId])); + setPhase("complete"); + } catch (reason) { + setPhase(null); + setError(reason instanceof Error ? reason.message : String(reason)); + } + } + + async function cancel() { + await cancelMotion(); + } + + if (!isTauri) { + return {t("motion.desktopOnly")}; + } + if (available === null) { + return {t("motion.checking")}; + } + if (!available) { + return {t("motion.unavailable")}; + } + + return ( +
+
+
+ {t("motion.heading")} +
+
+ {t("motion.description")} +
+
+ + + + +
+ + +
+ + {phase && ( +
+ {t(PHASE_KEYS[phase])} +
+ )} + {error && ( +
+ {error} +
+ )} + + {rendering ? ( + + ) : ( + + )} +
+ ); +} + +function MotionNotice({ children }: { children: string }) { + return ( +
+ {children} +
+ ); +} + +const fieldStyle = { + display: "flex", + flexDirection: "column", + gap: 5, + color: "var(--text-secondary)", + fontSize: "var(--fs-xs)", +} as const; + +const inputStyle = { + width: "100%", + border: "var(--bw-thin) solid var(--border-subtle)", + borderRadius: "var(--radius-sm)", + background: "var(--bg-elevated)", + color: "var(--text-primary)", + padding: "7px 9px", + fontFamily: "inherit", + fontSize: "var(--fs-sm)", +} as const; + +const primaryButtonStyle = { + minHeight: 36, + display: "inline-flex", + alignItems: "center", + justifyContent: "center", + gap: 7, + borderRadius: "var(--radius-sm)", + background: "var(--accent-primary)", + color: "#111", + fontSize: "var(--fs-sm)", + fontWeight: 650, +} as const; diff --git a/web/src/components/home/HomeView.interaction.test.tsx b/web/src/components/home/HomeView.interaction.test.tsx new file mode 100644 index 00000000..edfbdfa0 --- /dev/null +++ b/web/src/components/home/HomeView.interaction.test.tsx @@ -0,0 +1,325 @@ +// @vitest-environment happy-dom + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + newProjectAndEnter: vi.fn(), + openProjectPath: vi.fn(), + openProjectViaDialog: vi.fn(), +})); + +vi.mock("../../i18n", () => ({ + useT: () => (key: string) => key, +})); + +vi.mock("../../lib/api", () => ({ isTauri: false })); + +vi.mock("../../store/projectActions", () => ({ + newProjectAndEnter: mocks.newProjectAndEnter, + openProjectPath: mocks.openProjectPath, + openProjectViaDialog: mocks.openProjectViaDialog, +})); + +import { useRecentStore } from "../../store/recentStore"; +import { useEditorUiStore } from "../../store/uiStore"; +import { + HOME_NOTICE_STORAGE_KEY, + HOME_NOTICE_VERSION, + HomeView, +} from "./HomeView"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; + +const PROJECT_PATH = "/tmp/Recent Demo.opentake"; + +let root: Root | null = null; +let container: HTMLDivElement | null = null; + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +function buttonsNamed(label: string): HTMLButtonElement[] { + return [...(container?.querySelectorAll("button") ?? [])] + .filter((button) => button.textContent === label); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.newProjectAndEnter.mockResolvedValue(undefined); + mocks.openProjectPath.mockResolvedValue(undefined); + mocks.openProjectViaDialog.mockResolvedValue(undefined); + localStorage.clear(); + localStorage.setItem(HOME_NOTICE_STORAGE_KEY, HOME_NOTICE_VERSION); + useRecentStore.setState({ + recents: [{ path: PROJECT_PATH, name: "Recent Demo", openedAt: 1 }], + }); + useEditorUiStore.setState({ view: "home", settingsOpen: false }); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + if (root) await act(async () => root?.unmount()); + container?.remove(); + root = null; + container = null; +}); + +describe("Home recent-project controls", () => { + it("control-e2d0f1ed3415ea45 create a new project from the Home sidebar", async () => { + const pending = deferred(); + mocks.newProjectAndEnter.mockReturnValueOnce(pending.promise); + await act(async () => root?.render()); + const sidebarNew = buttonsNamed("home.newProject")[0]; + + await act(async () => sidebarNew?.click()); + + expect(mocks.newProjectAndEnter).toHaveBeenCalledTimes(1); + expect(buttonsNamed("home.creating")).toHaveLength(2); + expect(buttonsNamed("home.creating").every((button) => button.disabled)).toBe(true); + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(buttonsNamed("home.newProject")).toHaveLength(2)); + }); + + it("control-575978f9bced5959 create a new project from the empty launcher", async () => { + const pending = deferred(); + mocks.newProjectAndEnter.mockReturnValueOnce(pending.promise); + useRecentStore.setState({ recents: [] }); + await act(async () => root?.render()); + const emptyNew = buttonsNamed("home.newProject")[1]; + + await act(async () => emptyNew?.click()); + + expect(mocks.newProjectAndEnter).toHaveBeenCalledTimes(1); + expect(buttonsNamed("home.creating")).toHaveLength(2); + expect(buttonsNamed("home.openProject").every((button) => button.disabled)).toBe(true); + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(buttonsNamed("home.newProject")).toHaveLength(2)); + }); + + it("control-74f414d717baaed3 create a new project from the populated launcher", async () => { + const pending = deferred(); + mocks.newProjectAndEnter.mockReturnValueOnce(pending.promise); + await act(async () => root?.render()); + const populatedNew = buttonsNamed("home.newProject")[1]; + + await act(async () => populatedNew?.click()); + + expect(mocks.newProjectAndEnter).toHaveBeenCalledTimes(1); + expect(buttonsNamed("home.creating")).toHaveLength(2); + expect(container?.querySelector("button.home-project-card")?.disabled).toBe( + true, + ); + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(buttonsNamed("home.newProject")).toHaveLength(2)); + }); + + it("restores all project controls after new-project creation rejects", async () => { + mocks.newProjectAndEnter.mockRejectedValueOnce(new Error("project create failed")); + await act(async () => root?.render()); + + await act(async () => buttonsNamed("home.newProject")[0]?.click()); + + expect(mocks.newProjectAndEnter).toHaveBeenCalledTimes(1); + expect(buttonsNamed("home.newProject")).toHaveLength(2); + expect(buttonsNamed("home.newProject").every((button) => !button.disabled)).toBe(true); + expect(buttonsNamed("home.openProject").every((button) => !button.disabled)).toBe(true); + expect(container?.querySelector("button.home-project-card")?.disabled).toBe( + false, + ); + }); + + it("control-ef78873f98fcab84 open a project from the Home sidebar", async () => { + const pending = deferred(); + mocks.openProjectViaDialog.mockReturnValueOnce(pending.promise); + await act(async () => root?.render()); + const sidebarOpen = buttonsNamed("home.openProject")[0]; + + await act(async () => sidebarOpen?.click()); + expect(mocks.openProjectViaDialog).toHaveBeenCalledTimes(1); + expect(sidebarOpen?.textContent).toBe("home.opening"); + expect(sidebarOpen?.disabled).toBe(true); + expect(buttonsNamed("home.opening")).toHaveLength(2); + expect(buttonsNamed("home.opening").every((button) => button.disabled)).toBe(true); + expect(buttonsNamed("home.newProject").every((button) => button.disabled)).toBe(true); + const recent = container?.querySelector("button.home-project-card"); + expect(recent?.disabled).toBe(true); + + await act(async () => buttonsNamed("home.opening")[1]?.click()); + expect(mocks.openProjectViaDialog).toHaveBeenCalledTimes(1); + + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(sidebarOpen?.textContent).toBe("home.openProject")); + expect(sidebarOpen?.disabled).toBe(false); + expect(buttonsNamed("home.newProject").every((button) => !button.disabled)).toBe(true); + expect(recent?.disabled).toBe(false); + }); + + it("control-2121d7b9fdc279b9 open a project from the empty launcher", async () => { + const pending = deferred(); + mocks.openProjectViaDialog.mockReturnValueOnce(pending.promise); + useRecentStore.setState({ recents: [] }); + await act(async () => root?.render()); + const emptyOpen = buttonsNamed("home.openProject")[1]; + + await act(async () => emptyOpen?.click()); + expect(mocks.openProjectViaDialog).toHaveBeenCalledTimes(1); + expect(emptyOpen?.textContent).toBe("home.opening"); + expect(emptyOpen?.disabled).toBe(true); + + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(emptyOpen?.textContent).toBe("home.openProject")); + expect(emptyOpen?.disabled).toBe(false); + }); + + it("control-ab109c708bb0efbf open a project from the populated launcher", async () => { + const pending = deferred(); + mocks.openProjectViaDialog.mockReturnValueOnce(pending.promise); + await act(async () => root?.render()); + const populatedOpen = buttonsNamed("home.openProject")[1]; + + await act(async () => populatedOpen?.click()); + expect(mocks.openProjectViaDialog).toHaveBeenCalledTimes(1); + expect(populatedOpen?.textContent).toBe("home.opening"); + expect(populatedOpen?.disabled).toBe(true); + + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(populatedOpen?.textContent).toBe("home.openProject")); + expect(populatedOpen?.disabled).toBe(false); + }); + + it("restores an open control after the native project command rejects", async () => { + const failure = new Error("project open timed out"); + mocks.openProjectViaDialog.mockRejectedValueOnce(failure); + await act(async () => root?.render()); + const sidebarOpen = buttonsNamed("home.openProject")[0]; + + await act(async () => sidebarOpen?.click()); + + expect(mocks.openProjectViaDialog).toHaveBeenCalledTimes(1); + expect(sidebarOpen?.textContent).toBe("home.openProject"); + expect(sidebarOpen?.disabled).toBe(false); + }); + + it("serializes a recent-card open through the shared Home pending state", async () => { + const pending = deferred(); + mocks.openProjectPath.mockReturnValueOnce(pending.promise); + await act(async () => root?.render()); + const card = container?.querySelector("button.home-project-card"); + + await act(async () => + card?.dispatchEvent(new MouseEvent("dblclick", { bubbles: true })), + ); + + expect(mocks.openProjectPath).toHaveBeenCalledTimes(1); + expect(buttonsNamed("home.opening")).toHaveLength(2); + expect(card?.disabled).toBe(true); + + await act(async () => + card?.dispatchEvent(new MouseEvent("dblclick", { bubbles: true })), + ); + expect(mocks.openProjectPath).toHaveBeenCalledTimes(1); + + await act(async () => pending.resolve()); + await vi.waitFor(() => expect(card?.disabled).toBe(false)); + expect(buttonsNamed("home.openProject")).toHaveLength(2); + }); + + it("control-f4a6b4f8789ea013 open the global Library from Home", async () => { + await act(async () => root?.render()); + const library = [...(container?.querySelectorAll("button") ?? [])] + .find((button) => button.textContent === "library.entry"); + + expect(library).not.toBeUndefined(); + await act(async () => library?.click()); + expect(useEditorUiStore.getState().view).toBe("library"); + }); + + it("control-810a7d793fcd8323 open Settings from Home", async () => { + await act(async () => root?.render()); + const settings = [...(container?.querySelectorAll("button") ?? [])] + .find((button) => button.textContent === "home.settings"); + + expect(settings).not.toBeUndefined(); + await act(async () => settings?.click()); + expect(useEditorUiStore.getState().settingsOpen).toBe(true); + }); + + it("control-acd6238c08e790cc clear recent-project card selection", async () => { + await act(async () => root?.render()); + const card = container?.querySelector( + 'button.home-project-card[aria-label="Recent Demo"]', + ); + const launcher = card?.parentElement?.parentElement?.parentElement; + + expect(launcher).not.toBeNull(); + await act(async () => card?.click()); + expect(card?.getAttribute("aria-pressed")).toBe("true"); + await act(async () => launcher?.click()); + expect(card?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("control-ec1cd7a2d49bb97a remove a recent project entry", async () => { + await act(async () => root?.render()); + const card = container?.querySelector( + 'button.home-project-card[aria-label="Recent Demo"]', + ); + const wrapper = card?.parentElement; + + await act(async () => wrapper?.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }))); + const remove = container?.querySelector( + 'button[aria-label="home.remove"]', + ); + expect(remove).not.toBeNull(); + + await act(async () => remove?.click()); + expect(useRecentStore.getState().recents).toEqual([]); + expect(container?.querySelector("button.home-project-card")).toBeNull(); + expect(mocks.openProjectPath).not.toHaveBeenCalled(); + }); + + it("control-9697b53d4d2cf1ca select or open a recent project card", async () => { + await act(async () => root?.render()); + const card = container?.querySelector( + 'button.home-project-card[aria-label="Recent Demo"]', + ); + + expect(card).not.toBeNull(); + expect(card?.tabIndex).toBe(0); + expect(card?.getAttribute("aria-label")).toBe("Recent Demo"); + expect(card?.getAttribute("aria-pressed")).toBe("false"); + + await act(async () => card?.focus()); + expect(card?.getAttribute("aria-pressed")).toBe("true"); + const sidebarNew = [...(container?.querySelectorAll("button") ?? [])] + .find((button) => button.textContent === "home.newProject"); + await act(async () => sidebarNew?.dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true }), + )); + expect(mocks.openProjectPath).not.toHaveBeenCalled(); + + await act(async () => card?.dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true }), + )); + expect(mocks.openProjectPath).toHaveBeenCalledTimes(1); + expect(mocks.openProjectPath).toHaveBeenLastCalledWith(PROJECT_PATH); + + mocks.openProjectPath.mockClear(); + await act(async () => card?.click()); + expect(card?.getAttribute("aria-pressed")).toBe("true"); + expect(mocks.openProjectPath).not.toHaveBeenCalled(); + + await act(async () => card?.dispatchEvent(new MouseEvent("dblclick", { bubbles: true }))); + expect(mocks.openProjectPath).toHaveBeenCalledTimes(1); + expect(mocks.openProjectPath).toHaveBeenLastCalledWith(PROJECT_PATH); + }); +}); diff --git a/web/src/components/home/HomeView.test.tsx b/web/src/components/home/HomeView.test.tsx new file mode 100644 index 00000000..8066d948 --- /dev/null +++ b/web/src/components/home/HomeView.test.tsx @@ -0,0 +1,247 @@ +// @vitest-environment happy-dom + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + newProjectAndEnter: vi.fn(), + openProjectPath: vi.fn(), + openProjectViaDialog: vi.fn(), + openSampleProject: vi.fn(), +})); + +vi.mock("../../i18n", () => ({ useT: () => (key: string) => key })); +vi.mock("../../lib/api", () => ({ isTauri: false })); +vi.mock("../../lib/asset", () => ({ assetUrl: (path: string | null) => path })); +vi.mock("../../store/projectActions", () => mocks); + +import { useRecentStore } from "../../store/recentStore"; +import { + HOME_NOTICE_STORAGE_KEY, + HOME_NOTICE_VERSION, + HomeView, +} from "./HomeView"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }) + .IS_REACT_ACT_ENVIRONMENT = true; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.clearAllMocks(); + for (const mock of Object.values(mocks)) mock.mockResolvedValue(undefined); + localStorage.clear(); + localStorage.setItem(HOME_NOTICE_STORAGE_KEY, HOME_NOTICE_VERSION); + useRecentStore.setState({ + recents: [{ path: "/tmp/Existing.opentake", name: "Existing", openedAt: 1 }], + }); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +it("new_open_sample_register_only_after_success_and_route_tutorial", async () => { + let finish!: () => void; + mocks.openSampleProject.mockReturnValueOnce(new Promise((resolve) => { + finish = resolve; + })); + await act(async () => root.render()); + + const tutorial = [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.sampleTutorial"); + expect(tutorial).toBeDefined(); + + await act(async () => tutorial?.click()); + expect(mocks.openSampleProject).toHaveBeenCalledWith("quick-tutorial", true); + expect(useRecentStore.getState().recents.map(({ name }) => name)).toEqual(["Existing"]); + expect(tutorial?.disabled).toBe(true); + + await act(async () => finish()); + await vi.waitFor(() => expect(tutorial?.disabled).toBe(false)); + expect(useRecentStore.getState().recents.map(({ name }) => name)).toEqual(["Existing"]); +}); + +it("project_card_renders_thumbnail_relative_time_and_missing_state", async () => { + useRecentStore.setState({ + recents: [{ + path: "/tmp/Metadata.opentake", + name: "Metadata", + openedAt: Date.now() - 86_400_000, + modifiedAt: Date.now(), + thumbnailPath: "/tmp/Metadata.opentake/thumbnail.jpg", + missing: false, + }], + }); + await act(async () => root.render()); + + expect(container.querySelector("img")?.src).toContain("thumbnail.jpg"); + expect(container.textContent).toContain("home.relative.today"); + + act(() => useRecentStore.setState({ + recents: [{ + path: "/tmp/Metadata.opentake", + name: "Metadata", + openedAt: Date.now() - 86_400_000, + modifiedAt: Date.now(), + thumbnailPath: "/tmp/Metadata.opentake/thumbnail.jpg", + missing: true, + }], + })); + await vi.waitFor(() => expect(container.textContent).toContain("home.fileMissing")); + expect(container.querySelector("img")).toBeNull(); +}); + +it("missing_card_reveal_remove_and_trash_states", async () => { + const reveal = vi.fn().mockResolvedValue(undefined); + const trash = vi + .fn() + .mockRejectedValueOnce(new Error("permission denied")) + .mockImplementationOnce(async () => { + useRecentStore.setState({ recents: [] }); + }); + const remove = vi + .fn() + .mockRejectedValueOnce(new Error("registry is read-only")) + .mockImplementationOnce(async () => { + useRecentStore.setState({ recents: [] }); + }); + useRecentStore.setState({ + recents: [{ + path: "/tmp/Missing.opentake", + name: "Missing", + openedAt: 1, + missing: true, + }], + reveal, + trash, + remove, + }); + await act(async () => root.render()); + + expect(container.textContent).toContain("home.fileMissing"); + await act(async () => container.querySelector( + "button[aria-label='home.projectActions']", + )?.click()); + expect(container.textContent).toContain("home.revealInFinder"); + expect(container.textContent).toContain("home.removeFromRecents"); + expect(container.textContent).toContain("home.moveToTrash"); + + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent?.includes("home.revealInFinder"))?.click()); + expect(reveal).toHaveBeenCalledWith("/tmp/Missing.opentake"); + + await act(async () => container.querySelector( + "button[aria-label='home.projectActions']", + )?.click()); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent?.includes("home.moveToTrash"))?.click()); + expect(container.textContent).toContain("home.confirmTrashBody"); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent?.includes("home.moveToTrash"))?.click()); + await vi.waitFor(() => expect(container.textContent).toContain("home.trashFailed")); + expect(useRecentStore.getState().recents).toHaveLength(1); + + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent?.includes("home.moveToTrash"))?.click()); + await vi.waitFor(() => expect(useRecentStore.getState().recents).toEqual([])); + + act(() => useRecentStore.setState({ + recents: [{ + path: "/tmp/Missing.opentake", + name: "Missing", + openedAt: 1, + missing: true, + }], + })); + await vi.waitFor(() => expect(container.textContent).toContain("home.fileMissing")); + await act(async () => container.querySelector( + "button[aria-label='home.projectActions']", + )?.click()); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.removeFromRecents")?.click()); + await vi.waitFor(() => expect(container.textContent).toContain("home.removeFailed")); + expect(useRecentStore.getState().recents).toHaveLength(1); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.removeFromRecents")?.click()); + await vi.waitFor(() => expect(remove).toHaveBeenCalledTimes(2)); + expect(useRecentStore.getState().recents).toEqual([]); +}); + +it("upstream_home_children_close_one_composite_acceptance", async () => { + localStorage.removeItem(HOME_NOTICE_STORAGE_KEY); + useRecentStore.setState({ recents: [] }); + await act(async () => root.render()); + + expect(container.querySelector("aside")).not.toBeNull(); + expect(container.textContent).toContain("home.samples"); + expect(container.textContent).toContain("home.sampleDemo"); + expect(container.textContent).toContain("home.welcome"); + expect(container.querySelector('[role="dialog"][aria-modal="true"]')?.textContent).toContain( + "home.welcomeOverlayTitle", + ); + const welcomeDismiss = [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.welcomeOverlayStart"); + expect(document.activeElement).toBe(welcomeDismiss); + await act(async () => window.dispatchEvent( + new KeyboardEvent("keydown", { key: "Escape", bubbles: true }), + )); + expect(localStorage.getItem(HOME_NOTICE_STORAGE_KEY)).toBe(HOME_NOTICE_VERSION); + expect(container.querySelector('[role="dialog"][aria-modal="true"]')).toBeNull(); + + await act(async () => root.unmount()); + container.replaceChildren(); + root = createRoot(container); + localStorage.setItem(HOME_NOTICE_STORAGE_KEY, "0.9.0"); + const reveal = vi.fn().mockResolvedValue(undefined); + const trash = vi.fn().mockResolvedValue(undefined); + useRecentStore.setState({ + recents: [ + { path: "/tmp/Existing.opentake", name: "Existing", openedAt: Date.now() }, + { path: "/tmp/Missing.opentake", name: "Missing", openedAt: 1, missing: true }, + ], + reveal, + trash, + }); + await act(async () => root.render()); + + const updateDialog = container.querySelector('[role="dialog"][aria-modal="true"]'); + expect(updateDialog?.textContent).toContain("home.newInVersion"); + expect(updateDialog?.textContent).toContain("home.updateOverlayBody"); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.updateOverlayDismiss")?.click()); + + expect(container.querySelector('button[aria-label="Existing"]')).not.toBeNull(); + const missingCard = container.querySelector( + 'button[aria-label="Missing · home.fileMissing"]', + ); + expect(missingCard).not.toBeNull(); + await act(async () => missingCard?.parentElement?.dispatchEvent( + new MouseEvent("contextmenu", { bubbles: true }), + )); + expect(container.textContent).toContain("home.revealInFinder"); + expect(container.textContent).toContain("home.moveToTrash"); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent?.includes("home.moveToTrash"))?.click()); + expect(container.textContent).toContain("home.confirmTrashBody"); + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent === "common.cancel")?.click()); + expect(trash).not.toHaveBeenCalled(); + + const existingCard = container.querySelector('button[aria-label="Existing"]')!; + await act(async () => existingCard.focus()); + await act(async () => existingCard.dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true }), + )); + expect(mocks.openProjectPath).toHaveBeenCalledWith("/tmp/Existing.opentake"); + + await act(async () => [...container.querySelectorAll("button")] + .find((button) => button.textContent === "home.sampleDemo")?.click()); + expect(mocks.openSampleProject).toHaveBeenCalledWith("product-demo", false); +}); diff --git a/web/src/components/home/HomeView.tsx b/web/src/components/home/HomeView.tsx index e695de5b..d4e15f06 100644 --- a/web/src/components/home/HomeView.tsx +++ b/web/src/components/home/HomeView.tsx @@ -1,15 +1,47 @@ -import { useEffect, useState, type CSSProperties } from "react"; -import { Plus, FolderOpen, Settings as SettingsIcon, Film, Trash2, Library } from "lucide-react"; +import { useEffect, useRef, useState, type CSSProperties } from "react"; +import { + Plus, + FolderOpen, + Settings as SettingsIcon, + Film, + Trash2, + Library, + MoreHorizontal, + Sparkles, +} from "lucide-react"; import { Icon } from "../ui/Icon"; -import { useT } from "../../i18n"; +import { useT, type TFunction } from "../../i18n"; +import { assetUrl } from "../../lib/asset"; import { useEditorUiStore } from "../../store/uiStore"; import { useRecentStore, type RecentProject } from "../../store/recentStore"; import { newProjectAndEnter, + openSampleProject, openProjectViaDialog, openProjectPath, } from "../../store/projectActions"; +export function formatProjectRelativeTime( + t: TFunction, + timestamp: number, + now = Date.now(), +): string { + const then = new Date(timestamp); + const current = new Date(now); + const thenDay = new Date(then.getFullYear(), then.getMonth(), then.getDate()).getTime(); + const currentDay = new Date( + current.getFullYear(), + current.getMonth(), + current.getDate(), + ).getTime(); + const days = Math.max(0, Math.round((currentDay - thenDay) / 86_400_000)); + if (days === 0) return t("home.relative.today"); + if (days === 1) return t("home.relative.yesterday"); + if (days < 7) return t("home.relative.daysAgo", { count: days }); + if (days < 35) return t("home.relative.weeksAgo", { count: Math.floor(days / 7) }); + return t("home.relative.monthsAgo", { count: Math.max(1, Math.floor(days / 30)) }); +} + const homeShellStyle: CSSProperties = { display: "flex", height: "100%", @@ -48,8 +80,68 @@ const homeWorkspaceStyle: CSSProperties = { const subtleTransition = "background-color var(--anim-hover) var(--ease-out), border-color var(--anim-hover) var(--ease-out), color var(--anim-hover) var(--ease-out)"; +type ProjectAction = "new" | "open" | "sample" | null; + +export const HOME_NOTICE_STORAGE_KEY = "opentake.home.lastSeenVersion"; +export const HOME_NOTICE_VERSION = __APP_VERSION__; + +type HomeNotice = "welcome" | "whatsNew" | null; + +export function resolveHomeNotice( + lastSeenVersion: string | null, + hasRecentProjects: boolean, +): Exclude | null { + if (lastSeenVersion === HOME_NOTICE_VERSION) return null; + return lastSeenVersion === null && !hasRecentProjects ? "welcome" : "whatsNew"; +} + +function loadHomeNotice(hasRecentProjects: boolean): HomeNotice { + try { + return resolveHomeNotice(localStorage.getItem(HOME_NOTICE_STORAGE_KEY), hasRecentProjects); + } catch { + return hasRecentProjects ? "whatsNew" : "welcome"; + } +} + +function persistHomeNoticeSeen() { + try { + localStorage.setItem(HOME_NOTICE_STORAGE_KEY, HOME_NOTICE_VERSION); + } catch { + // Storage can be unavailable in privacy-restricted browser previews. The + // notice still dismisses for this session; the editor remains usable. + } +} + export function HomeView() { const recents = useRecentStore((s) => s.recents); + const [projectAction, setProjectAction] = useState(null); + const [homeNotice, setHomeNotice] = useState(() => + loadHomeNotice(recents.length > 0), + ); + const projectActionRef = useRef(null); + + const dismissHomeNotice = () => { + persistHomeNoticeSeen(); + setHomeNotice(null); + }; + + const runProjectAction = async ( + action: Exclude, + operation: () => Promise, + ) => { + if (projectActionRef.current) return; + projectActionRef.current = action; + setProjectAction(action); + try { + await operation(); + } catch { + // Project actions own their localized error toast; + // keep the UI gesture handled so a native rejection is not unhandled. + } finally { + projectActionRef.current = null; + setProjectAction(null); + } + }; // Validate recent projects on mount to filter out folders deleted on disk useEffect(() => { @@ -58,30 +150,188 @@ export function HomeView() { return (
- + void runProjectAction("new", newProjectAndEnter)} + onOpen={() => void runProjectAction("open", openProjectViaDialog)} + />
- {recents.length === 0 ? : } + + void runProjectAction("sample", () => openSampleProject(slug, tutorial)) + } + /> + {recents.length === 0 ? ( + void runProjectAction("new", newProjectAndEnter)} + onOpen={() => void runProjectAction("open", openProjectViaDialog)} + /> + ) : ( + void runProjectAction("new", newProjectAndEnter)} + onOpen={() => void runProjectAction("open", openProjectViaDialog)} + onOpenPath={(path) => + void runProjectAction("open", () => openProjectPath(path)) + } + /> + )}
+ {homeNotice && }
); } -function Sidebar() { +function HomeNoticeDialog({ + kind, + onDismiss, +}: { + kind: Exclude; + onDismiss: () => void; +}) { + const t = useT(); + const buttonRef = useRef(null); + + useEffect(() => { + buttonRef.current?.focus(); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + event.preventDefault(); + onDismiss(); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [onDismiss]); + + const isWelcome = kind === "welcome"; + return ( +
+
+ +

+ {isWelcome + ? t("home.welcomeOverlayTitle") + : t("home.newInVersion", { version: HOME_NOTICE_VERSION })} +

+

+ {isWelcome ? t("home.welcomeOverlayBody") : t("home.updateOverlayBody")} +

+ +
+
+ ); +} + +const SAMPLE_PROJECTS = [ + { slug: "product-demo", label: "home.sampleDemo", tutorial: false }, + { slug: "quick-tutorial", label: "home.sampleTutorial", tutorial: true }, + { slug: "template-project", label: "home.sampleTemplate", tutorial: false }, +] as const; + +function SampleProjectsStrip({ + busy, + onOpen, +}: { + busy: boolean; + onOpen: (slug: string, tutorial: boolean) => void; +}) { + const t = useT(); + return ( +
+

+ {t("home.samples")} +

+
+ {SAMPLE_PROJECTS.map((sample) => ( + + ))} +
+
+ ); +} + +function Sidebar({ + projectAction, + onNew, + onOpen, +}: { + projectAction: ProjectAction; + onNew: () => void; + onOpen: () => void; +}) { const t = useT(); const setView = useEditorUiStore((s) => s.setView); const setSettingsOpen = useEditorUiStore((s) => s.setSettingsOpen); - const [opening, setOpening] = useState(false); - - const handleOpen = async () => { - setOpening(true); - try { - await openProjectViaDialog(); - } finally { - setOpening(false); - } - }; + const busy = projectAction !== null; return (