diff --git a/MANIFEST.in b/MANIFEST.in index d5944a9b..35f8afd0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,7 +5,7 @@ include NOTICE THIRD_PARTY_NOTICES.md CONTRIBUTING.md SECURITY.md CITATION.cff recursive-include example_workflows *.json recursive-include web *.js *.css include web/vendor/three/LICENSE -recursive-include web/assets *.svg *.png *.webp +include web/assets/*.svg web/assets/*.png web/assets/*.webp recursive-include web/howto *.png recursive-include web/fonts *.woff2 recursive-include web/tutorial *.webp *.mp4 @@ -30,3 +30,39 @@ prune .github exclude AGENTS.md CHANGELOG.md include freevideo_source_bootstrap.py freevideo_ed25519.py freevideo_update_keys.py + +# Bundled feature examples; every file is registered explicitly. +include web/assets/tour/anime-first.webp +include web/assets/tour/anime-last.webp +include web/assets/tour/c1-s2.mp4 +include web/assets/tour/c1-s2.webp +include web/assets/tour/c2-product.webp +include web/assets/tour/c2-whitemodel.webp +include web/assets/tour/c2g-s3.mp4 +include web/assets/tour/c2g-s3.webp +include web/assets/tour/c3-s1.mp4 +include web/assets/tour/c3-s1.webp +include web/assets/tour/c4-high.mp4 +include web/assets/tour/c4-high.webp +include web/assets/tour/c4-light.mp4 +include web/assets/tour/c4-light.webp +include web/assets/tour/c4-max.mp4 +include web/assets/tour/c4-max.webp +include web/assets/tour/c4-medium.mp4 +include web/assets/tour/c4-medium.webp +include web/assets/tour/c5-forest-s1.mp4 +include web/assets/tour/c5-forest-s1.webp +include web/assets/tour/c5-snow-s1.mp4 +include web/assets/tour/c5-snow-s1.webp +include web/assets/tour/c6-s1.mp4 +include web/assets/tour/c6-s1.webp +include web/assets/tour/c7-s1.mp4 +include web/assets/tour/c7-s1.webp +include web/assets/tour/c7-src-s1.mp4 +include web/assets/tour/c7-src-s1.webp +include web/assets/tour/c8-original.webp +include web/assets/tour/c8-s1.mp4 +include web/assets/tour/c8-s1.webp +include web/assets/tour/c9-s1.mp4 +include web/assets/tour/c9-s1.webp +include web/assets/tour/fox-sheet.webp diff --git a/__init__.py b/__init__.py index 2ce8c69b..69298cab 100644 --- a/__init__.py +++ b/__init__.py @@ -3,6 +3,14 @@ async def comfy_entrypoint(): + from .freevideo_engine.comfy_environment import control_process + from .freevideo_engine.gpu_pool import pool_enabled + if control_process() and pool_enabled(): + from server import PromptServer + from .freevideo_engine import comfy_bridge + from .freevideo_engine.comfy_pool import register_pool + await register_pool(PromptServer.instance, root=comfy_bridge.installation_root(), + source=comfy_bridge.source_root()) from .freevideo_engine.comfy_progress import register as progress_routes progress_routes() try: @@ -41,6 +49,8 @@ async def comfy_entrypoint(): subtitle_file_routes() from .freevideo_engine.comfy_share import register as share_routes share_routes() + from .freevideo_engine.swap_lora import register as swap_lora_routes + swap_lora_routes() from .freevideo_engine.comfy_metadata import register as workflow_backfill workflow_backfill() try: diff --git a/docs/Linux.md b/docs/Linux.md index 426fee6e..e2c574fd 100644 --- a/docs/Linux.md +++ b/docs/Linux.md @@ -10,7 +10,7 @@ FreeVideo runs on x86_64 Linux with an NVIDIA GPU and driver 580 or newer. On a 2. Make it executable, either in the file's properties (**Allow executing file as program**) or with `chmod +x FreeVideo-Linux-x86_64.AppImage`, then open it. 3. Choose a folder for a new installation, or an existing ComfyUI, and click **Install & launch**. ComfyUI opens in the browser with the FreeVideo workspace. -FreeVideo then appears in the application menu and on the desktop, and later versions update the launcher by themselves, as on Windows. The AppImage needs a distribution from 2022 or newer (Ubuntu 22.04, Debian 12, Fedora 36, RHEL 9 or later); on older systems, use the terminal installation below. If the computer lacks the compiler FreeVideo builds its GPU acceleration with, the installation plan shows the command that installs it. +FreeVideo then appears in the application menu and on the desktop, and later versions update the launcher by themselves, as on Windows. The AppImage needs a distribution from 2022 or newer (Ubuntu 22.04, Debian 12, Fedora 36, RHEL 9 or later); on older systems, use the terminal installation below. If the computer lacks Git or a C++ compiler, FreeVideo downloads its own copies during the installation, without administrator rights. ## Terminal: one command diff --git a/docs/Linux.zh-CN.md b/docs/Linux.zh-CN.md index 92cd89a7..990bb1c5 100644 --- a/docs/Linux.zh-CN.md +++ b/docs/Linux.zh-CN.md @@ -10,7 +10,7 @@ FreeVideo 支持 x86_64 Linux，需要 NVIDIA 显卡和 580 及以上版本的 2. 在文件属性中勾选**允许作为程序执行**，或运行 `chmod +x FreeVideo-Linux-x86_64.AppImage`，然后打开它。 3. 选择新安装的文件夹或已有的 ComfyUI，点击**安装并启动**。ComfyUI 会在浏览器中打开，并进入 FreeVideo 创作面板。 -安装完成后，应用菜单和桌面上会出现 FreeVideo，启动器也会像 Windows 版一样自动更新。AppImage 需要 2022 年以后的发行版（Ubuntu 22.04、Debian 12、Fedora 36、RHEL 9 或更新版本）；更老的系统请使用下面的终端安装方式。如果系统缺少编译 GPU 加速所需的编译器，安装计划会给出安装它的命令。 +安装完成后，应用菜单和桌面上会出现 FreeVideo，启动器也会像 Windows 版一样自动更新。AppImage 需要 2022 年以后的发行版（Ubuntu 22.04、Debian 12、Fedora 36、RHEL 9 或更新版本）；更老的系统请使用下面的终端安装方式。如果系统缺少 Git 或 C++ 编译器，FreeVideo 会在安装时自行下载，不需要管理员权限。 ## 终端：一条命令 diff --git a/docs/community-results.md b/docs/community-results.md index 16b17660..f80ae50a 100644 --- a/docs/community-results.md +++ b/docs/community-results.md @@ -90,6 +90,39 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r | 5080 | 16 GB | 64 GB | 768×1344 | 3 s | Max | 0.3.20 | 216 s | 0.3.20 | Already optimal | | 4060 Ti | 8 GB | 32 GB | 960×544 | 5 s | Light | 0.4.12 | 333 s | 0.4.12 | Already optimal | | 5070 Ti | 16 GB | 64 GB | 1184×672 | 15 s | Medium | 0.4.13 | 487 s | 0.4.13 | Already optimal | +| 3070 | 16 GB | 32 GB | 768×1344 | 5.2 s | Light | 0.4.14 | 2380 s | 0.4.14 | Already optimal | +| M4 Max | Unified | 128 GB | 768×1344 | 10 s | Light | 0.4.14 | 3444 s | 0.4.14 | Already optimal | +| 4060 Ti | 8 GB | 32 GB | 672×1184 | 10.8 s | Light | 0.4.14 | 936 s | 0.4.14 | Already optimal | +| 4060 Laptop | 8 GB | 16 GB | 1344×768 | 10 s | Light | 0.4.15 | 1147 s | 0.4.15 | Already optimal | +| 3060 | 12 GB | 24 GB | 1344×768 | 13.7 s | Light | 0.4.15 | 1747 s | 0.4.20 | 1700 s (−50 s) | +| 5070 Laptop | 8 GB | 64 GB | 1184×672 | 10 s | Light | 0.4.15 | 822 s | int8 model² | 720 s (−100 s) | +| 5080 | 16 GB | 64 GB | 1024×1024 | 15 s | Light | 0.4.16 | 412 s | 0.4.16 | Already optimal | +| M1 Max | Unified | 64 GB | 960×544 | 5.2 s | Light | 0.4.16 | 1816 s | 0.4.16 | Already optimal | +| 4060 Laptop | 8 GB | 32 GB | 1344×768 | 5.2 s | Light⁵ | 0.4.17 | 470 s | 0.4.17 | Already optimal | +| 3060 | 12 GB | 32 GB | 1344×768 | 10.8 s | Light⁵ | 0.4.17 | 967 s | 0.4.19 | 880 s (−90 s) | +| 4060 Laptop | 8 GB | 32 GB | 1344×768 | 10 s | Max | 0.4.17 | 2311 s | 0.4.17 | Already optimal | +| 5080 | 16 GB | 48 GB | 768×1344 | 12.3 s | Light | 0.4.17 | 293 s | 0.4.17 | Already optimal | +| 3060 | 12 GB | 64 GB | 960×544 | 9.4 s | Light | 0.4.17 | 743 s | 0.4.17 | Already optimal | +| 3060 | 12 GB | 32 GB | 768×1344 | 5.2 s | Light | 0.4.17 | 811 s | 0.4.17 | Already optimal | +| 5070 Ti | 16 GB | 32 GB | 1184×672 | 15 s | Light | 0.4.19 | 447 s | 0.4.19 | Already optimal | +| 4060 | 8 GB | 32 GB | 960×544 | 15 s | Light | 0.4.19 | 1311 s | 0.4.19 | Already optimal | +| M5 | Unified | 24 GB | 1344×768 | 10 s | Max | 0.4.12 | 30509 s | 0.4.14 | 27000 s (−3500 s) | +| 4070 Ti | 12 GB | 64 GB | 960×544 | 10 s | Light | 0.4.19 | 236 s | 0.4.19 | Already optimal | +| 3090 | 24 GB | 128 GB | 1344×768 | 10 s | Light | 0.4.19 | 377 s | 0.4.19 | Already optimal | +| 3060 | 12 GB | 32 GB | 672×1184 | 15 s | Light | 0.4.20 | 1608 s | 0.4.20 | Already optimal | +| M5 Max | Unified | 128 GB | 1664×960 | 3 s | Light | 0.4.18 | 755 s | 0.4.18 | Already optimal | +| 5050 Laptop | 8 GB | 24 GB | 544×960 | 8 s | Light | 0.4.21 | 534 s | 0.4.21 | Already optimal | +| 3080 | 10 GB | 32 GB | 768×1344 | 15 s | Max | 0.4.22 | 1965 s | 0.4.22 | Already optimal | +| 4090 Laptop | 16 GB | 64 GB | 1344×768 | 10 s | Medium | 0.4.14 | 590 s | 0.4.17 | 570 s (−20 s) | +| 4060 Laptop | 8 GB | 32 GB | 960×544 | 10 s | Light | 0.4.6 | 517 s | 0.4.6 | Already optimal | +| 4060 Laptop | 8 GB | 16 GB | 1344×768 | 10 s | Light | 0.4.14 | 1168 s | 0.4.14 | Already optimal | +| 5070 Laptop | 8 GB | 32 GB | 1344×768 | 11.5 s | Light | 0.4.17 | 1003 s | 0.4.24 | 900 s (−100 s) | +| 3070 Ti | 8 GB | 64 GB | 1344×768 | 16.5 s | Light | 0.4.17 | 1623 s | 0.4.24 | 1570 s (−50 s) | +| 3060 Ti | 8 GB | 32 GB | 768×1344 | 10 s | Light | 0.4.18 | 843 s | 0.4.24 | 810 s (−33 s) | +| 5060 | 8 GB | 32 GB | 672×1184 | 11.5 s | Light | 0.4.18 | 707 s | 0.4.24 | 680 s (−27 s) | +| 5070 Ti Laptop | 12 GB | 32 GB | 1344×768 | 10 s | Light | 0.4.23 | 373 s¹³ | 0.4.23 | Thermal‑limited¹ | +| 5060 | 8 GB | 48 GB | 768×1344 | 14 s | Medium | 0.3.25 | 3474 s | 0.3.25 | Already optimal | +| 3060 | 12 GB | 32 GB | 896×1120 | 10 s | Light | 0.3.25 | 1045 s | 0.4.0 | 1025 s (−20 s) | - Time: end to end, from submission until the video is saved. - After: the estimated time, derived from the report's stage times and from complete planned runs on our test machines (RTX 3090 and RTX 5060 Ti on Windows, RTX PRO 6000 on Linux) under the reporting machine's VRAM and RAM limits, not from reruns on that machine. @@ -106,3 +139,4 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r - ¹⁰ GPU shared: other programs were using about 29% of the graphics card during the run, so FreeVideo's steps ran at a little over 70% of their usual speed. With the card free, the same request takes about 7200 s. - ¹¹ With 16 GB of RAM, the text encoder and part of the video model cannot stay in memory and are read from the drive; text encoding and the first pass, which wait on these reads, took about 460 s of the 887 s. - ¹² Reference-video task: the reference video and audio roughly double the sequence the model processes, so each step takes about twice as long as a plain request of the same size. +- ¹³ Preview (first pass only). diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index 40db55d1..0d646c77 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -90,6 +90,39 @@ | 5080 | 16 GB | 64 GB | 768×1344 | 3 s | 极⁠致 | 0.3.20 | 216 s | 0.3.20 | 已⁠是⁠最⁠优 | | 4060 Ti | 8 GB | 32 GB | 960×544 | 5 s | 轻⁠量 | 0.4.12 | 333 s | 0.4.12 | 已⁠是⁠最⁠优 | | 5070 Ti | 16 GB | 64 GB | 1184×672 | 15 s | 标⁠准 | 0.4.13 | 487 s | 0.4.13 | 已⁠是⁠最⁠优 | +| 3070 | 16 GB | 32 GB | 768×1344 | 5.2 s | 轻⁠量 | 0.4.14 | 2380 s | 0.4.14 | 已⁠是⁠最⁠优 | +| M4 Max | 统⁠一⁠内⁠存 | 128 GB | 768×1344 | 10 s | 轻⁠量 | 0.4.14 | 3444 s | 0.4.14 | 已⁠是⁠最⁠优 | +| 4060 Ti | 8 GB | 32 GB | 672×1184 | 10.8 s | 轻⁠量 | 0.4.14 | 936 s | 0.4.14 | 已⁠是⁠最⁠优 | +| 4060 笔⁠记⁠本 | 8 GB | 16 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.15 | 1147 s | 0.4.15 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 24 GB | 1344×768 | 13.7 s | 轻⁠量 | 0.4.15 | 1747 s | 0.4.20 | 1700 s⁠（−50 s⁠） | +| 5070 笔⁠记⁠本 | 8 GB | 64 GB | 1184×672 | 10 s | 轻⁠量 | 0.4.15 | 822 s | int8 模⁠型² | 720 s⁠（−100 s⁠） | +| 5080 | 16 GB | 64 GB | 1024×1024 | 15 s | 轻⁠量 | 0.4.16 | 412 s | 0.4.16 | 已⁠是⁠最⁠优 | +| M1 Max | 统⁠一⁠内⁠存 | 64 GB | 960×544 | 5.2 s | 轻⁠量 | 0.4.16 | 1816 s | 0.4.16 | 已⁠是⁠最⁠优 | +| 4060 笔⁠记⁠本 | 8 GB | 32 GB | 1344×768 | 5.2 s | 轻⁠量⁵ | 0.4.17 | 470 s | 0.4.17 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 32 GB | 1344×768 | 10.8 s | 轻⁠量⁵ | 0.4.17 | 967 s | 0.4.19 | 880 s⁠（−90 s⁠） | +| 4060 笔⁠记⁠本 | 8 GB | 32 GB | 1344×768 | 10 s | 极⁠致 | 0.4.17 | 2311 s | 0.4.17 | 已⁠是⁠最⁠优 | +| 5080 | 16 GB | 48 GB | 768×1344 | 12.3 s | 轻⁠量 | 0.4.17 | 293 s | 0.4.17 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 64 GB | 960×544 | 9.4 s | 轻⁠量 | 0.4.17 | 743 s | 0.4.17 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 32 GB | 768×1344 | 5.2 s | 轻⁠量 | 0.4.17 | 811 s | 0.4.17 | 已⁠是⁠最⁠优 | +| 5070 Ti | 16 GB | 32 GB | 1184×672 | 15 s | 轻⁠量 | 0.4.19 | 447 s | 0.4.19 | 已⁠是⁠最⁠优 | +| 4060 | 8 GB | 32 GB | 960×544 | 15 s | 轻⁠量 | 0.4.19 | 1311 s | 0.4.19 | 已⁠是⁠最⁠优 | +| M5 | 统⁠一⁠内⁠存 | 24 GB | 1344×768 | 10 s | 极⁠致 | 0.4.12 | 30509 s | 0.4.14 | 27000 s⁠（−3500 s⁠） | +| 4070 Ti | 12 GB | 64 GB | 960×544 | 10 s | 轻⁠量 | 0.4.19 | 236 s | 0.4.19 | 已⁠是⁠最⁠优 | +| 3090 | 24 GB | 128 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.19 | 377 s | 0.4.19 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 32 GB | 672×1184 | 15 s | 轻⁠量 | 0.4.20 | 1608 s | 0.4.20 | 已⁠是⁠最⁠优 | +| M5 Max | 统⁠一⁠内⁠存 | 128 GB | 1664×960 | 3 s | 轻⁠量 | 0.4.18 | 755 s | 0.4.18 | 已⁠是⁠最⁠优 | +| 5050 笔⁠记⁠本 | 8 GB | 24 GB | 544×960 | 8 s | 轻⁠量 | 0.4.21 | 534 s | 0.4.21 | 已⁠是⁠最⁠优 | +| 3080 | 10 GB | 32 GB | 768×1344 | 15 s | 极⁠致 | 0.4.22 | 1965 s | 0.4.22 | 已⁠是⁠最⁠优 | +| 4090 笔⁠记⁠本 | 16 GB | 64 GB | 1344×768 | 10 s | 标⁠准 | 0.4.14 | 590 s | 0.4.17 | 570 s⁠（−20 s⁠） | +| 4060 笔⁠记⁠本 | 8 GB | 32 GB | 960×544 | 10 s | 轻⁠量 | 0.4.6 | 517 s | 0.4.6 | 已⁠是⁠最⁠优 | +| 4060 笔⁠记⁠本 | 8 GB | 16 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.14 | 1168 s | 0.4.14 | 已⁠是⁠最⁠优 | +| 5070 笔⁠记⁠本 | 8 GB | 32 GB | 1344×768 | 11.5 s | 轻⁠量 | 0.4.17 | 1003 s | 0.4.24 | 900 s⁠（−100 s⁠） | +| 3070 Ti | 8 GB | 64 GB | 1344×768 | 16.5 s | 轻⁠量 | 0.4.17 | 1623 s | 0.4.24 | 1570 s⁠（−50 s⁠） | +| 3060 Ti | 8 GB | 32 GB | 768×1344 | 10 s | 轻⁠量 | 0.4.18 | 843 s | 0.4.24 | 810 s⁠（−33 s⁠） | +| 5060 | 8 GB | 32 GB | 672×1184 | 11.5 s | 轻⁠量 | 0.4.18 | 707 s | 0.4.24 | 680 s⁠（−27 s⁠） | +| 5070 Ti 笔⁠记⁠本 | 12 GB | 32 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.23 | 373 s¹³ | 0.4.23 | 散⁠热⁠受⁠限¹ | +| 5060 | 8 GB | 48 GB | 768×1344 | 14 s | 标⁠准 | 0.3.25 | 3474 s | 0.3.25 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 32 GB | 896×1120 | 10 s | 轻⁠量 | 0.3.25 | 1045 s | 0.4.0 | 1025 s⁠（−20 s⁠） | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 @@ -106,3 +139,4 @@ - ¹⁰ 显卡被占用：生成过程中其他程序占用了约 29% 的显卡，FreeVideo 每一步只有独占时七成多一点的速度；显卡空出来时，同样的请求约需 7200 秒。 - ¹¹ 16 GB 内存留不下文本编码器和视频模型的一部分，需要从硬盘读取；887 秒里约 460 秒是等待这些读取的文本编码和第一遍。 - ¹² 参考视频任务：参考视频和参考音频使模型处理的序列约增加一倍，每步耗时约为相同尺寸普通请求的两倍。 +- ¹³ 预览（只做第一遍）。 diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py index 5be6329b..71edc9a5 100644 --- a/freevideo_engine/adaptive.py +++ b/freevideo_engine/adaptive.py @@ -712,6 +712,17 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry): from .continuation import validate_hide_tail hidden = validate_hide_tail(metrics.get('hide_tail_frames', 0)) output_frames = expected - hidden + original_audio = metrics.get('original_audio') + if original_audio is not None: + if (not isinstance(original_audio, dict) or original_audio.get('sample_identical') is not True + or type(original_audio.get('samples_per_channel')) is not int + or type(original_audio.get('sample_rate')) is not int + or original_audio['samples_per_channel'] <= 0 or original_audio['sample_rate'] <= 0): + raise ValueError('Original audio export was not verified') + count, rate = original_audio['samples_per_channel'], original_audio['sample_rate'] + output_frames = (count * canvas['fps'] + rate - 1) // rate + if hidden or not 0 < output_frames <= expected: + raise ValueError('Original audio exceeds the generated canvas') if continuation is not None: from .continuation import OVERLAP_FRAMES if (not isinstance(continuation, dict) or continuation.get('overlap_frames') != OVERLAP_FRAMES @@ -739,10 +750,11 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry): rgb_shape, dtype = _retained_array_header(Path(artifacts)/'rgb.npy') if str(dtype) != 'uint8' or rgb_shape != (output_frames, canvas['height'], canvas['width'], 3): raise ValueError('Retained video frames do not match the complete request.') - audio_shape, audio_dtype = _retained_array_header(Path(artifacts)/'audio.npy') - if (str(audio_dtype) != 'float32' or len(audio_shape) != 2 or audio_shape[0] != 2 - or abs(audio_shape[1]/32000. - output_frames/canvas['fps']) > 1/canvas['fps']): - raise ValueError('Retained stereo audio does not match the request duration.') + if original_audio is None: + audio_shape, audio_dtype = _retained_array_header(Path(artifacts)/'audio.npy') + if (str(audio_dtype) != 'float32' or len(audio_shape) != 2 or audio_shape[0] != 2 + or abs(audio_shape[1]/32000. - output_frames/canvas['fps']) > 1/canvas['fps']): + raise ValueError('Retained stereo audio does not match the request duration.') import av with av.open(str(output)) as container: video, audio = container.streams.video, container.streams.audio @@ -752,8 +764,13 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry): # A request rendered at a larger size is saved at the size asked for. shown = metrics.get('output_size') or canvas if ((stream.width, stream.height, stream.frames) != (shown['width'],shown['height'],output_frames) - or float(stream.average_rate) != canvas['fps'] or audio[0].codec_context.channels != 2): + or (original_audio is None and float(stream.average_rate) != canvas['fps']) + or audio[0].codec_context.channels != (original_audio['channels'] if original_audio else 2)): raise ValueError('MP4 metadata does not match the requested output.') + if original_audio is not None: + # A skipped audio decode is not an ordinary request timing observation; + # retain the existing conservative planner budgets. + return None if metrics.get('sampling_reused') is True: # The output is validated, but these observations cover only a decoder # retry. Do not teach placement/ETA that a full request cost this little diff --git a/freevideo_engine/admission.py b/freevideo_engine/admission.py new file mode 100644 index 00000000..0fd0cc8e --- /dev/null +++ b/freevideo_engine/admission.py @@ -0,0 +1,86 @@ +"""Atomic host-memory admission for the central, in-process GPU queue.""" +from dataclasses import dataclass +from threading import Lock + + +def _bytes(value): + return type(value) is int and value >= 0 + + +@dataclass(frozen=True) +class Envelope: + """Future whole-task peaks supplied by the resource planner. + + host_ram_bytes includes resident pinned storage. commit_bytes includes + non-pinned host and future GPU/WDDM commit; pinned_bytes is charged to + commit separately, exactly once. These are peaks, not planner free budgets. + """ + host_ram_bytes: int + commit_bytes: int + pinned_bytes: int + + def __post_init__(self): + if not all(_bytes(value) for value in + (self.host_ram_bytes, self.commit_bytes, self.pinned_bytes)): + raise ValueError('Envelope sizes must be nonnegative integer bytes') + if self.pinned_bytes > self.host_ram_bytes: + raise ValueError('Pinned storage must be included in the host RAM peak') + + +@dataclass(frozen=True) +class HostMemory: + """Fresh usable headroom after system reserves and applicable host caps. + + The probe must bound physical RAM by cgroup/user limits and supply Windows + commit headroom separately. A platform without a commit counter may use + its usable physical headroom as a conservative commit bound. Missing or + invalid measurements deny admission; swap never extends physical RAM. + """ + free_ram_bytes: int + commit_headroom_bytes: int + + +class AdmissionLedger: + """Reserve full peaks until confirmed exit; do not infer allocation credit. + + The two-counter probe cannot attribute live allocations to tasks. Keeping + their full envelopes against live free memory can underutilize the host, + but never credits another task's working set or idle cache. The caller + owns probe freshness, system reserves and the single central coordinator; + this lock is not a cross-process or cross-installation resource lease. + """ + + def __init__(self, probe): + self._probe = probe + self._lock = Lock() + self._reservations = {} + + def try_reserve(self, task_id, envelope) -> bool: + if not isinstance(envelope, Envelope): + raise TypeError('Admission requires a complete Envelope') + with self._lock: + previous = self._reservations.get(task_id) + if previous is not None: + if previous != envelope: + raise ValueError('Release a task before changing its envelope') + return True + try: + memory = self._probe() + free, commit = memory.free_ram_bytes, memory.commit_headroom_bytes + except (OSError, RuntimeError, ValueError, TypeError, AttributeError): + return False + if not _bytes(free) or not _bytes(commit): + return False + ram_need = envelope.host_ram_bytes + commit_need = envelope.commit_bytes + envelope.pinned_bytes + for reserved in self._reservations.values(): + ram_need += reserved.host_ram_bytes + commit_need += reserved.commit_bytes + reserved.pinned_bytes + if ram_need > free or commit_need > commit: + return False + self._reservations[task_id] = envelope + return True + + def release(self, task_id) -> None: + with self._lock: + self._reservations.pop(task_id, None) diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.py index 96deda12..77fbddd2 100644 --- a/freevideo_engine/agent/comfy.py +++ b/freevideo_engine/agent/comfy.py @@ -161,7 +161,13 @@ def walk(value, pointer=''): raise specs.SpecError(errors) # Queue --------------------------------------------------------------------------------------------- + def scheduler(self): + return getattr(self.server, '_freevideo_front', None) + def snapshot(self): + if self.scheduler() is not None: + running, pending = self.scheduler().current_queue() + return [row[1] for row in running], [row[1] for row in pending] running, pending = self.server.prompt_queue.get_current_queue() return [row[1] for row in running], [row[1] for row in sorted(pending, key=lambda r: r[0])] @@ -186,18 +192,25 @@ def position(self, prompt_id): return pending.index(prompt_id) + 1 if prompt_id in pending else None def history(self, prompt_id): + if self.scheduler() is not None: + return self.scheduler().history(prompt_id).get(prompt_id) return self.server.prompt_queue.get_history(prompt_id=prompt_id).get(prompt_id) - def progress(self, node): + def progress(self, node, prompt_id=None): from ..comfy_progress import STATE with STATE.lock: - row = STATE.rows.get(str(node)) + key = (str(prompt_id), str(node)) if self.scheduler() is not None else str(node) + row = STATE.rows.get(key) if not row: return None # How old the message is, as a reopened page gets it, so a bar can go on from where the engine is now. return dict(json.loads(json.dumps(row['message'])), age_seconds=max(0., STATE.clock() - row['measured'])) async def submit(self, graph, extra): + if self.scheduler() is not None: + reply = await self.scheduler().submit(dict(prompt=graph, extra_data=extra, + client_id=extra.get('client_id', ''))) + return reply['prompt_id'] if self.handler is None: for route in self.server.routes: if getattr(route, 'method', None) == 'POST' and getattr(route, 'path', None) == '/prompt': @@ -236,6 +249,8 @@ async def post_prompt(self, value): return response.status, await response.json(content_type=None) def cancel(self, prompt_id, node): + if self.scheduler() is not None: + return self.scheduler().cancel(prompt_id, node) import nodes from ..comfy_progress import cancel_request return cancel_request(self.server.prompt_queue, prompt_id, node, nodes.interrupt_processing) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index a61ea514..a2d26c87 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -128,6 +128,10 @@ 'without studio prompts. POST /jobs/{id}/cancel cancels only this job.', 'POST /jobs/{id}/resume continues failed, interrupted or cancelled generation; successful blocks stay. ' 'refine_from is the separate preview second pass. Active or succeeded jobs cannot resume.', + 'After a video the model stays on the graphics card and in memory, so the next ' + 'video starts quickly; it goes at once when another program needs the memory. An agent that runs its own ' + 'model on the same card between videos (for example to check a result) calls POST /gpu/release (MCP freevideo_release_gpu) first ' + '(it answers released=false while a video is running), or sets release_gpu_after=true on the generate job.', 'Results include the joined library_id and each block library_id/report_id. Fetch videos and thumbnails ' 'with the token at /library/{day}/{run}/file and /thumbnail. MCP freevideo_asset and CLI agent asset use HTTP.', ], @@ -309,6 +313,7 @@ ('GET', '/jobs/{id}'): '查看任务进度和结果', ('GET', '/jobs/{id}/events'): '持续接收任务进度', ('POST', '/jobs/{id}/cancel'): '取消任务', + ('POST', '/gpu/release'): '空闲时释放显存和内存里的视频模型', ('POST', '/jobs/{id}/resume'): '接着生成一条视频里还没完成的部分', ('POST', '/media'): '上传图片、视频或音频内容', ('POST', '/imports'): '导入视频或音频到剪辑时间线；轮询 GET /imports 等待就绪', diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 80bd5db2..b1930b27 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -713,6 +713,13 @@ def finish(self, row, status, *, result=None, error=None, save=True): elif save: self.retry_pending() return + if row['kind'] == 'generate' and row.get('spec', {}).get('release_gpu_after') is True: + # The agent runs another model on this card next, so the card is free before the job reads as done. + from ..resident_process import OWNER + try: + OWNER.release_idle('agent_requested') + except Exception: + pass row.update(status=status, finished_at=self.clock(), updated_at=self.clock()) if result is not None: row['result'] = result @@ -1020,6 +1027,15 @@ def runner_busy(self): return any(r['state'] in ('queued', 'running', 'cancelling') or r['output_state'] == 'joining' for r in runner.records.values()) + def scheduler(self): + get = getattr(self.port, 'scheduler', None) + return get() if callable(get) else None + + def prompt_progress(self, node, prompt_id): + if self.scheduler() is not None: + return self.port.progress(node, prompt_id=prompt_id) + return self.port.progress(node) + def resume(self, identity, *, key=None, value=None, replay_result=False, client=None): value = value or {} schema.validate(schema.contracts()['POST', '/jobs/{id}/resume']['body'], value) @@ -1088,7 +1104,7 @@ def queue_view(self): entries, chains, prompts = [], [], {} if runner is not None: with runner.lock: - order = ([runner.active] if runner.active is not None else []) + runner.queue + order = runner.active_ids() + runner.queue order += [r['id'] for r in runner.records.values() if r['id'] not in order] records = [runner.records[identity] for identity in dict.fromkeys(order)] for record in records: @@ -1221,7 +1237,7 @@ def progress(self, row): position = self.port.position(row['comfy']['prompt_id']) if position: return dict(phase='queued', label='Waiting in the ComfyUI queue', queue_position=position) - message = self.port.progress(row['comfy']['node']) + message = self.prompt_progress(row['comfy']['node'], row['comfy']['prompt_id']) if not message: return dict(phase='starting', label='Starting') overall = message.get('overall') or {} @@ -1248,9 +1264,10 @@ def estimate(self, row): if row.get('chain_id'): # A video in several parts has one bar over all of them; the engine times each part by itself. chain = self.chain(row['chain_id']) - message = self.port.progress(chain['node']) or {} if chain['count'] == 1 and chain.get('node') else {} + current = next((b for b in chain['blocks'] if b['id'] == chain.get('current')), {}) + message = self.prompt_progress(chain['node'], current.get('prompt_id')) or {} if chain['count'] == 1 and chain.get('node') else {} elif row['kind'] == 'generate' and row.get('comfy'): - message = self.port.progress(row['comfy']['node']) or {} + message = self.prompt_progress(row['comfy']['node'], row['comfy']['prompt_id']) or {} else: return None except Exception: @@ -1336,11 +1353,16 @@ def _tick(self): if self.ready(row) != 'ready': continue if kind == 'generate': - if generation_ahead or vlm_ahead or self.port.vlm_busy() or self.port.image_busy(): + scheduler = self.scheduler() + if (scheduler is None and generation_ahead or vlm_ahead + or self.port.vlm_busy() or self.port.image_busy()): generation_ahead = True continue generation_ahead = True - if not row['spec'].get('refine_from') and (self.runner_busy() or self.port.queue_busy()): + if scheduler is not None: + if not scheduler.accepting(): + continue + elif not row['spec'].get('refine_from') and (self.runner_busy() or self.port.queue_busy()): continue elif kind in GPU_KINDS: chain_busy = self.runner_busy() @@ -1620,8 +1642,11 @@ def start_chain(self, row, *, payload=None, resume=False): with self.lock, runner.lock: if row.get('_cancel') or row['status'] == 'cancelled': return - if (any(r['state'] in ('queued', 'running', 'cancelling') or r['output_state'] == 'joining' - for r in runner.records.values()) or self.port.queue_busy() or self.port.vlm_busy() or self.port.image_busy()): + scheduler = self.scheduler() + if (scheduler is None and (any(r['state'] in ('queued', 'running', 'cancelling') or r['output_state'] == 'joining' + for r in runner.records.values()) or self.port.queue_busy()) + or scheduler is not None and not scheduler.accepting() + or self.port.vlm_busy() or self.port.image_busy()): return # Submit and plan refuse this up front; the backend can still change before a queued or resumed job starts. if (any(b['state'] != 'done' and (b['after'] or b['before']) @@ -1831,7 +1856,7 @@ def generation_failure(self, row, chain): except Exception as error: self.log_poll_failure(row, error) try: - report_id = (self.port.progress(chain['node']) or {}).get('report_id') + report_id = (self.prompt_progress(chain['node'], (block or {}).get('prompt_id')) or {}).get('report_id') except Exception: report_id = None code = reason.get('code', 'generation_failed') diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index 1b2ae34f..13c60877 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -66,6 +66,7 @@ def tool_list(): dict(name='freevideo_ask', title='Ask the local VLM', description='Ask FreeVideo\'s local vision-language model about images or sampled frames of a video (e.g. "Does this shot match its description? Answer as JSON {score, issues}").', annotations=write), dict(name='freevideo_enhance_prompt', title='Enhance a prompt', description='Rewrite a rough idea (any language) into a complete H3 prompt with the local VLM. Images become references.', annotations=write), dict(name='freevideo_assemble', title='Assemble clips', description='Join finished clips (job ids, saved video ids or files) into one video with cuts or crossfades; trims with start/end.', annotations=write), + dict(name='freevideo_release_gpu', title='Release the card', description='Free the graphics card and memory the video model holds; call before running another model. released=false while a video runs.', annotations=write), dict(name='freevideo_library', title='Saved videos', description='Saved FreeVideo videos, newest first, with prompts and the agent job that made them.', annotations=read)] from .schema import DRAMA_OPERATIONS rows.extend(dict(name='freevideo_drama_' + name, title='Film: ' + name, description=summary, @@ -415,6 +416,8 @@ def tool(self, identity, name, args, token): if name == 'freevideo_edit_image': made = c.edit(args['source'], args.get('recipe'), document=args.get('document'), target=args.get('target')) return made, self.picture(made['view']) + if name == 'freevideo_release_gpu': + return c.post('/gpu/release', {}), [] if name == 'freevideo_library': return c.library(**args), [] if name == 'freevideo_look': diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 28ff6608..65c92c0a 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -178,7 +178,7 @@ def features(continuation, vlm): return {'capabilities': True, 'plan': True, 'jobs.cancel': True, 'jobs.resume': True, 'takes': True, 'library.read': True, 'library.download': True, 'imports.create': True, 'imports.read': True, 'errors.validation': True, 'errors.missing_media': True, - 'generate.text': True, 'generate.size': True, 'generate.first_frame': True, 'generate.seed': True, + 'gpu.release': True, 'generate.text': True, 'generate.size': True, 'generate.first_frame': True, 'generate.seed': True, 'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation), 'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules, 'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True, @@ -1221,6 +1221,20 @@ def write(): request_id=errors.request_id())}, 409) return reply(done[1], done[0]) + @routes.post('/freevideo/agent/chat/rename') + @guarded + async def chat_rename(request): + value = await body(request) + def write(): + store = work_sessions() + sid = chat_session(store, value['session']) + if sid is None: + raise JobError('not_found', 'No conversation') + # An empty name gives the conversation back the first thing the user said. + store.append(sid, dict(type='control', action='renamed', title=value['title'].strip()), by='user') + return dict(session=sid, title=next((r['title'] for r in store.history(chat['client'], limit=None) if r['id'] == sid), None)) + return reply(await asyncio.to_thread(write)) + @routes.post('/freevideo/agent/chat/delete') @guarded async def chat_delete(request): @@ -1464,6 +1478,12 @@ async def get_job(request): row = await asyncio.to_thread(jobs.get, request.match_info['id']) return reply(job_answer(await wait_for_job(jobs, row, float(request[query_key].get('wait', '0') or 0)))) + @routes.post(PREFIX + '/gpu/release') + @guarded + async def gpu_release(request): + from ..resident_process import OWNER + return reply(await asyncio.to_thread(OWNER.release_idle, 'agent_requested')) + @routes.post(PREFIX + '/jobs/{id}/cancel') @guarded async def cancel(request): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 066a752c..21cd016c 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -186,6 +186,7 @@ def generate(*, local=False): obj(dict(name=string(512, pattern=r'^.+\.[sS][aA][fF][eE][tT][eE][nN][sS][oO][rR][sS]$'), strength=number(-4, 4, default=1)), ('name',))]}, default=[]), preview={'type': 'boolean', 'default': False}, + release_gpu_after={'type': 'boolean', 'default': False}, refine_from=nullable({'oneOf': [JOB, SAVED]}), force_regenerate={'type': 'boolean', 'default': False}, continue_from={'oneOf': [obj({'from_job': JOB}, ('from_job',)), obj({'library': SAVED}, ('library',))]}, first_frame_reference={'type': 'boolean', 'default': False, 'description': @@ -382,6 +383,8 @@ def route(summary, body=None, query=None, **kw): ('GET', '/jobs/{id}'): route('Read job progress and result', query=obj({'wait': WAIT})), ('GET', '/jobs/{id}/events'): route('Watch job progress'), ('POST', '/jobs/{id}/cancel'): route('Cancel a job', body=EMPTY, optional_body=True), + ('POST', '/gpu/release'): route('Release the video model from the graphics card and memory when idle', + body=EMPTY, optional_body=True), ('POST', '/jobs/{id}/resume'): route('Resume unfinished generation blocks', body=obj({'ref': REF}), optional_body=True, idempotent=True), ('POST', '/media'): route('Upload image, video or audio content', @@ -465,6 +468,7 @@ def route(summary, body=None, query=None, **kw): ('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None), ('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY), ('/chat/delete', 'POST', obj({'session': SESSION, 'stop': {'type': 'boolean'}}, ('session',))), + ('/chat/rename', 'POST', obj({'session': SESSION, 'title': {'type': 'string', 'maxLength': 60}}, ('session', 'title'))), ('/chat', 'POST', chat_request()), ('/chat/abandon', 'POST', obj({'session': SESSION})), ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}, @@ -1503,6 +1507,7 @@ def openapi(): 'freevideo_resume': ('POST', '/jobs/{id}/resume'), 'freevideo_add_media': ('POST', '/media'), 'freevideo_import_media': ('POST', '/imports'), 'freevideo_imports': ('GET', '/imports'), 'freevideo_edit_image': ('POST', '/media/edit'), 'freevideo_library': ('GET', '/library'), + 'freevideo_release_gpu': ('POST', '/gpu/release'), } TOOL_ENDPOINTS.update({'freevideo_drama_' + name: (method, suffix) for name, (method, suffix, _) in DRAMA_OPERATIONS.items()}) @@ -1521,7 +1526,7 @@ def openapi(): SESSION_ARGUMENT = dict(type='string', pattern='^s[0-9a-f]{12}$', - description='Work session; omit for your latest.') + description='Work session; omit for latest.') SHAPE_MIN_BYTES = 400 diff --git a/freevideo_engine/agent/sessions.py b/freevideo_engine/agent/sessions.py index 432acc11..a48a481a 100644 --- a/freevideo_engine/agent/sessions.py +++ b/freevideo_engine/agent/sessions.py @@ -71,6 +71,7 @@ def __init__(self, root: Path, *, clock=time.time, max_events=20000): # there: the conversation the user is in, and what the page calls each one in its history. self._acted = {} self._said = {} + self._titles = {} # What the user named a conversation, over the first thing they said. self._chosen = set() self._waiters = {} self._partial = set() @@ -151,6 +152,15 @@ def unreadable(self): def _fold(self, sid, event): meta = self._meta[sid] + if event['type'] == 'control' and event['action'] == 'renamed' and event['by'] == 'user': + # Naming a conversation is not activity in it: it keeps its place in the list and gives the agent nothing to read. + meta['last_seq'] = event['seq'] + if event['title'].strip(): + meta['title'] = event['title'] + self._titles[sid] = event['title'].strip()[:60] + else: + self._titles.pop(sid, None) + return meta.update(last_seq=event['seq'], updated_at=event['at']) turn = self._turn.setdefault(sid, [0, 0]) if event['by'] == 'user' and event['type'] == 'message': @@ -233,7 +243,7 @@ def delete(self, sid) -> None: self._path(sid).unlink(missing_ok=True) self._closing.discard(sid) waiters = self._waiters.pop(sid, ()) - for kept in (self._meta, self._ids, self._clients, self._initial_seq, self._user_seq, self._turn, self._stage, self._acted, self._said): + for kept in (self._meta, self._ids, self._clients, self._initial_seq, self._user_seq, self._turn, self._stage, self._acted, self._said, self._titles): kept.pop(sid, None) self._chosen.discard(sid) self._partial.discard(sid) @@ -455,7 +465,7 @@ def history(self, client, *, limit=50) -> list[dict]: with self.lock: rows = sorted((row for sid, row in self._meta.items() if client in self._clients[sid] and sid in self._said), key=lambda r: (r['updated_at'], r['id']), reverse=True) - return [dict(id=r['id'], title=self._said[r['id']], updated_at=r['updated_at'], current=r['id'] == current) + return [dict(id=r['id'], title=self._titles.get(r['id']) or self._said[r['id']], updated_at=r['updated_at'], current=r['id'] == current) for r in rows[:limit]] def read_until(self, sid, reader) -> int: diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 76c7bf82..8fe24d31 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -226,6 +226,8 @@ def normalize(value, *, port=None): result = dict(kind=kind, **_common(value)) if kind == 'generate': result['force_regenerate'] = value['force_regenerate'] + if value.get('release_gpu_after'): + result['release_gpu_after'] = True if value.get('refine_from'): ref = value['refine_from'] result['refine_from'] = {'from_job' if JOB_ID.fullmatch(ref) else 'library': ref} diff --git a/freevideo_engine/audio_use_request.py b/freevideo_engine/audio_use_request.py new file mode 100644 index 00000000..34c6af52 --- /dev/null +++ b/freevideo_engine/audio_use_request.py @@ -0,0 +1,87 @@ +"""File-backed original-audio preflight for editor and background requests.""" +from copy import deepcopy +import json + +from .original_audio import AudioUseError, audio_use_option, fields, inspect, canvas_for, original +from .chain_plan import MAX_SECONDS + + +def selection(value, input_directory): + from .comfy_assets import resolve_assets + if not isinstance(value, dict) or set(value) - {'file', 'use', 'range'}: + raise AudioUseError('audio_use_invalid') + assets = resolve_assets(json.dumps([dict(value, role='reference')]), input_directory) + row = original(assets) + if row is None: + return dict(use='reference') + info = inspect(row, max_seconds=MAX_SECONDS) + canvas = canvas_for(info, 256, 256, max_seconds=MAX_SECONDS) + samples, rate = info['end_sample'] - info['start_sample'], info['sample_rate'] + return dict(use='original', source_id=value['file'], sample_rate=rate, + range=dict(start_sample=info['start_sample'], end_sample=info['end_sample']), + samples=samples, seconds=samples / rate, frames=canvas['frames'], + output_frames=(samples * 24 + rate - 1) // rate, source_sha256=info['sha256']) + + +def graph_audio(prompt, node): + """Inspect only connected known media nodes; never log the graph or text.""" + from .chain_runner import node_prompt + from .comfy_assets import media_kind + rows = [] + for item in node_prompt(prompt, node).values(): + if not isinstance(item, dict): + continue + inputs = item.get('inputs') or {} + kind = item.get('class_type') + if kind == 'FreeVideoMedia': + assets = inputs.get('assets', '[]') + if not isinstance(assets, str): + continue # Dynamic sockets are validated when executed. + assets = json.loads(assets) + if not isinstance(assets, list): + raise AudioUseError('audio_use_invalid') + for asset in assets: + if not isinstance(asset, dict): + raise AudioUseError('audio_use_invalid') + if not asset.get('enabled', True): + continue + if 'use' in asset or 'range' in asset: + options = fields(dict(asset, kind=media_kind(asset.get('file', '')))) + if options: + rows.append(dict(kind='audio', file=asset['file'], **options)) + use = audio_use_option(inputs.get('audio_use')) + if isinstance(use, str) and fields(dict(kind='audio', use=use)): + rows.append(dict(kind='audio', use='original')) + elif kind == 'FreeVideoReference': + use = audio_use_option(inputs.get('use')) + if isinstance(use, str) and fields(dict(kind='audio', use=use)): + rows.append(dict(kind='audio', use='original')) + original(dict(references=rows)) + return rows + + +def prepare_chain(payload, input_directory): + rows = graph_audio(payload['prompt'], payload['node']) + if not rows: + return payload + value = deepcopy(payload) + blocks = value.get('blocks') + if blocks is not None and (not isinstance(blocks, list) or any(not isinstance(row, dict) for row in blocks)): + raise AudioUseError('audio_original_route') + if 'file' not in rows[0]: + # Tensor sockets are resolved in the node, not by the HTTP thread. + return value + from .comfy_assets import resolve_assets + asset = {k: v for k, v in rows[0].items() if k != 'kind'} + media = resolve_assets(json.dumps([dict(asset, role='reference')]), input_directory) + info = inspect(original(media), max_seconds=MAX_SECONDS) + selected = canvas_for(info, 256, 256, max_seconds=MAX_SECONDS) + value['original_audio'] = dict(file=asset['file'], use='original', info=info) + if blocks is not None: + if len(blocks) == 1 and not any(blocks[0].get(k) for k in ('after', 'before')): + if blocks[0]['frames'] < (info['end_sample'] - info['start_sample']) * 24 / info['sample_rate']: + raise AudioUseError('audio_original_slot') + blocks[0]['frames'] = selected['frames'] + else: + value['seconds'] = (info['end_sample'] - info['start_sample']) / info['sample_rate'] + return value diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index e9d02978..8dba831f 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -35,6 +35,7 @@ bootstrap_versions, uv_file_arguments, uv_build_environment) from . import disk_space from . import network +from .comfy_environment import with_private_compiler from . import processes from .system import install_root, venv_python, system_memory, nvidia_smi, memory_sample, curl_executable, missing_curl_message from .system import inference_headroom @@ -362,7 +363,6 @@ def inventory(gpu=None): 'cpu_threads': os.cpu_count(), 'swap_total_bytes': ram.get('swap_total_bytes'), 'swap_free_bytes': ram.get('swap_free_bytes'), 'system_memory': ram, 'compiler': shutil.which('cl' if platform.system() == 'Windows' else 'g++'), 'git': usable_git(), - **({} if platform.system() == 'Windows' else {'make': shutil.which('make')}), 'git_found': shutil.which('git'), **detection} @@ -523,10 +523,6 @@ def prepare_end(ui, success=True): ui.end(PREPARE_KEY, success=success) -def missing_sage_tools(snapshot): - return ([] if snapshot.get('compiler') else ['g++']) + ([] if snapshot.get('make', True) else ['make']) - - def plan(args, *, local_progress=None, stage=None): root = args.root.expanduser().resolve() warnings = [] @@ -571,24 +567,14 @@ def plan(args, *, local_progress=None, stage=None): if cuda_api < 13000 if cuda_api is not None else (major and int(major.group()) < 580): # The card names the graphics card too; brackets would end its pattern early. model = re.sub(r'[()\r\n]', '', str(snapshot['selected_gpu'].get('name') or '')).strip() - errors.append('NVIDIA driver %s%s is older than 580, the minimum for the CUDA 13 encoder; update the driver first.' - % (driver, ' (%s)' % model if model else '')) + # The CUDA fallback reads no driver version; leave it out instead of printing a gap. + errors.append('NVIDIA driver%s%s is older than 580, the minimum for the CUDA 13 encoder; update the driver first.' + % (' ' + driver if driver else '', ' (%s)' % model if model else '')) compatibility = hardware.cuda_compatibility() if compatibility['error']: errors.append(compatibility['error']) snapshot['cuda_compatibility'] = compatibility # A missing or too old Git is replaced by FreeVideo's own. - if not windows_target: - # ./setup.sh installs these before planning; the desktop launcher names the command instead. - missing = missing_sage_tools(snapshot) - if missing: - from .system import linux_tools_command - command = linux_tools_command(missing) - warnings.append('Missing %s, needed to build GPU acceleration for this GPU. %s, then check again.' % ( - ' and '.join(missing), 'In a terminal, run: ' + command if command else - 'Ask your administrator to install them')) - logging.getLogger(__name__).warning('%s Skipping Sage2 source compilation; ' - 'continuing with cudnn/torch-flash attention.', warnings[-1]) if offline_installation(root): # Its sources carry no Git history and its Python is the package's. errors.append('This folder holds an installation from offline packages; automatic installation does not change it. ' @@ -1344,6 +1330,7 @@ def device_environment(self, env): from .triton_compat import environment return environment(self.root, dict(env, CUDA_VISIBLE_DEVICES=self.plan['inventory']['selected_gpu']['uuid'], + CUDA_DEVICE_ORDER='PCI_BUS_ID', TRITON_CACHE_DIR=str(self.root / 'kernel-cache' / 'triton'), CUDA_CACHE_PATH=str(self.root / 'kernel-cache' / 'cuda'), TORCHINDUCTOR_CACHE_DIR=str(self.root / 'kernel-cache' / 'inductor'))) @@ -1486,6 +1473,7 @@ def command(self, label, args, env=None, cwd=None): from .triton_compat import temporary_environment env = temporary_environment(self.root, env or self.env) env = dict(env, FREEVIDEO_INSTALL_PROGRESS='1') + env = with_private_compiler(self.root, env) args, cwd = uv_file_arguments(args, cwd) env = uv_build_environment(args, self.root, env) with self.state_lock: @@ -1685,7 +1673,7 @@ def encoder_packages(): 'print("Windows runtime imports passed; GPU kernel probes follow")']), ('runtime',), ('environment:' + name,)) overlap = (self.plan.get('model_transfer') or {}).get('overlap_build', True) # Linux builds SageAttention from a Git checkout. - task('sage', lambda: self.install_sage(uv, python), ('runtime', 'git') if overlap else ('runtime', 'git', 'models'), + task('sage', lambda: self.install_sage_optional(uv, python), ('runtime', 'git') if overlap else ('runtime', 'git', 'models'), ('environment:' + name,)) if self.plan.get('disk_mode') == 'extreme': # Finish all environments before models occupy most of the disk. @@ -1793,14 +1781,20 @@ def attempt(source, route, retry_mode): failure.package_attempts = error.attempts raise failure from error + def install_sage_optional(self, uv, python): + """Sage2 is an optional speed-up: where FreeVideo's own compiler builds it, a failure leaves it out.""" + try: + return self.install_sage(uv, python) + except Exception as error: + if (self.system == 'Windows' or self.cancel.is_set() + or shutil.which('g++', path=self.env['PATH'])): + raise + self.ui.write('Sage2 could not be built (%s); continuing without it.\n' % type(error).__name__) + return None + def install_sage(self, uv, python): if self.system == 'Windows': return self.install_sage_windows(uv, python) - missing = missing_sage_tools(self.plan['inventory']) - if missing: - self.ui.write('Missing %s; skipping Sage2 source compilation and continuing with ' - 'cudnn/torch-flash attention.\n' % ' and '.join(missing)) - return None code = ('import json,platform,sys,torch; print(json.dumps(dict(' 'python="cp"+str(sys.version_info.major)+str(sys.version_info.minor), ' 'torch=str(torch.__version__),cuda=torch.version.cuda,cxx11_abi=torch.compiled_with_cxx11_abi(), ' @@ -1839,6 +1833,9 @@ def install_sage(self, uv, python): CPATH=headers, TORCH_CUDA_ARCH_LIST=identity['arch'], MAX_JOBS=str(build['jobs']), EXT_PARALLEL='1', NVCC_APPEND_FLAGS='--threads=' + str(build['nvcc_threads'])) build_env.update(compiler['env'] if compiler['env'] is not None else self.install_compiler_linux()) + if compiler['env'] is None: + # The private compiler is newer than an older system's libstdc++, which the Python process loads first. + build_env['LDFLAGS'] = (build_env.get('LDFLAGS', '') + ' -static-libstdc++ -static-libgcc').strip() self.command('sage2-toolchain-check', [python, '-c', 'from torch.utils.cpp_extension import is_ninja_available; ' 'assert is_ninja_available(), "Ninja must be visible in the build PATH; serial fallback is refused"'], env=build_env) @@ -1865,9 +1862,13 @@ def host_compiler(self, cuda): """ spec = json.loads((PACKAGE / 'gcc_linux.json').read_text(encoding='utf-8')) below = spec['host_compiler_below'].get(cuda) - choice = host_compiler_choice(below, self.env['PATH'], self.env) if below else {} + # Without any system g++, FreeVideo's own compiler builds Sage2. + if not shutil.which('g++', path=self.env['PATH']): + choice = None + else: + choice = host_compiler_choice(below, self.env['PATH'], self.env) if below else {} if choice is None: - return {'env': None, 'identity': 'conda-forge gcc ' + spec['version']} + return {'env': None, 'identity': 'conda-forge gcc ' + spec['version'] + ' static-libstdc++'} cxx = choice.get('CXX', 'g++') return {'env': choice, 'identity': subprocess_output.output_external([cxx, '--version']).splitlines()[0]} @@ -1896,6 +1897,13 @@ def install_compiler_linux(self): shutil.rmtree(explicit.parent, ignore_errors=True) micromamba.unlink(missing_ok=True) self.command('portable-gcc-check', [compilers['CXX'], '--version']) + # The same compiler under its plain names, for tools that look for gcc (Triton builds its launcher with it). + links = self.root / 'tools' / 'gcc-bin' + links.mkdir(parents=True, exist_ok=True) + for name, target in (('gcc', compilers['CC']), ('cc', compilers['CC']), ('g++', compilers['CXX'])): + if not (links / name).is_symlink() or Path(os.readlink(links / name)) != target: + (links / name).unlink(missing_ok=True) + (links / name).symlink_to(target) return {key: str(path) for key, path in compilers.items()} def install_sage_windows(self, uv, python): diff --git a/freevideo_engine/capabilities.py b/freevideo_engine/capabilities.py index a2c027d2..67eae19e 100644 --- a/freevideo_engine/capabilities.py +++ b/freevideo_engine/capabilities.py @@ -53,7 +53,7 @@ def normalize_references(task, references=None): if not isinstance(row, dict): raise ValueError('Invalid reference metadata.') kind = row.get('kind') - allowed = {'kind', 'seconds'} if kind == 'audio' else {'kind', 'width', 'height'} + allowed = {'kind', 'seconds', 'use'} if kind == 'audio' else {'kind', 'width', 'height'} if kind == 'video': allowed |= {'seconds', 'audio_seconds', 'guide'} if kind not in ('image', 'video', 'audio') or set(row) - allowed: @@ -83,7 +83,16 @@ def normalize_references(task, references=None): raise ValueError('Specify positive finite soundtrack seconds, or omit it for silent video.') if audio < 1 / 32000: raise ValueError('Reference audio is empty after normalization.') - rows.append((kind, width, height, seconds, audio)) + from .original_audio import fields, AudioUseError, MAX_SECONDS + use = fields(row) + if use: + if seconds < 1.625: + raise AudioUseError('audio_original_short') + if seconds > MAX_SECONDS: + raise AudioUseError('audio_original_long') + rows.append((kind, width, height, seconds, audio) + (('original',) if use else ())) + if sum(len(row) == 6 for row in rows) > 1: + raise AudioUseError('audio_original_multiple') visual = any(r[0] != 'audio' for r in rows) audio = any(r[0] == 'audio' or r[4] is not None for r in rows) expected = 'ref2va_av' if visual and audio else 'ref2va_audio' if audio else 'ref2va' if visual else None @@ -96,7 +105,7 @@ def reference_parameters(references): """JSON form of a normalized cache key, also used in notifications.""" if references is None: return None - return [{k: v for k, v in zip(('kind', 'width', 'height', 'seconds', 'audio_seconds'), row) + return [{k: v for k, v in zip(('kind', 'width', 'height', 'seconds', 'audio_seconds', 'use'), row) if v is not None} for row in references] @@ -124,7 +133,13 @@ def _material(canvas, task, references): video_rows = audio_rows = 0 duration = 0. limit = reference_frames(canvas) - for kind, width, height, seconds, soundtrack in references: + for row in references: + kind, width, height, seconds, soundtrack = row[:5] + if len(row) == 6: + # Original audio occupies the existing target rows. Keep all + # ordinary model/decoder budgets, without duplicating the track. + duration += seconds + continue if kind != 'audio': width, height = reference_size(width, height, canvas) patches = (width // 32) * (height // 32) @@ -283,7 +298,9 @@ def _admit(machine, width, height, frames, steps, two_pass, task='t2va', refine= return request, code, None hardware = Hardware(machine['name'], machine['cap'], machine['vram'], machine['vram_free'], machine['ram'], machine['ram_available'], system=machine['system']) - options = dict(precision=machine['precision'], attention='sage2/sage2', available_backends={'sage2'}) + # SageAttention 2 has no sm100 kernels (data-centre Blackwell); those cards run torch-flash. + backend = 'torch-flash' if machine['cap'][0] == 10 else 'sage2' + options = dict(precision=machine['precision'], attention=backend + '/' + backend, available_backends={backend}) choose(hardware, canvas=canvas, stage='encoding', **options) final = choose(hardware, canvas=dict(canvas, step_cache=step_cache) if step_cache else canvas, **options) if final.engine.get('step_cache'): diff --git a/freevideo_engine/chain_join.py b/freevideo_engine/chain_join.py index 5abfc0dd..d1fd11a1 100644 --- a/freevideo_engine/chain_join.py +++ b/freevideo_engine/chain_join.py @@ -5,6 +5,8 @@ sound is decoded and encoded once as a single stream: every block's sound follows the previous block's sound, not its own first frame, which keeps a long chain in step. +Original soundtracks instead bypass every block's sound and mux the selected +source range once, with a sample-identity receipt. """ from datetime import datetime from fractions import Fraction @@ -160,7 +162,8 @@ def _exact_tail(source, target, wanted, cancelled=None): temporary.unlink(missing_ok=True) -def join_files(sources, target, cancelled=None, frames=None, leads=None, expected=None, speaking=None): +def join_files(sources, target, cancelled=None, frames=None, leads=None, expected=None, speaking=None, *, + original_audio=None): """Write `target` from `sources` in order; returns the joined frame and sample counts. `frames`: stop at this length when a compatible short tail can close it exactly, @@ -168,32 +171,38 @@ def join_files(sources, target, cancelled=None, frames=None, leads=None, expecte `leads`: optional `leads(following, previous)` returning a seam lead kept by the chain; the shared continuation record is read only when it has none. `expected`: how many seams should be repaired; only reported back, the caller decides. + `original_audio`: mux the original selection and close the picture on its + source-sample clock; a rounded completed prefix keeps its existing frames. """ sources = list(sources) try: - return _join_ending(sources, target, cancelled, frames, leads, expected, speaking) + return _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio) except _VoiceFailed: logging.warning('[FreeVideo] The voice of the parts could not be evened out; they are joined as they are', exc_info=True) - return _join_ending(sources, target, cancelled, frames, leads, expected, None) + return _join_ending(sources, target, cancelled, frames, leads, expected, None, original_audio) -def _join_ending(sources, target, cancelled, frames, leads, expected, speaking): +def _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio=None): try: - return _join_files(sources, target, cancelled, frames, leads=leads, expected=expected, speaking=speaking) + return _join_files(sources, target, cancelled, frames, leads=leads, expected=expected, speaking=speaking, + original_audio=original_audio) except _ExactEndFailed: # A late mux or validation error must also leave the original ending available. return _join_files(sources, target, cancelled, frames, exact=False, leads=leads, expected=expected, - speaking=speaking) + speaking=speaking, original_audio=original_audio) -def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None): +def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None, + original_audio=None): import av from .comfy_metadata import _copy_stream - try: - from .comfy_continuation import audio_lead_in, apply_audio_lead_in - except ImportError: - audio_lead_in = apply_audio_lead_in = None + audio_lead_in = apply_audio_lead_in = None + if original_audio is None: + try: + from .comfy_continuation import audio_lead_in, apply_audio_lead_in + except ImportError: + audio_lead_in = apply_audio_lead_in = None if leads is not None and audio_lead_in is not None: stored = audio_lead_in def audio_lead_in(video, previous=None): @@ -203,7 +212,7 @@ def audio_lead_in(video, previous=None): return leads(video, previous) return leads(video, previous) or stored(video, previous=previous) sources, target = [Path(source) for source in sources], Path(target) - if len(sources) < 2: + if len(sources) < (2 if original_audio is None else 1): raise ValueError('Joining needs at least two videos') if target.exists() or target.is_symlink(): raise ValueError('The joined video already exists') @@ -216,7 +225,7 @@ def audio_lead_in(video, previous=None): if len(first.streams.video) != 1: raise ValueError('A block has no single picture stream') shape = _shape(first.streams.video[0]) - sound = first.streams.audio[0] if first.streams.audio else None + sound = first.streams.audio[0] if first.streams.audio and original_audio is None else None rate = sound.codec_context.sample_rate if sound is not None else None layout = sound.codec_context.layout.name if sound is not None else None bit_rate = max(sound.codec_context.bit_rate or 0, 192000) if sound is not None else None @@ -236,7 +245,7 @@ def audio_lead_in(video, previous=None): with av.open(str(source)) as container: if len(container.streams.video) != 1 or _shape(container.streams.video[0]) != shape: raise ValueError('The blocks differ in size or encoding') - if bool(container.streams.audio) != (sound is not None): + if original_audio is None and bool(container.streams.audio) != (sound is not None): raise ValueError('Only some of the blocks have sound') video = container.streams.video[0] tail = (_exact_tail(source, target, wanted - frames, cancelled) @@ -308,8 +317,33 @@ def audio_lead_in(video, previous=None): with av.open(str(temporary)) as check: if check.streams.video[0].frames != frames: raise ValueError('The joined video is incomplete') + receipt = None + if original_audio is not None: + from .original_audio import mux + required = ((original_audio['end_sample'] - original_audio['start_sample']) * FPS + + original_audio['sample_rate'] - 1) // original_audio['sample_rate'] + if frames < required: + # Completed prefixes end on nearest-sample global boundaries. + # Rounding up can put the endpoint half a sample past the + # last frame; it does not require a further picture frame. + count = original_audio['end_sample'] - original_audio['start_sample'] + if 2 * (count * FPS - frames * original_audio['sample_rate']) > FPS: + raise ValueError('Original audio is longer than the joined picture') + required = frames + receipt = mux(temporary, temporary, original_audio, video_frames=required) + _check(cancelled) + rate, samples = receipt['sample_rate'], receipt['samples_per_channel'] + with av.open(str(temporary)) as check: + video = check.streams.video[0] + frames = video.frames + if (video.duration is None or + abs(video.duration * video.time_base - Fraction(samples, rate)) > Fraction(1, rate)): + raise ValueError('The joined picture did not end on the selected audio sample') + if frames != required: + raise ValueError('The joined picture did not reach the selected audio endpoint') os.replace(temporary, target) - return dict(exact_end=wanted is not None and frames == wanted, frames=frames, width=shape[1], height=shape[2], + return dict(**({'original_audio': receipt} if receipt is not None else {}), + exact_end=wanted is not None and frames == wanted, frames=frames, width=shape[1], height=shape[2], audio_samples=samples, sample_rate=rate, seams_repaired=joined_audio.seams_repaired if joined_audio is not None else 0, seams_expected=expected, @@ -563,7 +597,7 @@ def _replace_target(root, replace, identity, sources): def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, leads=None, expected_seams=None, - speaking=None): + speaking=None, *, original_audio=None): """Join relative block outputs into a saved video, replacing only this chain's earlier join.""" import folder_paths root = Path(folder_paths.get_output_directory()).resolve() @@ -594,8 +628,10 @@ def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, l originals = {str(source): output for source, output in zip(sources, outputs)} def kept(following, previous): return leads(originals.get(str(following)), originals.get(str(previous)) if previous is not None else None) - joined = join_files(sources, staged, cancelled, frames, leads=kept, expected=expected_seams, speaking=speaking) - if expected_seams is not None and joined['seams_repaired'] < expected_seams: + joined = join_files(sources, staged, cancelled, frames, leads=kept, expected=expected_seams, speaking=speaking, + **({'original_audio': original_audio} if original_audio is not None else {})) + # The original selection replaces every block's sound, so there are no seam sounds to repair. + if original_audio is None and expected_seams is not None and joined['seams_repaired'] < expected_seams: # A seam sound left unrepaired is never reported as a finished video. raise ValueError('Only %d of %d seam sounds could be repaired' % (joined['seams_repaired'], expected_seams)) report = dict(success=True, geometry=dict(width=joined['width'], height=joined['height'], frames=joined['frames'], @@ -607,6 +643,9 @@ def kept(following, previous): report['chain_join']['seams_expected'] = expected_seams if joined.get('voice') is not None: report['chain_join']['voice'] = joined['voice'] + if original_audio is not None: + report['video']['original_audio'] = joined['original_audio'] + report['geometry']['seconds'] = joined['original_audio']['duration_seconds'] with temporary.open('w', encoding='utf-8') as stream: json.dump(report, stream, ensure_ascii=False) stream.flush() diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 401ee784..7011b013 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -1,4 +1,4 @@ -"""One active durable chain, submitting ordinary host prompts by dependency.""" +"""Durable chains, submitting ordinary host prompts by dependency.""" from copy import deepcopy import json import logging @@ -126,6 +126,9 @@ def __init__(self, store_dir, host, clock=time.monotonic, wall=time.time, estima self.starts, self.success, self.cancel_sent, self.progress_values = {}, {}, set(), {} self.progress_messages = {} self.estimates, self.queue, self.active = {}, [], None + self.concurrent = bool(getattr(host, 'concurrent', False)) + if self.concurrent: + self.active = set() self.recover = set() for record in self.records.values(): if record['state'] in store.ACTIVE: @@ -244,16 +247,16 @@ def _view(self, record, detailed=True, client_id=None): def _waiting(self, record): """What a chain that has not started waits for: how many chains are ahead and, for a single one, the time it has left (None while that is unknown).""" - if record['state'] != 'queued' or record['id'] == self.active or record['id'] not in self.queue: + if record['state'] != 'queued' or record['id'] in self.active_ids() or record['id'] not in self.queue: return None - ahead = [] if self.active is None else [self.active] + ahead = self.active_ids() for identity in self.queue[:self.queue.index(record['id'])]: row = self.records.get(identity) if row is not None and row['state'] == 'queued' and identity not in ahead: ahead.append(identity) if not ahead: return None - remaining = _number(self.progress_values.get(ahead[0], {}).get('remaining_seconds')) if len(ahead) == 1 and ahead[0] == self.active else None + remaining = _number(self.progress_values.get(ahead[0], {}).get('remaining_seconds')) if len(ahead) == 1 and ahead[0] in self.active_ids() else None return {'ahead': len(ahead), 'remaining_seconds': None if remaining is None or remaining < 0 else remaining} def _depths(self, record, seen=()): @@ -529,15 +532,28 @@ def load(self, chain_id): def hidden(self, exists=None): with self.lock: return {row['output'] for record in self.records.values() - if record['join'] and record['output_state'] == 'done' and record['output_blocks'] > 1 + if record['join'] and record['output_state'] == 'done' + and (record['output_blocks'] > 1 or record.get('original_audio') is not None) and record['output'] not in {block['output'] for block in record['blocks']} and (exists is None or exists(record['output'])) for row in record['blocks'][:record['output_blocks']] if row['output'] is not None} + def active_ids(self): + """Stable chain order for both the legacy turn and the concurrent host.""" + if self.concurrent: + return [identity for identity in self.records if identity in self.active] + return [self.active] if self.active is not None else [] + + def _release_active(self, identity): + if self.concurrent: + self.active.discard(identity) + elif self.active == identity: + self.active = None + def _find(self, prompt_id, node): - record = self.records.get(self.active) - if record is not None: - if record['node'] == str(node) and record['state'] in store.ACTIVE: + for identity in self.active_ids(): + record = self.records.get(identity) + if record is not None and record['node'] == str(node) and record['state'] in store.ACTIVE: block = next((row for row in record['blocks'] if row['prompt_id'] == prompt_id and row['state'] in ('queued', 'running')), None) if block is not None: @@ -597,8 +613,9 @@ def finished(self, prompt_id, node, output, report_id, continuation=None): self._save(record) self.wake.set() previous = self._sources(record, block)['after'] + # A kept original soundtrack replaces every seam's sound, so there is no lead to save. keep = (record['id'], block['index'], output, previous['output'] if previous is not None else None, - bool(saved)) + bool(saved) and record.get('original_audio') is None) self._keep_lead(*keep) def _keep_lead(self, identity, index, output, previous, continues): @@ -627,6 +644,9 @@ def _seams_kept(self, snapshot): block that continues from a block other than its neighbour cannot be repaired and counts as lost.""" identity, blocks = snapshot['id'], store.leading_blocks(snapshot) wanted, kept = set(), 0 + if snapshot.get('original_audio') is not None: + # The original selection replaces every block's sound, so no seam sound needs repair. + return wanted, kept for position, row in enumerate(blocks[1:] if getattr(self.host, 'repairs_seams', True) else (), 1): if not self._continues(row): continue @@ -647,7 +667,7 @@ def _finish_blocks(self, record, final_state='done'): if not blocks: record.update(output=None, output_blocks=0, output_state='pending', output_error=None) return final_state - if len(blocks) == 1: + if len(blocks) == 1 and record.get('original_audio') is None: output = blocks[0]['output'] record.update(output=output, output_blocks=1 if output else 0, output_state='done', output_error=None) return final_state @@ -677,8 +697,8 @@ def _complete(self, record, block, status): block['error'] = status.get('error') or 'Prompt execution failed' next_state = 'cancelled' if record['state'] == 'cancelling' else next_state record['state'] = self._finish_blocks(record, next_state) if next_state in ('cancelled', 'failed') else next_state - if record['state'] not in store.ACTIVE and self.active == record['id']: - self.active = None + if record['state'] not in store.ACTIVE: + self._release_active(record['id']) self.starts.pop(prompt_id, None) self.success.pop(prompt_id, None) self.cancel_sent.discard(prompt_id) @@ -704,24 +724,29 @@ def step(self): row['state'] in ('queued', 'running') for row in record['blocks']): record['state'] = self._finish_blocks(record, 'cancelled') self._save(record) - if self.active is not None and self.records[self.active]['state'] not in store.ACTIVE: - self.active = None + for active_id in self.active_ids(): + if self.records[active_id]['state'] not in store.ACTIVE: + self._release_active(active_id) self._interrupt_dependants() for queued_id in list(self.queue): waiting = self.records[queued_id] if waiting['state'] not in store.ACTIVE: self.queue.remove(queued_id) - elif self.active is None and (waiting['state'] == 'queued' + elif (self.concurrent or self.active is None) and (waiting['state'] == 'queued' and self._dependency_state(waiting) == 'ready' or waiting['state'] == 'cancelling' and waiting['output_state'] == 'joining'): self.queue.remove(queued_id) - self.active = queued_id - identity = self.active + if self.concurrent: + self.active.add(queued_id) + else: + self.active = queued_id + identities = self.active_ids() for recovering_id, snapshot in recovering: if not self.closed.is_set(): self._join(recovering_id, snapshot) - if identity is not None and not self.closed.is_set(): - self._step_chain(identity) + for identity in identities: + if not self.closed.is_set(): + self._step_chain(identity) def _step_chain(self, identity): with self.lock: @@ -763,13 +788,16 @@ def _step_chain(self, identity): self._save(record) prompt = deepcopy(record['snapshot']['prompt']) block_inputs = dict(block['inputs']) - block_inputs.update(index=block['index'], count=len(record['blocks'])) + block_inputs.update(index=block['index'], count=len(record['blocks']), + original_audio=block.get('original_audio') is not None) sources = self._sources(record, block) extra = deepcopy(record['snapshot']['extra_data']) if record['client_id'] is not None: extra['client_id'] = record['client_id'] extra['freevideo_chain'] = dict(id=identity, block=block['id'], part=block['part'], index=block['index'], count=len(record['blocks']), after=deepcopy(block['after']), before=deepcopy(block['before'])) + if self.concurrent: + extra['freevideo_chain']['node'] = node action = 'submit' elif record['state'] == 'cancelling' and prompt_id not in self.cancel_sent: action = 'cancel' @@ -804,6 +832,9 @@ def _step_chain(self, identity): if hasattr(self.host, 'continued'): self.host.continued(prompt, node) prompt[node]['inputs'].update(self.inputs(block_inputs)) + if record.get('original_audio') is not None: + from .original_audio_chain import block_prompt + block_prompt(prompt, node, record['original_audio'], block['original_audio']) self.host.submit(prompt_id, node_prompt(prompt, node), extra) except Exception as error: # A response may be lost after the host has accepted the prompt. @@ -937,8 +968,8 @@ def lead_reader(following, previous): else: record.update(state=final_state, output=output, output_blocks=len(store.leading_blocks(snapshot)), output_state='done', output_error=None, output_seams=dict(expected=promised, kept=kept)) - if record['state'] not in store.ACTIVE and self.active == identity: - self.active = None + if record['state'] not in store.ACTIVE: + self._release_active(identity) self._save(record) def _estimate(self, record, block): @@ -954,9 +985,12 @@ def _estimate(self, record, block): self.estimates[key] = None return self.estimates[key] - def progress(self, node, message): + def progress(self, node, message, prompt_id=None): with self.lock: - record = self.records.get(self.active) + if self.concurrent: + record, _ = self._find(prompt_id, node) + else: + record = self.records.get(self.active) if record is None or record['node'] != str(node) or record['state'] not in store.ACTIVE: return None block = self._current(record) diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py index cd4bb63f..6177fc86 100644 --- a/freevideo_engine/chain_store.py +++ b/freevideo_engine/chain_store.py @@ -91,9 +91,15 @@ def validate(payload): source = payload.get('after') if source is not None and not isinstance(source, dict): raise ValueError('A planned video can only continue a saved work') - planned = chain_plan.plan(payload['seconds'], payload.get('single', False), payload.get('max_frames'), - payload.get('single_limit_seconds'), payload.get('block_seconds'), - continued=source is not None) + if payload.get('original_audio') is not None: + from .original_audio_chain import plan as audio_plan, selection_info + planned = audio_plan(selection_info(payload['original_audio']), + **{key: payload[key] for key in ('single', 'max_frames', 'single_limit_seconds', 'block_seconds') + if key in payload}) + else: + planned = chain_plan.plan(payload['seconds'], payload.get('single', False), payload.get('max_frames'), + payload.get('single_limit_seconds'), payload.get('block_seconds'), + continued=source is not None) inputs = prompt[node]['inputs'] text, seed = payload.get('text', inputs.get('text')), payload.get('seed', inputs.get('seed')) if not isinstance(text, str): @@ -117,6 +123,11 @@ def validate(payload): rows = payload['blocks'] if not isinstance(rows, list) or not 1 <= len(rows) <= chain_plan.MAX_BLOCKS: raise ValueError('A chain requires 1 to %s blocks' % chain_plan.MAX_BLOCKS) + from .audio_use_request import graph_audio + from .original_audio import AudioUseError + if graph_audio(prompt, node) and payload.get('original_audio') is None and (len(rows) != 1 or not isinstance(rows[0], dict) + or any(rows[0].get(k) for k in ('after', 'before'))): + raise AudioUseError('audio_original_route') blocks, ids = [], set() for index, row in enumerate(rows): if not isinstance(row, dict): @@ -175,6 +186,9 @@ def visit(identity): output_error=None, output_blocks=0, split=split) if queue: record['queue'] = True + if payload.get('original_audio') is not None: + from .original_audio_chain import bind + bind(record, payload['original_audio']) try: json.dumps(record, allow_nan=False) except (ValueError, TypeError, OverflowError) as error: @@ -213,6 +227,12 @@ def detail(record): # Split parts repeat words put there by code; the page lists and edits only each part's own. from .prompt_vlm.rules import own_words value['part_words'] = own_words([str(row['inputs'].get('text', '')) for row in record['blocks']]) + if record.get('original_audio') is not None: + selected = record['original_audio'] + info = selected['info'] + value['original_audio'] = dict(file=selected['file'], use='original', + range={key: info[key] for key in ('start_sample', 'end_sample')}, + sample_rate=info['sample_rate'], source_sha256=info['sha256']) return value @@ -304,10 +324,13 @@ def load_all(directory, limit=200, *, identity=None): normalized = validate(dict(node=record['node'], client_id=record['client_id'], **snapshot, queue=record.get('queue', False), blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')} | row['inputs'] - for row in record['blocks']], join=record['join'])) + for row in record['blocks']], join=record['join'], + **({'original_audio': record['original_audio']} if 'original_audio' in record else {}))) if record['state'] not in CHAIN_STATES: raise ValueError('Invalid chain state') for row, clean in zip(record['blocks'], normalized['blocks']): + if record.get('original_audio') is not None and row.get('original_audio') != clean['original_audio']: + raise ValueError('Invalid original audio block range') row.setdefault('new_frames', clean['new_frames']) if type(row['new_frames']) is not int or row['new_frames'] != clean['new_frames']: raise ValueError('Invalid block new frame count') diff --git a/freevideo_engine/cli.py b/freevideo_engine/cli.py index 2767d473..7e15b6b3 100644 --- a/freevideo_engine/cli.py +++ b/freevideo_engine/cli.py @@ -27,6 +27,10 @@ def resource_arguments(parser): def main(): + if os.environ.get('FREEVIDEO_HOME'): + # Without a system compiler, run-time builds (Triton) in this process use the one FreeVideo installed. + from .comfy_environment import with_private_compiler + with_private_compiler(os.environ['FREEVIDEO_HOME'], os.environ) if len(sys.argv) > 1 and sys.argv[1] in ('projects', 'sessions'): from .agent.cli import main as agent return agent(sys.argv[1:]) @@ -92,6 +96,7 @@ def main(): source = generate.add_mutually_exclusive_group(required=True) source.add_argument('--prompt-file', type=Path) source.add_argument('--conditioning', type=Path) + generate.add_argument('--allow-empty-prompt', action='store_true', help='Generate from an empty prompt file') generate.add_argument('--cache', type=Path, required=True) generate.add_argument('--base', type=Path) generate.add_argument('--checkpoint', type=Path) @@ -99,6 +104,9 @@ def main(): from .media_request import TASKS generate.add_argument('--task', choices=TASKS, help='Input task; inferred when --media is provided') generate.add_argument('--media', type=Path, help='Versioned keyframe/reference/LoRA request JSON; local files only') + generate.add_argument('--audio', type=Path, help='Audio file; may be combined with image/video references in --media') + generate.add_argument('--audio-use', choices=('reference', 'original'), + help='Audio use (default: reference); with --media alone, applies to its single audio file') generate.add_argument('--continue-from', type=Path, help='Continue an existing MP4 using its absolute path') generate.add_argument('--first-frame-reference', action='store_true', help='With --continue-from: also give the model the first frame of the whole film as a ' @@ -167,6 +175,11 @@ def main(): anchor=args.continue_anchor is not None, two_pass=args.two_pass, preview=args.preview, refine_from=args.refine_from) if sys.platform == 'darwin' and args.command in ('plan', 'generate', 'encode', 'doctor'): + if args.command == 'generate': + from .media_request import read + from .original_audio import original, AudioUseError + if args.audio is not None or args.audio_use == 'original' or original(read(args.media)) is not None: + raise AudioUseError('audio_original_backend') from .macos_generate import dispatch return dispatch(args) if args.command == 'plan': diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 0d085fa3..4628d13e 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -75,7 +75,8 @@ def resolve_assets(serialized, input_directory): root = Path(input_directory).resolve() result = {} for row in rows: - if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth'}: + if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth', + 'use', 'range', 'segment'}: raise ValueError('Invalid media item') if 'depth' in row: if type(row['depth']) is not bool: @@ -104,6 +105,8 @@ def resolve_assets(serialized, input_directory): if not path.is_file(): raise ValueError('Upload or select the missing media file: ' + name) kind = media_kind(name) + from .original_audio import fields + audio_fields = fields(dict(row, kind=kind)) if role in ('first', 'last'): if kind != 'image': raise ValueError('First and last frames must be images') @@ -111,7 +114,7 @@ def resolve_assets(serialized, input_directory): raise ValueError('Select only one ' + role + ' frame') result[role] = str(path) elif role == 'reference': - reference = {'kind': kind, 'path': str(path)} + reference = {'kind': kind, 'path': str(path), **audio_fields} if row.get('depth') is True: reference.update(depth=True, early_steps=3) if cast_id(row) is not None: @@ -138,6 +141,8 @@ def resolve_assets(serialized, input_directory): result.setdefault('references', []).append(reference) else: raise ValueError('Media role must be first, last or reference') + from .original_audio import original + original(result) # Mixing keyframes and references is checked when the video is made (comfy_media.export): # a part that continues a video uses its keyframes as pictures. return result @@ -147,23 +152,27 @@ def continued_assets(serialized): """The panel rows a continued part uses: enabled image references in panel order, at most three (continuation.pick_pictures: with two or more characters, one picture each). - Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks.""" + Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks. + The one exception is a sound used as is: every part plays its own range of it (original_audio_chain.block_prompt).""" from .continuation import pick_pictures try: rows = json.loads(serialized) except (TypeError, ValueError): return '[]' - pictures = [] + pictures, original = [], [] for row in rows if isinstance(rows, list) else (): if (isinstance(row, dict) and row.get('enabled', True) is True and row.get('role') == 'reference' and isinstance(row.get('file'), str)): + if row.get('use') == 'original': + original.append(dict(file=row['file'], role='reference', use='original')) + continue try: image = media_kind(row['file']) == 'image' except ValueError: image = False if image: pictures.append(row) - return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)]) + return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)] + original) def cast_id(row): @@ -180,12 +189,19 @@ def _kept_row(row): return kept -def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, **images): +def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, audio_use='reference', + **images): """Keep connected IMAGE/AUDIO values native until request input export.""" result = dict(assets) if set(images) - set(REFERENCE_INPUTS[1:]): raise TypeError('Unknown reference image input') images = dict(reference=reference, **images) + from .original_audio import fields, original + audio_fields = fields(dict(kind='audio', use=audio_use)) + if audio_fields and reference_audio is None: + raise ValueError('Connect audio before selecting Use as is') + original(dict(references=[*result.get('references', []), *([dict(kind='audio', **audio_fields)] + if reference_audio is not None else [])])) connected = {name: value for name, value in (('first', first), ('last', last), *((name, images.get(name)) for name in REFERENCE_INPUTS), ('reference_audio', reference_audio)) if value is not None} @@ -202,6 +218,8 @@ def connect_assets(assets, *, first=None, last=None, reference=None, reference_a 'For a long image batch, connect a clip to the Reference node\'s video input; a clip counts as one reference.') if connected: result['_connected'] = connected + if audio_fields: + result['_audio_use'] = 'original' return result @@ -246,6 +264,8 @@ def output_summary(report, relative_video): # Resource reports keep the generated canvas; result cards show only saved frames. result['geometry'].update(frames=report['output_frames'], seconds=report['output_frames'] / (result['geometry'].get('fps') or 24)) + if video.get('original_audio'): + result['original_audio'] = {k: v for k, v in video['original_audio'].items() if k != 'path'} checkpoint = encoding.get('checkpoint') or {} sizes = checkpoint.get('bytes'), checkpoint.get('expected_bytes') # Reports are read back from disk; a malformed one gets no notice rather than failing the library. diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 76475544..1606790a 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -6,6 +6,7 @@ import json import logging import math +import contextlib import os from pathlib import Path import subprocess @@ -21,6 +22,31 @@ from .monitoring import save +@contextlib.contextmanager +def _device_generation_lease(environment): + """An executor of a multi-GPU pool holds its own device's lease, not the shared installation lock. + + The child inherits the device lease; hosts without a pool leave the environment alone and + the child takes the installation lock itself, as before. + """ + from .comfy_environment import executor_device + uuid = executor_device(environment) + if not uuid: + yield () + return + from types import SimpleNamespace + from .gpu_pool import child_lease_environment, generation_lease + from .locking import LOCK_ENV + device = SimpleNamespace(uuid=uuid) + with generation_lease(device, wait_seconds=3) as descriptor: + os.set_inheritable(descriptor, True) + environment.update(child_lease_environment(device), **{LOCK_ENV: str(descriptor)}) + try: + yield (descriptor,) + finally: + environment.pop(LOCK_ENV, None) + + def _timing_seconds(value): """Read a finite positive forecast value without inventing a duration.""" if isinstance(value, dict): @@ -852,12 +878,30 @@ def installation(source=None, environ=None): raise ValueError('The FreeVideo installation was moved. Rerun setup in the selected folder.') if not io_path(machine['python']).is_file(): raise ValueError('The FreeVideo Python environment is missing. Rerun setup in %s.' % root) + from .comfy_environment import executor_device + device_id = executor_device(environ) + if device_id: + from .gpu_pool import enumerate_devices + env = os.environ if environ is None else environ + if env.get('CUDA_VISIBLE_DEVICES') != device_id: + raise ValueError('Executor device binding changed') + device = next((item for item in enumerate_devices(env) if item.uuid == device_id), None) + if device is None: + raise ValueError('Executor device is unavailable') + machine = dict(machine, gpu_uuid=device.uuid, gpu_name=device.name) + if isinstance(machine.get('hardware'), dict): + hardware = dict(machine['hardware'], gpu_uuid=device.uuid, gpu_name=device.name, + capability=device.capability, vram_total=device.total_bytes) + hardware.pop('vram_free', None) + machine['hardware'] = hardware return root, plain_data(machine) def validate_request(prompt, width, height, seconds, seed, *, continue_from=None, frames=None, - two_pass=True, preview=False, refine_from=None, media=None): - if not isinstance(prompt, str) or not prompt.strip(): + two_pass=True, preview=False, refine_from=None, media=None, original_audio=False, + allow_empty_prompt=False): + # Only a request the user sent on purpose without a description may leave it empty. + if not isinstance(prompt, str) or not (prompt.strip() or allow_empty_prompt is True): raise ValueError('Enter a prompt describing the video and its audio.') if type(seed) is not int or not 0 <= seed <= 2**53 - 1: raise ValueError('Seed must be an integer between 0 and 9007199254740991.') @@ -869,7 +913,8 @@ def validate_request(prompt, width, height, seconds, seed, *, continue_from=None canvas = geometry(width, height, frames=frames, seconds=None if frames is not None else seconds) if continue_from is not None: from .continuation import validate_path, validate_route - validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from, media=media) + validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from, media=media, + original_audio=original_audio) validate_path(continue_from) return canvas @@ -1180,7 +1225,10 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, force_regenerate=False, base_steps=8, refine_steps=3, comfy_metadata=None, prompt_rewrite_report=None, preview=False, refine_from=None, report_extra=None, continue_from=None, frames=None, first_frame_reference=False, voice_reference=False, hide_tail=False, - step_cache=None, upscale_factor=2): + step_cache=None, upscale_factor=2, allow_empty_prompt=False): + from .comfy_environment import control_process + if control_process(environ): + raise RuntimeError('Generation requires a dispatched executor') from .tiers import STEP_CACHES from .two_pass import SCALES if upscale_factor not in SCALES: @@ -1222,6 +1270,8 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, refine_steps = int(argument(arguments, '--refine-steps', 3)) # The second pass never uses a step cache; the preview's own command keeps its flag. step_cache = argument(arguments, '--step-cache') + # The preview's own permission for an empty prompt travels with its saved request. + allow_empty_prompt = allow_empty_prompt or '--allow-empty-prompt' in arguments if preview or upscale: # A preview stays the quick half-size first pass, and its upscale continues that same pass. upscale_factor = 2 @@ -1229,7 +1279,8 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, validate_flags(preview, upscale, two_pass, base_steps) from .two_pass import validate_steps validate_steps(base_steps, refine_steps, two_pass) - canvas = validate_request(prompt, width, height, seconds, seed, continue_from=continue_from, frames=frames) + canvas = validate_request(prompt, width, height, seconds, seed, continue_from=continue_from, frames=frames, + allow_empty_prompt=allow_empty_prompt) source = Path(source or source_root()).resolve() root, machine = installation(source, environ) continuation = None @@ -1291,6 +1342,8 @@ def check_ending(): '--width', str(width), '--height', str(height), '--seconds', str(seconds), '--seed', str(seed), '--two-pass' if two_pass else '--no-two-pass', '--base-steps', str(base_steps), '--refine-steps', str(refine_steps)] + if allow_empty_prompt is True: + command.append('--allow-empty-prompt') if step_cache is not None: command += ['--step-cache', step_cache] if two_pass and upscale_factor != 2: @@ -1426,6 +1479,24 @@ def release_depth_models(): from .media_request import task_for from .two_pass import plan, sampling_canvas from .generate import hide_tail_geometry + from .original_audio import prepare_request + from copy import deepcopy + original_audio, audio_canvas = prepare_request(deepcopy(extra.get('media') or {}), width, height, + continuation=continue_from is not None, backend=machine.get('device_backend', 'cuda')) + if original_audio is not None: + canvas = audio_canvas + state['geometry'] = canvas + extra['original_audio'] = original_audio + flag = '--frames' if '--frames' in command else '--seconds' + position = command.index(flag) + command[position:position + 2] = (['--frames', str(canvas['frames'])] if 'segment' in original_audio else + ['--seconds', str((original_audio['end_sample'] - original_audio['start_sample']) / original_audio['sample_rate'])]) + # The sound used as is sets the length and is the part's one sound: the engine adds no hidden + # tail, no voice reference and no first frame of the film beside it (cast pictures stay). + first_frame_reference = voice_reference = hide_tail = anchored = False + for flag in ('--first-frame-reference', '--voice-reference', '--hide-tail'): + if flag in command: + command.remove(flag) generated_canvas, hidden = hide_tail_geometry(canvas, None if extra.get('conditioning') else prompt, hide_tail, preview=bool(preview or upscale)) hide_tail_frames = hidden['frames'] if hidden and hidden['used'] else 0 @@ -1439,7 +1510,11 @@ def release_depth_models(): from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, MISSING, validate_snapshot, summary validate_route(task='ref2va' if anchored or pictured else task_for(extra.get('media', {})), anchor=anchored or pictured, - media=extra.get('media')) + media=extra.get('media'), original_audio=original_audio is not None) + if original_audio is not None and continuation is not None: + # An encoded ending has no stored origin; the engine checks the part against its own. + from .original_audio_chain import validate_continuation + validate_continuation(original_audio, continuation) if continuation is not None: try: continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) @@ -1497,7 +1572,8 @@ def asset_progress(message): preparation_started = time.monotonic() try: # Reference audio selects its own tables; match the encoder's choice. - asset_task = ((ANCHOR_ADALN_TASK if anchored or pictured else ADALN_TASK) if continue_from is not None + asset_task = ((ANCHOR_ADALN_TASK if anchored or pictured else ADALN_TASK) + if continue_from is not None and original_audio is None else engine_task(extra.get('media', {}), media_base)) # Tables follow the step count; the cache is carried for exact level recognition. preparation = prepare_sampling_assets(root, machine, dict(planned, step_cache=step_cache) if step_cache else planned, @@ -1527,11 +1603,10 @@ def asset_progress(message): OWNER.close() environment.pop(ENV, None) else: - environment[ENV] = OWNER.start(root, source, machine['python'], environment) - OWNER.active = True - with (run / 'generate.log').open('wb') as log: + environment[ENV] = OWNER.start(root, source, machine['python'], environment, busy=True) + with _device_generation_lease(environment) as held, (run / 'generate.log').open('wb') as log: process = processes.popen(command, env=environment, cwd=source, stdin=subprocess.DEVNULL, - stdout=log, stderr=subprocess.STDOUT, start_new_session=True, supervise=True) + stdout=log, stderr=subprocess.STDOUT, start_new_session=True, supervise=True, pass_fds=held) state.update(status='running', pid=process.pid) save(run / 'comfy-request.json', state) while process.poll() is None: @@ -1598,7 +1673,11 @@ def asset_progress(message): # Dropping the video on the canvas restores this graph, as with ComfyUI's own video nodes. # Written before the result cache records the file's size and hash. from .comfy_metadata import embed_comfy_metadata - state['workflow_in_video'] = embed_comfy_metadata(output, **comfy_metadata) + timing = {} + if original_audio is not None: + from math import lcm + timing['exact_timescale'] = lcm(original_audio['sample_rate'], 24) + state['workflow_in_video'] = embed_comfy_metadata(output, **comfy_metadata, **timing) # The MP4 is saved and checked; its uncompressed frames and audio are no longer read. from .comfy_library import release_decoded, release_later state['decoded_release'] = release_decoded(run) diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 1b99620d..0dc9b560 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -48,6 +48,10 @@ class ComfyHost: def __init__(self, server): self.server = server + @property + def concurrent(self): + return getattr(self.server, '_freevideo_front', None) is not None + def continued(self, prompt, node): """A continued part's media panel keeps only the pictures it uses (prompt is this submission's copy).""" link = prompt[node]['inputs'].get('media') @@ -77,10 +81,15 @@ def inputs(self, block_inputs): # Each direction is filled once the node offers its input. if name in schema: values[name] = '' if independent else block_inputs[key] - if 'voice_reference' in schema and values.get('continue_from'): + # A part that plays its range of a sound used as is takes no first frame of the film, no voice + # reference and no hidden tail: the engine declines all three for it (generate.py). + original = bool(block_inputs.get('original_audio')) + if original and 'first_frame_reference' in schema: + values['first_frame_reference'] = False + if 'voice_reference' in schema and values.get('continue_from') and not original: values['voice_reference'] = True - if ('hide_tail' in schema and not independent and block_inputs.get('index', 0) < block_inputs.get('count', 0) - 1 - and continuation_supported()): + if ('hide_tail' in schema and not independent and not original + and block_inputs.get('index', 0) < block_inputs.get('count', 0) - 1 and continuation_supported()): # Only a backend that continues videos makes the extra tail frames; others ignore the option. values['hide_tail'] = True return values @@ -92,6 +101,9 @@ def work_exists(self, path): return output.is_relative_to(root) and output.is_file() def submit(self, prompt_id, prompt, extra_data): + front = getattr(self.server, '_freevideo_front', None) + if front is not None: + return front.submit_sync(prompt_id, prompt, extra_data) try: from comfy.cli_args import args except ImportError: @@ -134,12 +146,13 @@ def submit(self, prompt_id, prompt, extra_data): raise RuntimeError('Prompt validation failed') from None def status(self, prompt_id): + front = getattr(self.server, '_freevideo_front', None) queue = self.server.prompt_queue - running, pending = queue.get_current_queue() + running, pending = front.current_queue() if front is not None else queue.get_current_queue() for rows, state in ((running, 'running'), (pending, 'pending')): if any(row[1] == prompt_id for row in rows): return dict(state=state, error=None, output=None) - history = queue.get_history(prompt_id=prompt_id).get(prompt_id) + history = (front.history(prompt_id) if front is not None else queue.get_history(prompt_id=prompt_id)).get(prompt_id) if history is None: return dict(state='missing', error=None, output=None) status = history.get('status') @@ -163,6 +176,9 @@ def status(self, prompt_id): return dict(state='error', error=message or 'Prompt execution failed', output=None) def cancel(self, prompt_id, node): + front = getattr(self.server, '_freevideo_front', None) + if front is not None: + return front.cancel(prompt_id, node) from .comfy_progress import cancel_request import nodes return cancel_request(self.server.prompt_queue, prompt_id, node, nodes.interrupt_processing) @@ -176,6 +192,15 @@ def join(self, record, cancelled): blocks = chain_store.leading_blocks(record) # A planned chain shows exactly the requested length: what the last block made beyond it is left out. frames = planned['frames'] if planned and not planned['single'] and len(blocks) == len(record['blocks']) else None + if record.get('original_audio') is not None: + # The original selection replaces every part's sound: seam leads and voice levels do not apply. + from .original_audio_chain import joined_info + info = joined_info(record, blocks) + frames = ((info['end_sample'] - info['start_sample']) * 24 + info['sample_rate'] - 1) // info['sample_rate'] + if len(blocks) < len(record['blocks']): + frames = sum(block['new_frames'] for block in blocks) + return join_outputs([block['output'] for block in blocks], record['id'], cancelled, + frames=frames, replace=record.get('output'), original_audio=info) kept = {'leads': record['lead_reader'], 'expected_seams': record.get('expected_seams')} \ if record.get('lead_reader') is not None else {} from .chain_level import voiced @@ -266,12 +291,37 @@ def source(value): if isinstance(value, str): return dict(type='block', id=value) return dict(type='work') if value is not None else None - return dict(index=chain['index'], count=chain['count'], - after=source(chain['after']), before=source(chain['before'])) + value = dict(index=chain['index'], count=chain['count'], + after=source(chain['after']), before=source(chain['before'])) + if chain.get('_dispatch'): + value['freevideo_dispatch'] = deepcopy_dispatch(chain['_dispatch']) + return value + + +def deepcopy_dispatch(dispatch): + from copy import deepcopy + return deepcopy({key: dispatch[key] for key in ('task_id', 'executor_id', 'device', 'logical_device') + if key in dispatch}) + + +def execution_dispatch(server): + """Private workers receive context in the prompt, never a second durable runner.""" + if server is None or not os.environ.get('FREEVIDEO_EXECUTOR_DEVICE'): + return None + running, _ = server.prompt_queue.get_current_queue() + row = next((row for row in running if row[1] == server.last_prompt_id), None) + return row[3].get('freevideo_dispatch') if row is not None else None def block_context(server, node_id): try: + dispatch = execution_dispatch(server) + if dispatch is not None: + chain = dispatch.get('chain_context') + if dispatch.get('chain_node', str(node_id)) != str(node_id): + return None + return dict(chain, _prompt_id=server.last_prompt_id, _remote=True, + _dispatch=deepcopy_dispatch(dispatch)) if chain is not None else None if RUNNER is not None and server is not None: prompt_id = server.last_prompt_id chain = RUNNER.context(prompt_id, node_id) @@ -283,6 +333,16 @@ def block_context(server, node_id): def block_finished(chain, node_id, output, report_id, continuation=None): try: + if chain is not None and chain.get('_remote'): + from server import PromptServer + data = dict( + prompt_id=chain['_prompt_id'], task_id=chain['_dispatch']['task_id'], + node=str(node_id), output=output, report_id=report_id, continuation=continuation) + dispatch = execution_dispatch(PromptServer.instance) + if dispatch is not None: + dispatch.setdefault('finished', {})[str(node_id)] = data + PromptServer.instance.send_sync('freevideo_chain_finished', data) + return if RUNNER is not None and chain is not None: # Capture the executing attempt, so a late callback cannot finish a retry. RUNNER.finished(chain['_prompt_id'], node_id, output, report_id, continuation) @@ -308,7 +368,7 @@ def register(host_factory=None): from server import PromptServer from .comfy_bridge import installation, installation_root server = PromptServer.instance - if server is None or getattr(server, '_freevideo_chain', False): + if server is None or getattr(server, '_freevideo_chain', False) or os.environ.get('FREEVIDEO_EXECUTOR_DEVICE'): return fake = host_factory is not None root = installation_root() if fake else installation()[0] @@ -357,6 +417,11 @@ async def plan(request): async def create(request): try: value = await body(request) + from .audio_use_request import graph_audio, prepare_chain + if isinstance(value.get('prompt'), dict) and value.get('node') in value['prompt']: + if graph_audio(value['prompt'], value['node']): + import folder_paths + value = await asyncio.to_thread(prepare_chain, value, folder_paths.get_input_directory()) normalized = chain_store.validate(value) if not fake and os.environ.get('FREEVIDEO_CHAIN_INDEPENDENT') != '1': missing = {key for row in normalized['blocks'] for key in CONTINUE_INPUTS @@ -377,7 +442,7 @@ async def create(request): except chain_plan.PlanError as failure: return response({'error': str(failure), 'code': failure.code}, 400) except ValueError as failure: - return response({'error': str(failure)}, 400) + return response({'error': str(failure), **(failure.as_dict() if hasattr(failure, 'as_dict') else {})}, 400) def asking(request): # ?client_id=... marks each chain as this page's own or not. diff --git a/freevideo_engine/comfy_environment.py b/freevideo_engine/comfy_environment.py index f3d1625b..14c9d86a 100644 --- a/freevideo_engine/comfy_environment.py +++ b/freevideo_engine/comfy_environment.py @@ -1,9 +1,41 @@ """Keep ComfyUI process configuration out of managed engine children.""" import os +import shutil from pathlib import Path import sys +def with_private_compiler(root, env): + """Without a system compiler, tools that build at run time (Triton) use the one FreeVideo installed.""" + private = Path(root) / 'tools' / 'gcc-bin' / 'gcc' + if os.name != 'nt' and 'CC' not in env and private.exists() and not shutil.which('gcc', path=env.get('PATH')): + env['CC'] = str(private) + return env + + +EXECUTOR_DEVICE = 'FREEVIDEO_EXECUTOR_DEVICE' +POOL_CONTROL = 'FREEVIDEO_POOL_CONTROL' + + +def executor_device(environ=None): + return (os.environ if environ is None else environ).get(EXECUTOR_DEVICE, '') + + +def control_process(environ=None): + env = os.environ if environ is None else environ + return not executor_device(env) and env.get(POOL_CONTROL) == '1' + + +def configure_control(command, environment): + """Configure before Comfy imports Torch; leave the single-device launch intact.""" + from .gpu_pool import pool_enabled + if executor_device(environment) or not pool_enabled(environment): + return command, environment + env = dict(environment, FREEVIDEO_POOL_CONTROL='1', FREEVIDEO_BACKGROUND_COMPILE='off', + FREEVIDEO_PROMPT_PRESTART='off') + return [*command, '--cpu'], env + + def isolated_environment(root, source, environ=None): """Remove inherited Python/package targets only in the engine's child.""" from .proxy import inherited @@ -34,6 +66,7 @@ def isolated_environment(root, source, environ=None): env.pop(name, None) env.update(FREEVIDEO_HOME=str(root), FREEVIDEO_PROXY_MODE=proxy_mode, PYTHONPATH=str(source), PYTHONNOUSERSITE='1', PYTHONUTF8='1', PYTHONIOENCODING='utf-8', NO_COLOR='1', FREEVIDEO_UI_EVENTS='1') + with_private_compiler(root, env) from .triton_compat import temporary_environment # Pruning sys.prefix above also prunes the frozen launcher's static curl. # Restore its fallback for managed Python after isolating Python packages. diff --git a/freevideo_engine/comfy_front.py b/freevideo_engine/comfy_front.py new file mode 100644 index 00000000..590c66b1 --- /dev/null +++ b/freevideo_engine/comfy_front.py @@ -0,0 +1,760 @@ +"""Control-process prompt registry; private executors never own public queue state.""" +import asyncio +from copy import deepcopy +from dataclasses import dataclass +import inspect +import json +import logging +import os +import re +import signal +import threading +import uuid + + +# The error panel recognises these texts; keep them in step (web/error_panel.js). +NO_EXECUTOR = 'No GPU is available: every GPU process has stopped. Restart FreeVideo.' +EXITED_AGAIN = 'The GPU process exited again while running this task, so it was not moved to another GPU' +LOG_ERROR = re.compile(r'\b\w*(?:Error|Exception)\b:|Fatal Python error|Segmentation fault|CUDA error|\bKilled\b|\bAborted\b') + + +class PromptRejected(ValueError): + def __init__(self, body): + super().__init__('Prompt validation failed') + self.body = body + + +@dataclass +class Submission: + task: object + number: float + prompt: dict + extra: dict + outputs: list + public_extra: dict + state: str = 'pending' + worker: object = None + worker_prompt_id: str | None = None + result: dict | None = None + chain_notified: bool = False + cancelled: bool = False + # Exit causes of executors that stopped while running this task; one move is allowed. + exits: tuple = () + + +def _last_error(path, limit=65536): + """The last error-looking line of an executor log, if any.""" + try: + with open(path, 'rb') as handle: + handle.seek(max(0, os.fstat(handle.fileno()).st_size - limit)) + lines = handle.read().decode('utf-8', 'replace').splitlines() + except (OSError, TypeError, ValueError): + return None + line = next((line.strip() for line in reversed(lines) if LOG_ERROR.search(line)), None) + return line[:300] if line else None + + +def exit_cause(worker): + """Exit code (with its signal name, or hex for Windows status codes) and the last logged error.""" + code = getattr(getattr(worker, 'process', None), 'poll', lambda: None)() + if code is None: + cause = 'exit code unknown' + elif code < 0: + try: + cause = 'exit code %d (%s)' % (code, signal.Signals(-code).name) + except ValueError: + cause = 'exit code %d' % code + elif code > 255: + cause = 'exit code %d (0x%08X)' % (code, code & 0xFFFFFFFF) + else: + cause = 'exit code %d' % code + error = _last_error(getattr(worker, 'log', None)) + return cause + ('; last error: ' + error if error else '') + + +class Front: + """Adapt the B/C contracts without changing Comfy's local execution queue. + + envelope(prompt, extra) is a trusted, read-only policy adapter. Unknown + graphs retain Envelope=None and therefore use the scheduler's exclusive lane. + Only confirmed worker history releases an assigned device or reservation. + """ + def __init__(self, server, scheduler, workers, *, envelope=None, validate=None, + run_prompt=None, interrupt=None, worker_history=None, worker_lookup=None): + self.server, self.scheduler, self.workers = server, scheduler, workers + self.envelope, self.validate = envelope, validate + if run_prompt is None or interrupt is None or worker_history is None or worker_lookup is None: + from .comfy_pool import (run_prompt as submit_worker, interrupt as cancel_worker, + worker_history as read_history, worker_lookup as find_prompt) + run_prompt = run_prompt or submit_worker + interrupt = interrupt or cancel_worker + worker_history = worker_history or read_history + worker_lookup = worker_lookup or find_prompt + self.run_prompt, self.interrupt, self.worker_history = run_prompt, interrupt, worker_history + self.worker_lookup = worker_lookup + self.records, self.reports, self.seen = {}, {}, {} + self.lock = threading.RLock() + self.dispatch_lock = asyncio.Lock() + self.loop = getattr(server, 'loop', None) + self.closed = False + self.background = set() + # Devices whose executor exited. Their work moves to the others until the control process restarts. + self.unavailable = set() + + def accepting(self): + # Busy devices and host pressure belong in the scheduler's pending queue. + return not self.closed + + async def submit(self, value, headers=None): + from .gpu_scheduler import Task + value = deepcopy(value) + if not isinstance(value, dict) or not isinstance(value.get('prompt'), dict): + raise PromptRejected({'error': {'type': 'no_prompt', 'message': 'No prompt provided'}, 'node_errors': {}}) + trigger = getattr(self.server, 'trigger_on_prompt', None) + if trigger is not None: + value = trigger(value) + identity = value.get('prompt_id') + if identity is None: + identity = str(uuid.uuid4()) + try: + if str(uuid.UUID(identity)) != identity: + raise ValueError + except (ValueError, TypeError, AttributeError): + raise PromptRejected({'error': {'type': 'invalid_prompt_id', 'message': 'prompt_id must be a valid UUID'}, + 'node_errors': {}}) from None + prompt, extra = value['prompt'], deepcopy(value.get('extra_data') or {}) + if not isinstance(extra, dict): + raise PromptRejected({'error': {'type': 'invalid_request', 'message': 'Invalid extra_data'}, 'node_errors': {}}) + # Execution metadata may only be minted by this control process. + extra.pop('freevideo_dispatch', None) + if 'client_id' in value: + extra['client_id'] = value['client_id'] + if headers and headers.get('Comfy-Usage-Source'): + extra.setdefault('comfy_usage_source', headers['Comfy-Usage-Source']) + replace = getattr(self.server, 'node_replace_manager', None) + if replace is not None: + replace.apply_replacements(prompt) + if self.validate is None: + import execution + validate = execution.validate_prompt + sensitive = execution.SENSITIVE_EXTRA_DATA_KEYS + else: + validate, sensitive = self.validate, ('auth_token_comfy_org', 'api_key_comfy_org') + partial = value.get('partial_execution_targets') + # run_prompt has no partial-target argument; do not silently execute a larger graph. + if partial is not None: + raise PromptRejected({'error': {'type': 'invalid_request', 'message': 'Partial execution is unavailable'}, + 'node_errors': {}}) + valid = validate(identity, prompt, partial) + if inspect.isawaitable(valid): + valid = await valid + if not valid[0]: + raise PromptRejected({'error': valid[1], 'node_errors': valid[3]}) + public = {key: deepcopy(item) for key, item in extra.items() if key not in sensitive} + chain = extra.get('freevideo_chain') or {} + envelope = self.envelope(prompt, extra) if self.envelope is not None else None + task = Task(id=identity, kind='prompt', envelope=envelope, depends_on=(), + chain_id=chain.get('id'), client_id=extra.get('client_id') or '') + with self.lock: + if self.closed: + raise RuntimeError('Executors are stopping') + if self.unavailable and not self._any_executor(): + raise PromptRejected({'error': {'type': 'executors_unavailable', 'message': NO_EXECUTOR}, + 'node_errors': {}}) + old = self.records.get(identity) + if old is not None: + if old.prompt != prompt or old.extra != extra: + raise PromptRejected({'error': {'type': 'invalid_prompt_id', 'message': 'Prompt ID already used'}, + 'node_errors': {}}) + return dict(prompt_id=identity, number=old.number, node_errors=valid[3]) + number = float(value['number']) if 'number' in value else self.server.number + if 'number' not in value: + self.server.number += 1 + if value.get('front'): + number = -number + row = Submission(task, number, prompt, extra, valid[2], public) + self.scheduler.submit(task) + self.records[identity] = row + pending = [r for r in self.records.values() if r.state == 'pending'] + if pending != sorted(pending, key=lambda r: r.number): + self._requeue_pending(sorted(pending, key=lambda r: r.number)) + self.updated() + return dict(prompt_id=identity, number=number, node_errors=valid[3]) + + def submit_sync(self, prompt_id, prompt, extra): + if self.loop is None or not self.loop.is_running(): + raise RuntimeError('Control event loop unavailable') + future = asyncio.run_coroutine_threadsafe(self.submit(dict( + prompt_id=prompt_id, prompt=prompt, extra_data=extra, + client_id=extra.get('client_id', ''))), self.loop) + # No timeout/retry here: an accepted request must never be replayed. + return future.result()['prompt_id'] + + def updated(self): + self.server.queue_updated() + + @staticmethod + def receipt(row): + if row.worker is None: + return dict(task_id=row.task.id) + device = row.worker.device + return dict(task_id=row.task.id, executor_id=device.uuid, worker_prompt_id=row.worker_prompt_id, + device=dict(uuid=device.uuid, index=device.index, name=device.name), logical_device=0) + + def _queue_row(self, row): + extra = dict(deepcopy(row.public_extra), freevideo_dispatch=self.receipt(row)) + if row.state in ('submitting', 'running', 'cancelling', 'unconfirmed'): + from .comfy_progress import STATE + extra['freevideo_progress'] = {message['node']: message for message in STATE.snapshot() + if message.get('task_id') == row.task.id} + return (row.number, row.task.id, deepcopy(row.prompt), extra, deepcopy(row.outputs)) + + def current_queue(self): + with self.lock: + running = [self._queue_row(row) for row in self.records.values() + if row.state in ('submitting', 'running', 'cancelling', 'unconfirmed')] + pending = [self._queue_row(row) for row in self.records.values() if row.state == 'pending'] + return running, sorted(pending, key=lambda row: row[0]) + + def queue_info(self): + running, pending = self.current_queue() + return {'exec_info': {'queue_remaining': len(running) + len(pending)}, + 'freevideo_busy': self.busy()} + + def busy(self): + running, pending = self.current_queue() + if running or pending: + return True + from .comfy_chain import RUNNER + if RUNNER is not None: + with RUNNER.lock: + if any(row['state'] in ('queued', 'running', 'cancelling') or row['output_state'] == 'joining' + for row in RUNNER.records.values()): + return True + jobs = getattr(self.server, '_freevideo_agent', None) + return jobs is not None and bool(jobs.active()) + + def history(self, prompt_id=None, max_items=None, offset=-1): + with self.lock: + rows = [(identity, deepcopy(row.result)) for identity, row in self.records.items() if row.result is not None] + if prompt_id is not None: + return {identity: result for identity, result in rows if identity == prompt_id} + if max_items is not None and offset < 0: + offset = max(0, len(rows) - max_items) + elif offset < 0: + offset = 0 + return dict(rows[offset:offset + max_items if max_items is not None else None]) + + def _matches(self, row, node): + return node is None or row.prompt.get(str(node), {}).get('class_type') == 'FreeVideoGenerate' + + def _background(self, coro): + def schedule(): + task = asyncio.create_task(coro) + self.background.add(task) + task.add_done_callback(self.background.discard) + if self.loop is not None and self.loop.is_running(): + self.loop.call_soon_threadsafe(schedule) + else: + coro.close() + raise RuntimeError('Control event loop unavailable') + + async def _interrupt(self, row): + try: + state = await asyncio.to_thread(self.interrupt, row.worker, row.worker_prompt_id) + except Exception: + with self.lock: + if row.result is None: + row.state = 'unconfirmed' + self.updated() + return + if state == 'removed': + # The executor deleted it before starting: no history will ever follow. + self._finish(row, 'execution_interrupted', {}) + + def _finish(self, row, kind, detail): + with self.lock: + if row.result is not None: + return + row.state = 'finished' + row.result = dict(prompt=self._queue_row(row), outputs={}, + status=dict(status_str='error', completed=False, + messages=[[kind, dict(detail, prompt_id=row.task.id)]]), + freevideo_dispatch=self.receipt(row)) + self.scheduler.finished(row.task.id, False) + if kind == 'execution_error': + # No executor will report this failure, so the page hears it from here. + node = next((key for key, value in row.prompt.items() if value.get('class_type') == 'FreeVideoGenerate'), None) + self.server.send_sync('execution_error', dict( + prompt_id=row.task.id, task_id=row.task.id, node_id=node, node_type=node and 'FreeVideoGenerate', + exception_type='RuntimeError', traceback=[], executed=[], current_inputs={}, current_outputs={}, + **detail), row.task.client_id or None) + self.updated() + + @staticmethod + def _exited(worker): + poll = getattr(getattr(worker, 'process', None), 'poll', None) + return poll is not None and poll() is not None + + def _recovered(self, row, worker_id): + # An acknowledgement lost after acceptance is resolved from the executor's own records. + with self.lock: + if row.worker_prompt_id is not None or row.result is not None: + return False + row.worker_prompt_id = worker_id + row.state = 'cancelling' if row.cancelled else 'running' + return True + + def cancel(self, prompt_id, node=None): + with self.lock: + row = self.records.get(prompt_id) + if row is None or not self._matches(row, node) or row.result is not None: + return 'finished' + if row.state == 'cancelling': + return 'cancelling' + state = self.scheduler.cancel(prompt_id) + if state == 'queued': + row.state = 'finished' + row.result = dict(prompt=self._queue_row(row), outputs={}, + status=dict(status_str='error', completed=False, + messages=[['execution_interrupted', {'prompt_id': prompt_id}]]), + freevideo_dispatch=self.receipt(row)) + elif state == 'running': + row.state, row.cancelled = 'cancelling', True + if row.worker_prompt_id is not None: + self._background(self._interrupt(row)) + else: + return 'finished' + self.updated() + return 'cancelled' if state == 'queued' else 'cancelling' + + def move_first(self, prompt_id, node=None, protected=lambda identity: False): + with self.lock: + row = self.records.get(prompt_id) + if row is None or row.result is not None or not self._matches(row, node): + return 'finished' + if row.state != 'pending': + return 'running' + if protected(prompt_id): + return 'unavailable' + pending = sorted((r for r in self.records.values() if r.state == 'pending'), key=lambda r: r.number) + if pending[0] is row: + return 'first' + # The contract has no reprioritization method. Requeue only pending + # tasks, under the same lock used by submit/dispatch/cancel. + row.number = pending[0].number - 1 + self._requeue_pending([row] + [item for item in pending if item is not row]) + self.updated() + return 'moved' + + def _requeue_pending(self, pending): + reorder = getattr(self.scheduler, 'reprioritize', None) + if reorder is not None: + reorder([row.task.id for row in pending]) + return + for item in pending: + if self.scheduler.cancel(item.task.id) != 'queued': + raise RuntimeError('Scheduler pending state changed') + for item in pending: + self.scheduler.submit(item.task) + + def auxiliary_busy(self): + service = getattr(self.server, '_freevideo_prompt_vlm', None) + if service is not None and service.busy(): + return True + # Use the same image-service lookup as Agent jobs; importing this adapter + # does not start either service. + from .agent.comfy import ComfyPort + port = ComfyPort.__new__(ComfyPort) + port.server = self.server + return port.image_busy() + + async def pump(self): + async with self.dispatch_lock: + with self.lock: + running = [row for row in self.records.values() if row.worker_prompt_id is not None + and row.result is None] + await asyncio.gather(*(self._poll(row) for row in running)) + with self.lock: + lost = [row for row in self.records.values() if row.state in ('unconfirmed', 'cancelling') + and row.worker is not None and row.worker_prompt_id is None and row.result is None] + await asyncio.gather(*(self._reconcile(row) for row in lost)) + self._reap_exited() + auxiliary_busy = self.auxiliary_busy() + with self.lock: + assignments = self.scheduler.dispatch() if not self.closed and not auxiliary_busy else [] + workers = {worker.device.uuid: worker for worker in self.workers()} + selected, moved = [], False + for task, device in assignments: + row = self.records[task.id] + row.worker, row.state = workers.get(device.uuid), 'submitting' + if row.worker is not None and self._exited(row.worker): + # Nothing was sent: put it back for another device. + self._mark_unavailable(row.worker) + self._requeue(row) + moved = True + continue + selected.append(row) + if moved: + self._requeue_pending(sorted((r for r in self.records.values() if r.state == 'pending'), + key=lambda r: r.number)) + self._fail_unplaceable() + await asyncio.gather(*(self._dispatch(row) for row in selected)) + if assignments: + self.updated() + + async def _reconcile(self, row): + try: + worker_id = await asyncio.to_thread(self.worker_lookup, row.worker, row.task.id) + except Exception: + return + if isinstance(worker_id, str) and worker_id and self._recovered(row, worker_id) and row.cancelled: + await self._interrupt(row) + + def _any_executor(self): + return any(worker.device.uuid not in self.unavailable for worker in self.workers()) + + def _mark_unavailable(self, worker): + uuid = worker.device.uuid + if uuid in self.unavailable: + return + self.unavailable.add(uuid) + mark = getattr(self.scheduler, 'mark_unavailable', None) + if mark is not None: + mark(uuid) + if any(other.device.uuid not in self.unavailable and not self._exited(other) for other in self.workers()): + logging.warning('[FreeVideo] The executor for GPU %s exited (%s); its tasks move to the remaining GPUs', + worker.device.index, exit_cause(worker)) + else: + logging.warning('[FreeVideo] The executor for GPU %s exited (%s); no GPU remains', + worker.device.index, exit_cause(worker)) + + def _requeue(self, row): + # The executor is gone, so the attempt cannot still run: it may start again elsewhere. + row.worker, row.worker_prompt_id, row.state = None, None, 'pending' + self.scheduler.finished(row.task.id, False, retry=True) + + def _reap_exited(self): + """A confirmed executor exit makes its device unavailable and moves its work to the others.""" + failed = [] + with self.lock: + for worker in self.workers(): + if self._exited(worker): + self._mark_unavailable(worker) + dead = [row for row in self.records.values() if row.result is None and row.worker is not None + and row.state in ('running', 'cancelling', 'unconfirmed') + and row.worker.device.uuid in self.unavailable] + moved = False + for row in dead: + if row.cancelled: + failed.append((row, 'execution_interrupted', {})) + elif row.chain_notified: + # Its part already reported a result; running it again would repeat that part. + failed.append((row, 'execution_error', dict(exception_message='Executor exited'))) + else: + row.exits += ('GPU %s: %s' % (row.worker.device.index, exit_cause(row.worker)),) + if len(row.exits) > 1: + # The task itself is the likely cause (host memory, a native crash): + # moving it again would take down the next GPU as well. + failed.append((row, 'execution_error', dict( + exception_message='%s (%s).' % (EXITED_AGAIN, ' / '.join(row.exits))))) + continue + self._requeue(row) + moved = True + if moved: + self._requeue_pending(sorted((r for r in self.records.values() if r.state == 'pending'), + key=lambda r: r.number)) + for row, kind, detail in failed: + self._finish(row, kind, detail) + if dead: + self.updated() + + def _fail_unplaceable(self): + # With every executor gone nothing could ever start: say so instead of waiting silently. + unplaceable = getattr(self.scheduler, 'unplaceable', None) + if not self.unavailable or unplaceable is None: + return + with self.lock: + rows = [self.records[identity] for identity in unplaceable() + if identity in self.records and self.records[identity].result is None] + rows = [row for row in rows if self.scheduler.cancel(row.task.id) == 'queued'] + for row in rows: + self._finish(row, 'execution_error', dict(exception_message=NO_EXECUTOR)) + + async def _dispatch(self, row): + if row.worker is None: + row.state = 'unconfirmed' + return + dispatch = self.receipt(row) + chain = row.extra.get('freevideo_chain') or {} + if chain: + from .comfy_chain import RUNNER + if RUNNER is not None: + node = chain.get('node') or next((node for node, value in row.prompt.items() + if value.get('class_type') == 'FreeVideoGenerate'), None) + dispatch['chain_context'] = RUNNER.context(row.task.id, node) + dispatch['chain_node'] = node + try: + worker_id = await asyncio.to_thread(self.run_prompt, row.worker, deepcopy(row.prompt), + client_id=row.task.client_id, task_id=row.task.id, + extra=dict(deepcopy(row.extra), freevideo_dispatch=dispatch)) + if not isinstance(worker_id, str) or not worker_id: + raise RuntimeError('Executor returned no prompt ID') + with self.lock: + row.worker_prompt_id = worker_id + cancelled = row.state == 'cancelling' + row.state = 'cancelling' if cancelled else 'running' + if cancelled: + await self._interrupt(row) + except Exception: + # A lost acknowledgement does not prove the prompt was rejected. + # Retain its reservation and never automatically replay it. + with self.lock: + row.state = 'unconfirmed' + + async def _poll(self, row): + try: + history = await asyncio.to_thread(self.worker_history, row.worker, row.worker_prompt_id) + except Exception: + return + if not isinstance(history, dict): + return + # Accept both a single history entry and the native keyed response. + history = history.get(row.worker_prompt_id, history) + if not isinstance(history, dict) or 'status' not in history: + return + status = history.get('status') or {} + if status.get('status_str') not in ('success', 'error'): + return + result = deepcopy(history) + result['freevideo_dispatch'] = self.receipt(row) + # The worker also retains node callbacks in native history, so a dropped + # WebSocket event cannot lose continuation/seam receipts. + prompt = result.get('prompt') or [] + extra = prompt[3] if len(prompt) > 3 and isinstance(prompt[3], dict) else {} + finished = (extra.get('freevideo_dispatch') or {}).get('finished') or {} + if not row.chain_notified: + for node, data in finished.items(): + self._chain_finished(row, node, data) + if prompt: + result['prompt'] = list(prompt[:5]) + if len(result['prompt']) > 3: + result['prompt'][3] = {key: value for key, value in extra.items() + if key not in ('auth_token_comfy_org', 'api_key_comfy_org')} + with self.lock: + row.result, row.state = result, 'finished' + self.scheduler.finished(row.task.id, status.get('status_str') == 'success') + self.updated() + + def _event_row(self, worker, data): + prompt_id = data.get('prompt_id') or data.get('worker_prompt_id') + candidates = [row for row in self.records.values() if row.worker is worker and row.result is None] + return next((row for row in candidates if row.worker_prompt_id == prompt_id + or row.task.id == data.get('task_id') or row.task.id == prompt_id), None) + + def event(self, worker, event): + kind, data = event.get('type'), event.get('data') + if not isinstance(data, dict): + return + with self.lock: + row = self._event_row(worker, data) + if row is None: + return + sent = data.get('prompt_id') + recovered = False + if (row.state in ('unconfirmed', 'cancelling') and row.worker_prompt_id is None + and isinstance(sent, str) and sent and sent != row.task.id): + row.worker_prompt_id = sent + row.state = 'cancelling' if row.cancelled else 'running' + recovered = row.cancelled + data = dict(deepcopy(data), prompt_id=row.task.id, task_id=row.task.id, + worker_prompt_id=data.get('prompt_id') or row.worker_prompt_id, + executor_id=worker.device.uuid) + token = data.get('report_id') + if isinstance(token, str): + # A token is scoped to its executor; do not let another worker + # replace an already registered report owner. + previous = self.reports.get(token) + if previous is None or previous is row: + self.reports[token] = row + if recovered: + self._background(self._interrupt(row)) + if kind == 'freevideo_progress': + key = (row.task.id, str(data.get('node')), data.get('stream_id')) + sequence = data.get('sequence') + if isinstance(sequence, int): + if self.seen.get(key, -1) >= sequence: + return + self.seen[key] = sequence + from .comfy_progress import STATE + from .comfy_chain import RUNNER + if RUNNER is not None: + RUNNER.context(row.task.id, data.get('node')) + chain = RUNNER.progress(data.get('node'), data, prompt_id=row.task.id) + if chain is not None: + data['chain'] = chain + data = STATE.publish(data.get('node'), data) + elif kind == 'freevideo_chain_finished': + self._chain_finished(row, data.get('node'), data) + return + elif kind == 'status': + self.updated() + return + if kind == 'freevideo_progress': + self.server.send_sync(kind, data) + else: + self.server.send_sync(kind, data, row.task.client_id or None) + + def _chain_finished(self, row, node, data): + if row.chain_notified: + return + from .comfy_chain import RUNNER + if RUNNER is not None: + RUNNER.finished(row.task.id, node, data.get('output'), data.get('report_id'), data.get('continuation')) + row.chain_notified = True + token = data.get('report_id') + if isinstance(token, str): + self.reports.setdefault(token, row) + + async def download(self, token, *, preview=False): + with self.lock: + row = self.reports.get(token) + if row is None: + raise KeyError('Unknown report') + worker = row.worker + if worker is None: + # Its executor exited and the task is waiting for another GPU. + raise KeyError('Report unavailable') + import aiohttp + from urllib.parse import quote + path = '/freevideo/%s/%s' % ('frame-preview' if preview else 'report', quote(token, safe='')) + async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=30)) as session: + async with session.get(worker.base_url + path) as response: + if response.status != 200: + raise KeyError('Report unavailable') + data = await response.read() + if not preview: + report = json.loads(data) + report['freevideo_dispatch'] = self.receipt(row) + data = json.dumps(report, ensure_ascii=False).encode('utf-8') + return data + + +def register(server, *, scheduler=None, supervisor=None, envelope=None): + """Install only after pool_enabled; single-device callers never import here.""" + from aiohttp import web + from .comfy_pool import PoolSupervisor, worker_progress_stream + if getattr(server, '_freevideo_front', None) is not None: + return server._freevideo_front + if scheduler is None: + from .admission import AdmissionLedger + from .gpu_pool import eligible_devices + from .gpu_scheduler import Scheduler + from .pool_resources import probe_host_memory + scheduler = Scheduler(eligible_devices(), AdmissionLedger(probe_host_memory)) + if supervisor is None: + supervisor = getattr(server, '_freevideo_pool', None) + owns_supervisor = supervisor is None + if owns_supervisor: + from .gpu_pool import eligible_devices + from .comfy_bridge import installation_root + from .comfy_service import source_root + supervisor = PoolSupervisor(eligible_devices(), root=installation_root(), source=source_root()) + server._freevideo_pool = supervisor + server._freevideo_scheduler = scheduler + front = Front(server, scheduler, supervisor.workers, + envelope=envelope or getattr(server, '_freevideo_task_envelope', None)) + server._freevideo_front = front + server.get_queue_info = front.queue_info + + @web.middleware + async def middleware(request, handler): + path = request.path.removeprefix('/api') if request.path.startswith('/api/') else request.path + if request.method == 'POST' and path == '/prompt': + try: + return web.json_response(await front.submit(await request.json(), request.headers)) + except PromptRejected as error: + return web.json_response(error.body, status=400) + if request.method == 'GET' and path == '/queue': + running, pending = front.current_queue() + return web.json_response(dict(queue_running=running, queue_pending=pending, freevideo_busy=front.busy())) + if request.method == 'GET' and (path == '/history' or path.startswith('/history/')): + identity = path[len('/history/'):] if path.startswith('/history/') else None + try: + maximum = int(request.query['max_items']) if 'max_items' in request.query else None + offset = int(request.query.get('offset', -1)) + return web.json_response(front.history(identity, maximum, offset)) + except ValueError: + raise web.HTTPBadRequest() from None + if request.method == 'POST' and path in ('/queue', '/interrupt', '/history'): + value = await request.json() + if path == '/queue': + ids = value.get('delete', []) + if value.get('clear'): + ids = [row[1] for row in front.current_queue()[1]] + pending = {row[1] for row in front.current_queue()[1]} + for identity in ids: + if identity in pending: + front.cancel(identity) + elif path == '/interrupt': + identity = value.get('prompt_id') + # Native Comfy's global stop explicitly targets all current + # prompts. Resolve each owned task; never broadcast to an idle + # worker or interrupt a task accepted after this snapshot. + ids = [identity] if identity else [row[1] for row in front.current_queue()[0]] + for identity in ids: + front.cancel(identity) + else: + with front.lock: + for identity in list(front.records) if value.get('clear') else value.get('delete', []): + row = front.records.get(identity) + if row is not None and row.result is not None: + del front.records[identity] + return web.json_response({}) + return await handler(request) + + server.app.middlewares.append(middleware) + + async def stream(worker): + while not front.closed and worker.device.uuid not in front.unavailable: + try: + async for event in worker_progress_stream(worker): + front.event(worker, event) + except Exception: + pass # A disconnected worker remains busy until its history confirms exit. + if not front.closed: + await asyncio.sleep(1) + + async def poll(): + last_error = None + while not front.closed: + try: + await front.pump() + last_error = None + except Exception as error: + # Keep reservations on protocol errors; never report an unknown worker idle. + kind = type(error).__name__ + if kind != last_error: + logging.warning('[FreeVideo] Pool scheduling pass failed (%s)', kind) + last_error = kind + await asyncio.sleep(.5) + + async def lifecycle(app): + front.loop = asyncio.get_running_loop() + tasks = [] + try: + if owns_supervisor: + await asyncio.to_thread(supervisor.start) + tasks = [asyncio.create_task(stream(worker)) for worker in supervisor.workers()] + tasks.append(asyncio.create_task(poll())) + yield + finally: + front.closed = True + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + await asyncio.gather(*list(front.background), return_exceptions=True) + if owns_supervisor: + await asyncio.to_thread(supervisor.stop) + server.app.cleanup_ctx.append(lifecycle) + return front diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index f2006644..d5764a7d 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -495,12 +495,19 @@ def launcher_comfy(selected, info): def queue_busy(url): - """Whether ComfyUI is running or holding jobs. An unreadable queue counts as idle.""" + """Preserve the legacy check; a pool must confirm every executor is idle.""" + from .gpu_pool import pool_enabled + pooled = pool_enabled() try: value = get_json(loopback_url(url) + '/queue', timeout=3) - return bool(value.get('queue_running') or value.get('queue_pending')) - except (OSError, ValueError, AttributeError): + if value.get('queue_running') or value.get('queue_pending'): + return True + if pooled: + state = get_json(loopback_url(url) + '/freevideo/pool', timeout=3) + return state.get('busy') is not False or state.get('confirmed') is not True return False + except (OSError, ValueError, AttributeError): + return pooled @plain_errors @@ -1402,6 +1409,8 @@ def _connect_comfy(self): custom_nodes='FreeVideo' if scoped else 'all') if selected.get('portable') and not selected['separate']: command.append('--windows-standalone-build') + from .comfy_environment import configure_control + command, env = configure_control(command, env) view = launch_view.for_engine(selected['engine']) started = command if view.used: diff --git a/freevideo_engine/comfy_library.py b/freevideo_engine/comfy_library.py index f8469bf7..590ca999 100644 --- a/freevideo_engine/comfy_library.py +++ b/freevideo_engine/comfy_library.py @@ -66,10 +66,53 @@ def _agent(folder): return {key: str(value[key])[:120] for key in ('job', 'label', 'project', 'client') if value.get(key)} -def list_videos(output_directory, *, before=None, limit=24): +def _favorite(path): + """A missing or damaged optional sidecar does not hide a saved video.""" + saved = path.with_suffix('.favorite.json') + try: + if not _plain(saved): + return False + with os.fdopen(os.open(saved, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) + | getattr(os, 'O_BINARY', 0)), 'rb') as stream: + data = stream.read(1025) + value = json.loads(data) if len(data) <= 1024 else None + return isinstance(value, dict) and value.get('favorite') is True + except (OSError, ValueError): + return False + + +def set_favorite(output_directory, identity, favorite): + """Change only a completed creation's favorite sidecar, under the delete locks.""" + if type(favorite) is not bool: + raise ValueError('Invalid favorite state') + output_directory = checked_real_path(output_directory) + root = Path(output_directory).resolve() / 'FreeVideo' + with _PENDING, _RELEASING: + path = _video(output_directory, identity) + expected = root / identity / 'video.mp4' + if path != expected or not all(_plain(p) for p in (root, path.parent.parent, path.parent, path)) or not path.is_file(): + raise ValueError('Saved video is not a regular library creation') + report = path.with_suffix('.request.json') + saved = path.with_suffix('.favorite.json') + if not _plain(report) or saved.is_symlink() or (saved.exists() and (not _plain(saved) or not saved.is_file())): + raise ValueError('Saved video sidecar is not a regular file') + _report(path) + if not path.stat().st_size: + raise ValueError('Video is empty') + if favorite: + from .monitoring import save + save(saved, dict(favorite=True)) + else: + saved.unlink(missing_ok=True) + return dict(id=identity, favorite=favorite) + + +def list_videos(output_directory, *, before=None, limit=24, favorite=None): """Return a stable newest-first page, with only relative output paths.""" if not 1 <= limit <= 48: raise ValueError('Invalid page size') + if favorite is not None and type(favorite) is not bool: + raise ValueError('Invalid favorite filter') cursor = None if before: stamp, identity = before.split(':', 1) @@ -95,6 +138,9 @@ def list_videos(output_directory, *, before=None, limit=24): relative = Path('FreeVideo') / key[1] / 'video.mp4' if relative.as_posix() in hidden: continue + is_favorite = _favorite(path) + if favorite is not None and is_favorite != favorite: + continue try: try: report = _report(path) @@ -125,7 +171,7 @@ def list_videos(output_directory, *, before=None, limit=24): agent = None if agent: summary['agent'] = agent - rows.append(dict(summary, id=key[1], bytes=info.st_size, + rows.append(dict(summary, id=key[1], bytes=info.st_size, favorite=is_favorite, created_at=datetime.fromtimestamp(info.st_mtime, timezone.utc).isoformat(), cursor=str(key[0]) + ':' + key[1])) except (OSError, ValueError, TypeError, AttributeError): @@ -1174,9 +1220,14 @@ async def request_assets(request): @server.routes.get('/freevideo/library') async def library(request): try: + favorite = request.query.get('favorite') + if favorite is not None: + if favorite not in ('true', 'false', '1', '0'): + raise ValueError('Invalid favorite filter') + favorite = favorite in ('true', '1') rows = await asyncio.to_thread(list_videos, folder_paths.get_output_directory(), before=request.query.get('before'), - limit=int(request.query.get('limit', '24'))) + limit=int(request.query.get('limit', '24')), favorite=favorite) return web.json_response(rows, headers={'Cache-Control': 'no-store'}) except ValueError as error: raise web.HTTPBadRequest(text=str(error)) from error @@ -1189,6 +1240,23 @@ async def prompt(request): raise web.HTTPNotFound(text='Prompt unavailable') from None return web.json_response(dict(prompt=text), headers={'Cache-Control': 'no-store'}) + @server.routes.post('/freevideo/library/favorite') + async def favorite(request): + try: + body = await request.json() + if not isinstance(body, dict) or type(body.get('favorite')) is not bool: + raise ValueError('Invalid favorite state') + result = await asyncio.to_thread(set_favorite, folder_paths.get_output_directory(), + body.get('id'), body['favorite']) + except (ValueError, TypeError) as error: + raise web.HTTPBadRequest(text=str(error)) from error + except FileNotFoundError: + raise web.HTTPNotFound(text='Saved video unavailable') from None + except OSError: + # The creation is there but its favorite could not be written; another try may work. + raise web.HTTPInternalServerError(text='Could not save the favorite') from None + return web.json_response(result, headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/library/thumbnail') async def preview(request): async with thumbnails: diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.py index d9899926..5b396861 100644 --- a/freevideo_engine/comfy_media.py +++ b/freevideo_engine/comfy_media.py @@ -49,21 +49,24 @@ def continued(exported, notes=None): """What a continued part keeps of exported inputs: up to three pictures, and the LoRAs. It starts from the footage, so a first or last frame image becomes one more picture after the - reference pictures; pre-encoded conditioning, clips and sound are left out. When keyframes are - used or pictures are left out, notes gets one line saying so.""" + reference pictures; pre-encoded conditioning, clips and sound are left out, except a sound used + as is, which every part plays its own range of. When keyframes are used or pictures are left out, + notes gets one line saying so.""" from .continuation import PICTURES_MAX, pick_pictures media = (exported or {}).get('media') or {} kept = {key: value for key, value in media.items() if key not in ('first', 'last', 'references', 'conditioning_info')} pictures = [row for row in media.get('references', []) if row.get('kind') == 'image'] + original = [row for row in media.get('references', []) if row.get('kind') == 'audio' and row.get('use') == 'original'] keyframes = [name for name in ('first', 'last') if media.get(name)] # Characters added by an @ mention keep one picture each when there are several (pick_pictures). chosen = pick_pictures(pictures, lambda row: row.get('cast')) candidates = [_plain(row) for row in chosen] + [{'kind': 'image', 'path': media[name]} for name in keyframes] - if candidates: - kept['references'] = candidates[:PICTURES_MAX] + if candidates or original: + kept['references'] = candidates[:PICTURES_MAX] + original lines = [continued_note(len(pictures), keyframes, len(candidates[:PICTURES_MAX]))] - if any(row.get('kind') != 'image' for row in media.get('references', [])) or media.get('conditioning_info') \ + if any(row.get('kind') != 'image' and row not in original for row in media.get('references', [])) \ + or media.get('conditioning_info') \ or (exported or {}).get('conditioning'): lines.append('A continued part uses picture references only. Video, audio and pre-encoded inputs are skipped.') for line in filter(None, lines): @@ -100,7 +103,7 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c if conditioning is not None or first is not None or last is not None or references: raise ValueError('Use the Media panel or direct media/conditioning sockets; disable duplicate inputs') connected = assets.get('_connected', {}) - media.update({key: value for key, value in assets.items() if key != '_connected'}) + media.update({key: value for key, value in assets.items() if key not in ('_connected', '_audio_use')}) first, last = connected.get('first'), connected.get('last') references = [] for name in REFERENCE_INPUTS: @@ -112,6 +115,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c references.extend({'kind': 'image', 'value': frame} for frame in frames) if connected.get('reference_audio') is not None: references.append({'kind': 'audio', 'value': connected['reference_audio']}) + if assets.get('_audio_use') == 'original': + next(row for row in references if row['kind'] == 'audio')['use'] = 'original' if conditioning is not None and (first is not None or last is not None or references): raise ValueError('Preencoded conditioning already includes media; do not connect a second set') keyframes = first is not None or last is not None or any(name in media for name in ('first', 'last')) @@ -145,7 +150,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c _audio(value, path) else: raise ValueError('Unknown reference type') - row = {'kind': kind, 'path': str(path)} + from .original_audio import fields + row = {'kind': kind, 'path': str(path), **fields(reference)} if 'early_steps' in reference: row['early_steps'] = reference['early_steps'] media['references'].append(row) @@ -162,5 +168,7 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c if not continuing and media.get('references'): media['references'] = [_plain(row) for row in media['references']] if len(media) > 1: + from .original_audio import original + original(media) result['media'] = media return result diff --git a/freevideo_engine/comfy_metadata.py b/freevideo_engine/comfy_metadata.py index 0abafdf8..ca988054 100644 --- a/freevideo_engine/comfy_metadata.py +++ b/freevideo_engine/comfy_metadata.py @@ -24,7 +24,7 @@ def _room(path, size): return disk_space.free_bytes(path.parent) >= 2 * size + 512 * 1024 * 1024 -def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None): +def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None, exact_timescale=None): """Rewrite the MP4 at `path` with the graph in its metadata, copying the streams. The new file replaces the original only after every audio and video packet matches the original byte for byte and the graph reads back; with @@ -48,7 +48,9 @@ def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None): return False # faststart puts moov, and with it the metadata, before the media data. with av.open(str(path)) as source, av.open(str(temporary), 'w', format='mp4', - options={'movflags': 'use_metadata_tags+faststart'}) as target: + options={'movflags': 'use_metadata_tags+faststart', + **({'video_track_timescale': str(exact_timescale), 'movie_timescale': str(exact_timescale)} + if exact_timescale is not None else {})}) as target: if not source.streams or any(stream.type not in ('video', 'audio') or stream.codec_context is None for stream in source.streams): raise ValueError('Only videos with nothing but audio and video streams are rewritten') @@ -61,6 +63,8 @@ def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None): target.mux(packet) if _media(temporary) != _media(path): raise ValueError('The rewritten video does not match the original') + if exact_timescale is not None and _timing(temporary) != _timing(path): + raise ValueError('The rewritten original soundtrack changed its timing') with av.open(str(temporary)) as check: if any(check.metadata.get(key) != value for key, value in tags.items()): raise ValueError('The workflow did not read back') @@ -110,6 +114,17 @@ def _media(path): return layout, [(counts[index], digests[index].hexdigest()) for index in sorted(digests)] +def _timing(path): + import av + with av.open(str(path)) as reader: + streams = {s.index: [] for s in reader.streams} + for p in reader.demux(): + if p.size: + streams[p.stream.index].append((p.pts * p.time_base if p.pts is not None else None, + p.dts * p.time_base if p.dts is not None else None, p.duration * p.time_base)) + return streams + + def _copy_stream(container, template): try: return container.add_stream_from_template(template=template, opaque=True) diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index c1326297..77f9dbb3 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -7,6 +7,7 @@ from . import comfy_bridge, workflow_repairs from .comfy_continuation import node_frames, optional_text, part_steps, request_options from .default_prompt import DEFAULT_PROMPT +from .original_audio import audio_use_option References = io.Custom('FREEVIDEO_REFERENCES') LoRAs = io.Custom('FREEVIDEO_LORAS') @@ -136,6 +137,8 @@ def define_schema(cls): io.Combo.Input('upscale_factor', display_name='Upscale factor', options=list(UPSCALE_OPTIONS), default='2', optional=True, advanced=True, tooltip='Two-pass only: a smaller factor starts from a larger first pass, for more detail and a longer run.'), + io.Boolean.Input('allow_empty_prompt', display_name='Allow empty prompt', default=False, + optional=True, advanced=True, tooltip='Generates from an empty prompt instead of asking for one.'), ], outputs=[io.Video.Output('video'), io.String.Output('report', display_name='Report JSON')], hidden=[io.Hidden.unique_id, io.Hidden.prompt, io.Hidden.extra_pnginfo], @@ -145,7 +148,7 @@ def define_schema(cls): @classmethod def validate_inputs(cls, text, width, height, seconds, seed, continue_from='', frames=0, two_pass=True, preview=False, refine_from='', base_steps=8, refine_steps=3, step_cache='none', - upscale_factor='2'): + upscale_factor='2', allow_empty_prompt=False): # ComfyUI marks every input named here when a check fails, so only the checked widgets are # listed. Linked pictures, media and conditioning have no value before the run anyway. # step_cache and upscale_factor are named so ComfyUI skips their option lists: an older saved @@ -159,7 +162,8 @@ def validate_inputs(cls, text, width, height, seconds, seed, continue_from='', f base_steps, refine_steps = part_steps(base_steps, refine_steps, step_cache)[:2] options.update(two_pass=True, preview=False, refine_from=refine_from or None, media=dict(first=False, last=False, conditioning_info=False)) - comfy_bridge.validate_request(text, width, height, seconds, seed, **options) + comfy_bridge.validate_request(text, width, height, seconds, seed, **options, + allow_empty_prompt=allow_empty_prompt is True) if not refine_from: from .two_pass import validate_steps validate_steps(base_steps, refine_steps, two_pass) @@ -186,7 +190,11 @@ def execute(cls, text, width, height, seconds, seed, first=None, last=None, references=None, loras=None, conditioning=None, media=None, two_pass=True, force_regenerate=False, base_steps=8, refine_steps=3, preview=False, refine_from='', continue_from='', frames=0, first_frame_reference=False, voice_reference=False, hide_tail=False, - step_cache='none', upscale_factor='2'): + step_cache='none', upscale_factor='2', allow_empty_prompt=False): + from .comfy_environment import control_process + from .gpu_pool import pool_enabled + if control_process() and pool_enabled(): + raise RuntimeError('Generation requires a dispatched executor') from server import PromptServer from .comfy_chain import block_context, block_finished, report_fields refine_from, continue_from = optional_text(refine_from), optional_text(continue_from) @@ -334,6 +342,7 @@ def kept(exported): **({'first_frame_reference': True} if first_frame_reference is True and options.get('continue_from') else {}), **({'voice_reference': True} if voice_reference is True and options.get('continue_from') else {}), **({'hide_tail': True} if hide_tail is True else {}), + **({'allow_empty_prompt': True} if allow_empty_prompt is True else {}), **step_cache_option(step_cache), **upscale_option(upscale_factor), **({'report_extra': {'chain': report_fields(chain)}} if chain is not None else {})) @@ -364,8 +373,8 @@ def kept(exported): server = PromptServer.instance def busy(): return server is not None and server.prompt_queue.get_tasks_remaining() > 0 - # Keep the models only while the next part is on its way; otherwise give - # the memory back now and let the file cache make the next load quick. + # A finished request keeps the models for the next one while memory is ample; + # a failed one, a tight host or another program's growth gives it back at once. try: # The running prompt is this one; only prompts still waiting are a next part. queued = server is not None and bool(server.prompt_queue.get_current_queue()[1]) @@ -383,9 +392,10 @@ def busy(): gpu = comfy_bridge.installation()[1].get('gpu_uuid') except (OSError, ValueError, KeyError, TypeError): gpu = None - OWNER.finished(next_part=next_part, gpu=gpu, + OWNER.finished(next_part=next_part, gpu=gpu, warm=True, agent=server is not None and str(getattr(server, 'client_id', '') or '').startswith('agent:')) - if target and not result_reused[0]: + from .comfy_environment import executor_device + if target and not result_reused[0] and not executor_device(): client = server.client_id if server is not None else None def warmed(value): if server is not None: @@ -425,7 +435,8 @@ def define_schema(cls): io.Audio.Input('reference_audio', display_name='Reference audio', optional=True, tooltip='Voice, music or rhythm reference, up to 15 seconds. Connect a Load Audio ' 'or other AUDIO output. Describe its role using