diff --git a/MANIFEST.in b/MANIFEST.in index 3b2bfbcd..d5944a9b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ recursive-include web/fonts *.woff2 recursive-include web/tutorial *.webp *.mp4 recursive-include freevideo_engine/assets *.png *.ico recursive-include freevideo_engine/launcher *.qml +recursive-include freevideo_engine/agent/skill *.txt recursive-include freevideo_engine/launcher/licenses *.txt recursive-include freevideo_engine/licenses *.txt recursive-include constraints *.txt diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 614bfccd..66b947ee 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -5,6 +5,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th | Component | License | | --- | --- | | [Three.js](https://threejs.org/) r160, © 2010–2023 three.js authors (bundled whitebox renderer) | [MIT](web/vendor/three/LICENSE) | +| [Depth Anything V2 Small](https://github.com/DepthAnything/Depth-Anything-V2/tree/a561b849ebae10a6f5ef49e26c83cbbcd36c71bf), © 2024 Depth Anything authors; DINOv2 © Meta Platforms, Inc. and affiliates. Small-only DPT/DINOv2 inference and preprocessing are adapted and bundled; Small weights are an optional download pinned at `03876f8651c73a60fe4c2c48294e09fcb6838fcf`. | [Apache-2.0 (code and Small weights)](freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt) | | Qt, PySide6 and Shiboken6 | LGPL-3.0; [component notices](freevideo_engine/launcher/licenses/NOTICE.txt) | | PyInstaller bootloader | GPL-2.0 with the bootloader exception | | CPython / psutil | PSF / BSD-3-Clause | diff --git "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 03fb88e0..53e0c03e 100644 --- "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -32,6 +32,10 @@ - 编号顺序是：先是上传的素材，再是从上到下连接的图片，最后是音频。参考视频的声音也占一个音频编号。 - 删除或关闭某张素材后，描述里指向它的标记会改成“[素材已移除]”，其余标记按新编号更新。 +## 用一句话做白模 + +未连接外部 Agent 时，可开启聊天框中的白模按钮，用一句话描述镜头。有合适的现成场景时，会先生成白模供您确认。 + ## 做更长的视频 有三种做法，可以单独用，也可以搭配。 @@ -46,7 +50,7 @@ ### 限制 -- 两种做法一样，一条视频最多 8 段（第一段加 7 次接着生成）；创作面板里时长最多填 60 秒。接续的每一段至少 10 秒，整条约 110 秒。剪辑视图里逐段生成同样适用。段数越多，衔接处越容易出现变化，适合切换镜头处，可直接切镜头，用对应的参考图开始新的一段。 +- 两种做法一样，一条视频最多 8 段（第一段加 7 次接着生成），有短于 10 秒的段时最多 4 段；创作面板里时长最多填 60 秒。接续的每一段至少 5 秒，每段都是 10 秒以上时整条约 110 秒。剪辑视图里逐段生成同样适用。段数越多，衔接处越容易出现变化，适合切换镜头处，可直接切镜头，用对应的参考图开始新的一段。 - 接着生成使用轻量档，需要 NVIDIA 显卡或 Mac，也不能和“继续二采”同时使用。 - 单次生成的最长时长随画幅和显卡而不同，在创作面板的“可生成范围”中可查看各尺寸的最长时长和预计用时。 - 分段生成同样需要 NVIDIA 显卡或 Mac。 diff --git a/freevideo_engine/agent/cli.py b/freevideo_engine/agent/cli.py index 793729b4..473135f3 100644 --- a/freevideo_engine/agent/cli.py +++ b/freevideo_engine/agent/cli.py @@ -16,10 +16,11 @@ import sys from .client import ApiError, Client -from . import schema, spec, errors +from . import errors def read(path): + from . import schema text = sys.stdin.read() if path == '-' else Path(path).read_text(encoding='utf-8') value = json.loads(text) schema.validate({'type': 'object'}, value) @@ -40,6 +41,7 @@ def error(self, message): def build_parser(): + from . import schema parser = Parser(prog='./freevideo agent', description='Drive FreeVideo through its agent API.') parser.add_argument('--root', type=Path) parser.add_argument('--url') @@ -47,6 +49,8 @@ def build_parser(): commands.add_parser('status', help='Readiness, limits, LoRAs, VLM and queue') commands.add_parser('guide', help='How to write H3 prompts') commands.add_parser('schema', help='JSON Schemas of jobs') + command = commands.add_parser('install-skill', help='Copy the FreeVideo skill where Codex, Gemini CLI, Cursor and Claude Code read skills') + command.add_argument('--target', choices=('agents', 'claude'), action='append', help='Default: both') command = commands.add_parser('capabilities', help='Read the studio capability cache') schema.cli_arguments(command, schema.contracts()['GET', '/capabilities']['query']) command = commands.add_parser('asset', help='Download a saved library video or thumbnail over HTTP') @@ -159,10 +163,39 @@ def build_parser(): return parser +def install_skill(argv): + """Run `install-skill` without the job schemas, whose import loads the image libraries (numpy, Pillow). + + A deep installation folder on Windows cannot load those libraries by their full path, and copying + the skill needs none of them. None where the command line is anything else.""" + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--root', type=Path) + parser.add_argument('--url') + parser.add_argument('command', nargs='?') + known, rest = parser.parse_known_args(argv) + if known.command != 'install-skill': + return None + parser = argparse.ArgumentParser(add_help=False, exit_on_error=False) + parser.add_argument('--target', choices=('agents', 'claude'), action='append') + try: + args, extra = parser.parse_known_args(rest) + except argparse.ArgumentError: + return None + if extra: + return None + from . import skill_install + print(json.dumps({'installed': skill_install.install(tuple(args.target or ('agents', 'claude')))}, indent=2)) + return 0 + + def main(argv=None): for stream in (sys.stdin, sys.stdout): if hasattr(stream, 'reconfigure'): stream.reconfigure(encoding='utf-8') + done = install_skill(sys.argv[1:] if argv is None else argv) + if done is not None: + return done + from . import schema, spec parser = build_parser() args = parser.parse_args(argv) values = vars(args) diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py index a34f83ed..ac0b32b9 100644 --- a/freevideo_engine/agent/drama.py +++ b/freevideo_engine/agent/drama.py @@ -20,7 +20,7 @@ import time from . import spec as specs -from ..chain_plan import MIN_PART_SECONDS +from ..chain_plan import CONTINUE_SECONDS EXAMPLE = { 'title': '末班车', 'name': 'the-last-train', 'aspect': '9:16', 'quality': 'light', @@ -451,9 +451,9 @@ def intent(play, shot): def shot_seconds(play, shot): - """A shot says its length; else the film does; else 6 s, and 10 s for a shot that continues the one before - (a part that continues another video is at least 10 s).""" - return shot.get('seconds', play.get('seconds', MIN_PART_SECONDS if shot.get('continue') else 6)) + """A shot says its length; else the film does; else 6 s, and CONTINUE_SECONDS for a shot that continues the + one before (a full continuing part, which keeps the video to the full number of parts).""" + return shot.get('seconds', play.get('seconds', CONTINUE_SECONDS if shot.get('continue') else 6)) def shot_job(play, shot, index, ref): diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.py index b8bbf755..456783fb 100644 --- a/freevideo_engine/agent/editor_runtime.py +++ b/freevideo_engine/agent/editor_runtime.py @@ -16,7 +16,7 @@ # Definition order only; evaluation starts at editor_actions like the page's # import does. editor_model and editor_text_model import each other: the # module still loading is reached only through a namespace import. -MODULES = ('editor_picture_fields', 'editor_transport_model', 'editor_audio_model', 'editor_speed_model', +MODULES = ('chain_rules', 'editor_picture_fields', 'editor_transport_model', 'editor_audio_model', 'editor_speed_model', 'editor_text_model', 'editor_model', 'editor_room', 'editor_actions', 'editor_generation', 'sampling_effort', 'chain_support', 'editor_store') TIME_LIMIT = 5.0 diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index dd4dccc4..59974fad 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -67,6 +67,7 @@ 'chain_missing': (409, True, 'The saved chain receipt is unavailable.', 'Restore the chain receipt and retry resume, or submit a new job.'), 'join_failed': (409, True, 'The generated blocks could not be joined; completed blocks are retained.', 'POST /freevideo/agent/v1/jobs/{id}/resume to retry joining the completed blocks.'), 'chain_too_long': (422, False, 'One continuous shot can have up to 8 parts. Make it shorter.', 'Submit a new job with a smaller seconds, or cut to a new shot with a new job without continue_from; continue_from counts the parts of the video it continues.'), + 'chain_short_parts': (422, False, 'With a part shorter than 10 seconds, one continuous shot can have up to 4 parts. Make every part at least 10 seconds long, or use fewer parts.', 'Submit a new job whose continuing parts are at least 10 seconds, with fewer parts, or with first_frame_reference true; continue_from counts the parts of the video it continues.'), 'chain_rejected': (409, False, 'The continuous backend rejected this generation.', 'Read GET /freevideo/agent/v1/capabilities and submit a supported generation request.'), 'invalid_wait': (422, False, 'wait must be a finite nonnegative number of seconds.', 'Use wait=0 or wait=25; larger valid values are clamped to 25 seconds.'), 'invalid_resume': (422, False, 'resume accepts an empty object or an optional ref only.', 'Send {} or {"ref":"resume-1"}; submit a new job to change generation parameters.'), @@ -302,6 +303,7 @@ def safe_details(value): 'invalid_dimensions': '图片尺寸超出了支持的范围', 'too_many_images': '参考图片数量超过了支持的范围', 'chain_too_long': '接续后的视频超过了支持的段数', + 'chain_short_parts': '有短于 10 秒的段时，接续后的视频超过了支持的段数', 'chain_rejected': '生成服务无法使用这组参数', 'generation_failed': '生成没有完成', 'internal_error': '没有完成', @@ -350,6 +352,7 @@ def safe_details(value): 'invalid_dimensions': 'The picture size is outside the supported range', 'too_many_images': 'There are more reference pictures than supported', 'chain_too_long': 'The continued video has more parts than supported', + 'chain_short_parts': 'With a part shorter than 10 seconds, the continued video has more parts than supported', 'chain_rejected': 'The generation service cannot use these settings', 'generation_failed': 'Generation did not finish', 'internal_error': 'Did not finish', diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 2a4268da..a61ea514 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -99,7 +99,7 @@ 'timing': [ 'Single-block frames are 24 fps on a 17n+5 grid (6 s -> 6.583 s, 10 s -> 10.125 s). ' 'Under 30 s defaults to one generation when the device supports this size and quality; unknown limits use 15 s. ' - 'Otherwise it uses as few parts as possible, each at least 10 s; how many depends on how long this computer can generate in one run, and the parts continue and join automatically. ' + 'Otherwise it uses as few parts as possible, each continuing part 10 s or longer where the length allows and never under 5 s; how many depends on how long this computer can generate in one run, and the parts continue and join automatically. ' 'Multiple blocks or continue_from need two-pass text generation: quality "light" or "medium" ' 'or sampling.two_pass=true and no preview. Their only media is media.references with at most 3 pictures, ' 'which every continuing part also sees (keeps the same people); first_frame_reference then gives way to them. ' @@ -250,9 +250,10 @@ 'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should ' 'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a ' 'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and ' - 'continuing again and again is not recommended. One video has at most 8 parts (the first and 7 ' - 'continuations), each continued part at least 10 seconds, about 110 seconds in all; filling that is not ' - 'recommended. Past it, cut to a new shot.', + 'continuing again and again is not recommended. Each continued part is at least 5 seconds. One video has at ' + 'most 8 parts (the first and 7 continuations, about 110 seconds in all) when every continued part is at ' + 'least 10 seconds; with a continued part shorter than 10 seconds it has at most 4 parts, unless ' + 'first_frame_reference is true. Filling that is not recommended. Past it, cut to a new shot.', 'A question is not a request. When the user asks about progress or status ("is it working?", "what happened?") ' 'answer it in one sentence from what you know or from the job and white-model status, and start nothing; never ' 'use a question as a prompt. When a row carries "whitebox", the user was starting a white-model scene that is ' @@ -287,7 +288,8 @@ '先用 people（角色图和各自的位置）做出首帧，再用它生成视频。', '默认切镜头。能切镜头的地方就切：新镜头直接用对应的参考图（人物卡、场景图）生成，不接上一段视频。' '只有不能切镜的长镜头、必须连贯的动作、必须连贯的声音，才接着上一段生成，而且不建议一段接一段地连续接。' - '一条视频最多 8 段（第一段加 7 次接着生成），接着生成的每一段至少 10 秒，总长约 110 秒；不建议接满，超过就切镜头。', + '接着生成的每一段至少 5 秒。每段都不短于 10 秒时，一条视频最多 8 段（第一段加 7 次接着生成，总长约 110 秒）；' + '有短于 10 秒的段时最多 4 段，first_frame_reference 为 true 时除外。不建议接满，超过就切镜头。', '问话不是要求。用户问进度或状态（“在做吗”“怎么了”）时，用一句话照实回答，依据是你所知道的、任务和白模的状态，' '什么都不要开始；不要把问句当提示词。某条 inbox 带有 whitebox 时，说明用户刚启动了白模场景，' '它还在准备或已经失败：照实说是哪一种，再等用户回话。', @@ -573,8 +575,9 @@ def connection_text(value): 'before uploading. Unknown fields are rejected and every batch is validated before it is recorded.', 'POST /plan gives the length supported for the current size and quality. Without continuation, only a ' 'single block can run; unknown device limits use 15 s. Multiple blocks return continuation_unavailable.', - 'One video has at most 8 parts (60 s in one request). continue_from counts the parts of the video it ' - 'continues; plan and submit refuse a longer one and name the longest seconds that still fits.', + 'One video has at most 8 parts (60 s in one request), or 4 when a continued part is shorter than 10 s and ' + 'first_frame_reference is not true. continue_from counts the parts of the video it continues; plan and ' + 'submit refuse a longer one and name the longest seconds that still fits.', 'features in the status reply names what this installation offers; false means not usable now (for example ' 'chain.auto while continuation is false), and a missing name means not offered here.', ] diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 73da1fd6..80bd5db2 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -187,7 +187,7 @@ def part_prompt_problems(spec, prefix=''): def _continued_part_problem(seconds, chain): - """True when a video that continues another has a part the backend refuses, or is under 10 s. + """True when a video that continues another has a part the backend refuses, or is under MIN_PART_SECONDS. The backend compares each part's node input frames (overlap included); a part that continues is also at least MIN_PART_SECONDS long. @@ -232,8 +232,8 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N """ if spec['kind'] != 'generate' or spec.get('refine_from'): return [] - from ..chain_plan import MIN_PART_SECONDS, PlanError, plan - from ..chain_runner import ChainTooLong + from ..chain_plan import MIN_PART_SECONDS, PlanError, parts_limit, plan + from ..chain_runner import ChainShortPartsTooMany, ChainTooLong from ..chain_store import reference continued, frames = bool(spec.get('continue_from')), single_frame_limit(spec) if editor is None: @@ -246,40 +246,50 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N return [length_problem(spec['seconds'], continued, prefix, minimum=MIN_PART_SECONDS, frames=frames)] if runner is None: return [] - limit = runner.max_parts - count = lambda seconds: len(plan(seconds, max_frames=frames, continued=continued)['blocks']) + # A video with a continuing part shorter than full_part_frames has fewer parts, unless every part sees the first frame. + limit, full = runner.max_parts, runner.full_part_frames + short_limit = parts_limit(True, spec.get('first_frame_reference') is True, limit) + + def short_blocks(blocks): + return any(block['frames'] < full for index, block in enumerate(blocks) if index or continued) try: with runner.lock: if editor is not None: - rows = [dict(id=b['id'], after=reference(b.get('after')), before=reference(b.get('before'))) - for b in editor['blocks']] - earlier, needed = 0, max(runner._depths(dict(id=None, blocks=rows)).values()) + rows = [dict(id=b['id'], after=reference(b.get('after')), before=reference(b.get('before')), + inputs=dict(frames=b['frames'])) for b in editor['blocks']] + lineage = runner._lineage(dict(id=None, blocks=rows)).values() + earlier, needed = 0, max(parts for parts, _ in lineage) + short = max((parts for parts, flag in lineage if flag), default=0) else: # A source the backend has no record of counts as one part, as it does there. - earlier = (runner._work_parts(work, ()) if work else planned or 1) if continued else 0 + earlier, made_short = ((runner._work_lineage(work, ()) if work else (planned or 1, False)) + if continued else (0, False)) needed = earlier + len(chain['blocks']) + short = needed if made_short or short_blocks(chain['blocks']) else 0 except (ValueError, KeyError, TypeError): return [] # Not a chain the backend accepts; it says why when the chain is created. - if needed <= limit: + if needed <= limit and short <= short_limit: return [] + error = ChainTooLong(needed, limit) if needed > limit else ChainShortPartsTooMany(short, short_limit) if editor is not None: - problem, got = schema.diagnostic(prefix + '/target', 'a range that takes at most %d parts' % limit, None), '%d parts' % needed + problem, got = schema.diagnostic(prefix + '/target', 'a range that takes at most %d parts' % error.limit, None), '%d parts' % error.parts else: # A length the planner refuses does not fit; a part that continues a video is at least MIN_PART_SECONDS. - def parts(seconds): + def fits(seconds): try: - return count(seconds) + blocks = plan(seconds, max_frames=frames, continued=continued)['blocks'] except ValueError: - return limit + 1 + return False + return earlier + len(blocks) <= (short_limit if made_short or short_blocks(blocks) else limit) shortest = MIN_PART_SECONDS if continued else 2 - fit = next((s for s in range(int(spec['seconds']), shortest - 1, -1) if earlier + parts(s) <= limit), None) + fit = next((s for s in range(int(spec['seconds']), shortest - 1, -1) if fits(s)), None) if fit is None: - problem = schema.diagnostic(prefix + '/continue_from', 'a video of fewer than %d parts' % limit, None) + problem = schema.diagnostic(prefix + '/continue_from', 'a video of fewer than %d parts' % error.limit, None) got = 'a video of %d parts' % earlier else: problem, got = schema.diagnostic(prefix + '/seconds', 'at most %d' % fit, spec['seconds']), None message = problem['message'] if got is None else problem['message'].replace('received None', 'received ' + got) - return [dict(problem, got=got or problem['got'], message='%s %s' % (ChainTooLong(needed, limit), message))] + return [dict(problem, got=got or problem['got'], message='%s %s' % (error, message))] def continuation_unavailable(): @@ -1635,7 +1645,7 @@ def start_chain(self, row, *, payload=None, resume=False): except ChainTooLong as error: # Submit and plan refuse this up front; a source that was still being made can only be counted here. row['resume_pending'] = False - self.finish(row, 'failed', error=errors.build('chain_too_long', validation=True, message=str(error))) + self.finish(row, 'failed', error=errors.build(error.code, validation=True, message=str(error))) return except (PlanError, ChainPartTooShort) as error: row['resume_pending'] = False diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index ff1f1815..1b2ae34f 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -21,11 +21,12 @@ from .client import ApiError, Client, DONE +JOB_WAIT = 45. # freevideo_wait only; the other waits keep 25 s. Fewer empty turns, each of which carries the whole context again. PROTOCOLS = ('2025-06-18', '2025-03-26', '2024-11-05') INSTRUCTIONS = """FreeVideo generates videos (MiniMax H3) on this computer's GPU. -Start with freevideo_status and freevideo_capabilities. Write prompts as the guide says (freevideo_prompt_guide): English sections +Start with freevideo_status and freevideo_capabilities. Picture editing, storyboards (submit_batch), editor operations and plan are not listed in full: freevideo_tool_help {name} gives the schema, freevideo_call runs them. Write prompts as the guide says (freevideo_prompt_guide): English sections integrated_multimodal_description / overall_soundscape / non_diegetic_music, [Shot N] cuts, (S1) [Language] ... -dialogue. Generation uses continuous chains and takes minutes: submit, then call freevideo_wait (at most 25 seconds) +dialogue. Generation uses continuous chains and takes minutes: submit, then call freevideo_wait (up to 45 seconds; it returns as soon as the job ends) or freevideo_job. Interrupted/failed/cancelled generation can continue with freevideo_resume; completed blocks stay. Chain shots by starting the next shot from the previous one's last frame (media.first = {"from_job": id, "frame": "last"}), look at results with freevideo_look, ask the local VLM with freevideo_ask, and join clips @@ -49,11 +50,11 @@ def tool_list(): dict(name='freevideo_prompt_guide', title='H3 prompt guide', description='How to write prompts: sections, shots and cuts, camera motion, dialogue, references, short-drama tips, and a full example.', annotations=read), dict(name='freevideo_schema', title='Job JSON Schemas', description='JSON Schemas of every job kind and of image edit recipes.', annotations=read), dict(name='freevideo_plan', title='Plan a job', description="Validate a job without running it: actual frames/seconds, sampling passes, warnings and a time estimate from this computer's history. Same fields as freevideo_generate.", annotations=read), - dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise the video is cut into as few parts as possible, each at least 10 s, and how many parts depends on how long this computer can generate in one run; the parts continue and join automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), + dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise the video is cut into as few parts as possible, each continuing part 10 s or longer where the length allows and never under 5 s, and how many parts depends on how long this computer can generate in one run; the parts continue and join automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), dict(name='freevideo_generate_image', title='Generate a picture', description='Generate a picture from a prompt; with images, change or combine input pictures; with people, make a first frame with them near the places you say, in the scene from scene_image or the prompt. The result is an input file to use as a first frame, as a reference, or in images again.', annotations=write), dict(name='freevideo_submit_batch', title='Submit a storyboard', description='Submit several jobs at once. Each row is a job object with "kind" (generate default, assemble, vlm, enhance, image) and an optional "ref"; later rows use "@ref" wherever a job id is expected (from_job, after, clips, refine_from). FreeVideo runs them in dependency order.', annotations=write), dict(name='freevideo_job', title='Job status', description='Status, progress, result or error of a job.', annotations=read), - dict(name='freevideo_wait', title='Wait for a job or the user', description="Wait at most 25 seconds for a job (with id) or, without id, at most 50 seconds for what the user says in FreeVideo (inbox). again=true means nothing new yet: call it again. A job wait returns early when the user says something. If the job is still active, call it again with the same ID; do not resubmit.", annotations=read), + dict(name='freevideo_wait', title='Wait for a job or the user', description="Wait up to 45 seconds for a job (with id) or, without id, at most 50 seconds for what the user says in FreeVideo (inbox). again=true means nothing new yet: call it again. A job wait returns early when the user says something. If the job is still active, call it again with the same ID; do not resubmit.", annotations=read), dict(name='freevideo_jobs', title='List jobs', description='Recent agent jobs, optionally by status (comma-separated), project or kind.', annotations=read), dict(name='freevideo_cancel', title='Cancel a job', description='Cancel a waiting, queued or running job.', annotations={'readOnlyHint': False, 'destructiveHint': True, 'openWorldHint': False}), dict(name='freevideo_resume', title='Resume generation', description='Resume a failed, interrupted or cancelled generation; only unfinished blocks run. Poll the same job ID.', annotations=write), @@ -95,6 +96,34 @@ def tool_list(): return rows +GATEWAY_TOOLS = ('freevideo_tool_help', 'freevideo_call') + + +def hidden_tools(): + """Tools whose full schemas stay out of tools/list: every client sends the whole list with every request, + and these are the largest and the least often used. They are described by name and listed in the help tool.""" + from .schema import IMAGE_TOOLS + return frozenset(IMAGE_TOOLS.values()) | {'freevideo_submit_batch', 'freevideo_edit_image', 'freevideo_editor_operations', 'freevideo_plan'} + + +def listed_tools(rows=None): + """What tools/list returns: the common tools in full, the rest through freevideo_tool_help and freevideo_call.""" + rows = rows if rows is not None else tool_list() + hidden = hidden_tools() + brief = '; '.join('%s (%s)' % (r['name'], r['description'].split('. ')[0].rstrip('.')[:90]) for r in rows if r['name'] in hidden) + read = {'readOnlyHint': True, 'openWorldHint': False} + visible = [r for r in rows if r['name'] not in hidden] + visible.append(dict(name='freevideo_tool_help', title='Schema of a tool not listed', annotations=read, + description='Full description and input schema of one of the tools not listed here: ' + brief + '. Call it once before the first use, then call the tool through freevideo_call.', + inputSchema=dict(type='object', properties=dict(name=dict(type='string', enum=sorted(hidden))), required=['name'], additionalProperties=False))) + visible.append(dict(name='freevideo_call', title='Call a tool not listed', + annotations={'readOnlyHint': False, 'destructiveHint': False, 'openWorldHint': False}, + description='Call one of the tools that freevideo_tool_help describes with its arguments; the result is the tool\'s own result.', + inputSchema=dict(type='object', properties=dict(tool=dict(type='string', enum=sorted(hidden)), arguments=dict(type='object')), + required=['tool'], additionalProperties=False))) + return visible + + PROMPTS = [ dict(name='short_drama', title='Make a short drama', description='Plan and produce a short drama: script, consistent characters, chained shots, review, assembly.', @@ -116,7 +145,7 @@ def short_drama(arguments): 4. Follow todo: freevideo_drama_sheet shows each take, then freevideo_drama_choose or freevideo_drama_reshoot with a concrete note. Continuations wait for the previous chosen take. Prefer aspect "{aspect}" and quality light. 5. When every shot is chosen, freevideo_drama_finish accepts assembly; get until the output library_id is ready. -Use ref to replay writes safely. Ordinary job batches remain available through freevideo_submit_batch.""" +Use ref to replay writes safely. Ordinary job batches go through freevideo_call with tool freevideo_submit_batch.""" def preload(): @@ -218,7 +247,7 @@ def dispatch(self, identity, method, params): return {} if method == 'tools/list': if self.tools is None: - self.tools = tool_list() + self.tools = listed_tools() return dict(tools=self.tools) if method == 'tools/call': return self.call(identity, params.get('name'), params.get('arguments') or {}, @@ -245,6 +274,19 @@ def dispatch(self, identity, method, params): # Tools ----------------------------------------------------------------------------------------------- def call(self, identity, name, arguments, progress_token): from . import schema, spec, errors + if name == 'freevideo_tool_help': + row = next((r for r in tool_list() if r['name'] == arguments.get('name') and r['name'] in hidden_tools()), None) + if row is None: + problem = errors.build('invalid_request', validation=True, details=[dict(field='name', allowed=sorted(hidden_tools()))]) + return dict(content=[dict(type='text', text=errors.text(problem))], structuredContent={'error': problem}, isError=True) + return dict(content=[dict(type='text', text=json.dumps(row, ensure_ascii=False))], structuredContent=row, isError=False) + if name == 'freevideo_call': + target, inner = arguments.get('tool'), arguments.get('arguments') or {} + if target not in hidden_tools() or not isinstance(inner, dict): + problem = errors.build('invalid_request', validation=True, details=[dict(field='tool' if target not in hidden_tools() else 'arguments', + allowed=sorted(hidden_tools()) if target not in hidden_tools() else 'object')]) + return dict(content=[dict(type='text', text=errors.text(problem))], structuredContent={'error': problem}, isError=True) + return self.call(identity, target, inner, progress_token) try: schema.validate(schema.tool_schema(name), arguments) arguments = dict(arguments) @@ -357,7 +399,7 @@ def tool(self, identity, name, args, token): if name == 'freevideo_wait': if not args.get('id'): return c.get('/inbox', wait=min(50., float(args.get('timeout', 50)))), [] - return self.wait(identity, args['id'], min(25., float(args.get('timeout', 25))), token), [] + return self.wait(identity, args['id'], min(JOB_WAIT, float(args.get('timeout', JOB_WAIT))), token), [] if name == 'freevideo_jobs': return dict(jobs=c.jobs(**args)), [] if name == 'freevideo_cancel': @@ -380,7 +422,7 @@ def tool(self, identity, name, args, token): raise LookupError('Unknown tool: %s' % name) def wait(self, identity, job, timeout, token): - deadline = time.monotonic() + max(0., min(25., float(timeout))) + deadline = time.monotonic() + max(0., min(JOB_WAIT, float(timeout))) last = None while True: left = max(0., deadline - time.monotonic()) diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 6ee2c115..68345481 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -347,7 +347,7 @@ def _material(self, state, *, local=True): continue if continued or shot['continues']: # The same length rule as at submission, with this computer's single-run limit rather than the - # 15 s default: a shot that cannot be cut into parts of at least 10 s is refused here instead of + # 15 s default: a shot that cannot be cut into parts of at least 5 s is refused here instead of # starting a film in which that shot never gets a job. A shot not yet submitted carries no parent. try: pinned = spec.normalize(dict(item, continue_from=dict(from_job=placeholder)), port=self.jobs.port) diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 450dc543..28ff6608 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -31,6 +31,7 @@ PREFIX = '/freevideo/agent/v1' from .whitebox import MAX_VIDEO_BYTES as MAX_WHITEBOX_VIDEO +from . import whitebox_pick from .whitebox_presets import PRESETS, preset as whitebox_preset # What an empty wait tells the agent: the conversation is open for as long as the user keeps the page. KEEP_WAITING = ('Nothing new yet. The user is still in FreeVideo and may speak at any moment: call this wait ' @@ -46,6 +47,8 @@ 'say nothing until the user speaks, then carry on from there.') BEFORE_LINES, BEFORE_TEXT = 8, 600 # The user started a white-model scene and then spoke: the words may be about it, not a new request. +# Seconds one question to the local prompt model may take, loading the model included. +WHITEBOX_ASK_SECONDS = 120 WHITEBOX_PENDING = ('The user started a white-model scene in this conversation ("whitebox" lists it). If the words ask about ' 'it or about progress, answer from its status in one sentence (waiting or rendering: still being made; ' 'failed: say it was not made, and why) and start nothing. A question is never a prompt.') @@ -2191,24 +2194,63 @@ async def whitebox_next(request): async def whitebox_presets(request): return reply(dict(presets=[{k: row[k] for k in ('id', 'name', 'shot', 'prompt')} for row in PRESETS])) + def whitebox_ask(question, system, answer_format, max_tokens): + """One question to the local prompt model; a refusal, a timeout or a lost job is an OSError for pick().""" + try: + key = port.vlm_start('ask', dict(questions=[dict(text=question, images=[])], format=answer_format, + max_tokens=max_tokens, system=system)) + except JobError as error: + raise OSError(error.code) from None + deadline = time.monotonic() + WHITEBOX_ASK_SECONDS + while True: + try: + state = port.vlm_status(key) + except (ValueError, JobError): + raise OSError('lost') from None + phase = state.get('phase') + if phase == 'complete': + answers = state.get('answers') or [] + if len(answers) != 1 or not isinstance(answers[0], dict): + raise OSError('invalid_output') + return answers[0] + if phase not in ('starting', 'loading', 'rewriting', 'answering', 'finishing', 'cancelling'): + raise OSError(str(state.get('error') or phase)) + if time.monotonic() > deadline: + port.vlm_cancel(key) + raise TimeoutError('timeout') + time.sleep(.5) + @routes.post('/freevideo/agent/whitebox/try') @guarded async def whitebox_try(request): if request.content_type != 'application/json': return problem('unsupported_media_type', 'Send JSON', 415) value = await body(request) - row = whitebox_preset(value.get('preset') if isinstance(value, dict) else None) - if row is None: - raise JobError('not_found', 'No such ready-made scene') + if not isinstance(value, dict): + raise JobError('invalid_request', 'Send a JSON object') sid = value.get('session') - notice = None if sid is not None: store = work_sessions() if store is None or not isinstance(sid, str) or not store.has(chat['client'], sid): raise JobError('not_found', 'No such conversation') - # The conversation gets the confirmation card once the page has rendered the scene. - notice = dict(question=row['prompt'], summary=row['name']) - scene = await asyncio.to_thread(jobs.whitebox.create, row['code'], row['shot'], sid, notice) + shot = value.get('shot') + if (shot is None) == (value.get('preset') is None): + raise JobError('invalid_request', 'Send either a preset or a shot') + if shot is not None: + # The built-in AI picks a ready-made scene for the shot; when none fits, the video is made as usual. + picked = await asyncio.to_thread(whitebox_pick.pick, shot, whitebox_ask) + if picked['code'] is None: + return reply(dict(scene=None, reason=picked['reason'])) + row = whitebox_preset(picked['params']['template']) + code = picked['code'] + else: + row = whitebox_preset(value.get('preset')) + if row is None: + raise JobError('not_found', 'No such ready-made scene') + shot, code = row['shot'], row['code'] + # The conversation gets the confirmation card once the page has rendered the scene. + notice = dict(question=row['prompt'], summary=row['name']) if sid is not None else None + scene = await asyncio.to_thread(jobs.whitebox.create, code, shot, sid, notice) return reply(dict(scene, preset=row['id'], name=row['name'], prompt=row['prompt']), 201) @routes.post('/freevideo/agent/whitebox/{id}/video') diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index ad76ce1b..066a752c 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -491,8 +491,8 @@ def route(summary, body=None, query=None, **kw): for method, suffix, summary, extra in ( ('GET', '/next', 'The page asks for a white-model scene to render', {}), ('GET', '/presets', 'The ready-made white-model scenes the page can try', {}), - ('POST', '/try', 'The page renders one ready-made white-model scene (no model writes code)', - dict(body=obj(dict(preset=string(32, pattern=r'^[a-z0-9-]{1,32}$'), session=SESSION), ('preset',)))), + ('POST', '/try', 'The page renders one ready-made white-model scene, named (preset) or picked by the built-in AI for a shot (shot); no model writes code', + dict(body=obj(dict(preset=string(32, pattern=r'^[a-z0-9-]{1,32}$'), shot=string(4000), session=SESSION)))), ('POST', '/{id}/video', 'The page sends the rendered white-model video (the raw mp4 body)', dict(path=obj({'id': WHITEBOX}, ('id',)))), ('POST', '/{id}/failed', 'The page reports that a white-model scene could not be rendered', @@ -1648,7 +1648,7 @@ def _tool_schema(name): value['required'].append('shot') if name == 'freevideo_wait': value['properties'].pop('wait') - value['properties']['timeout'] = number(0, 50, default=25, description='With id at most 25 seconds; without id (the user) up to 50, default 50.') + value['properties']['timeout'] = number(0, 50, default=45, description='With id up to 45 seconds; without id (the user) up to 50, default 50.') value['required'] = [k for k in value['required'] if k != 'id'] return value if name == 'freevideo_asset': diff --git a/freevideo_engine/agent/skill/freevideo/SKILL.txt b/freevideo_engine/agent/skill/freevideo/SKILL.txt new file mode 100644 index 00000000..bf1211f1 --- /dev/null +++ b/freevideo_engine/agent/skill/freevideo/SKILL.txt @@ -0,0 +1,38 @@ +--- +name: freevideo +description: Make videos with the user's local FreeVideo through its MCP tools or the `freevideo agent` command line. Use when the user asks to generate or continue a video, edit a picture, cut a timeline or make a multi-shot film in FreeVideo. +--- + +# FreeVideo + +FreeVideo runs on the user's own computer. Everything below goes through its MCP server (tools named `freevideo_*`) or the same operations from a shell: `./freevideo agent ` (Windows: `freevideo.cmd agent`). + +## Before anything else + +1. Call `freevideo_status`. If it is not ready, tell the user what is missing; do not retry in a loop. +2. Read `freevideo_prompt_guide` once before writing a prompt. Prompts are English sections; dialogue and on-screen text stay in the user's language. +3. Call `freevideo_plan` with the job you intend to run. It returns the real frame count, the number of parts and a time estimate for this computer. Tell the user the estimate before a long run. +4. The user sees the same work in the FreeVideo window. A tool may answer `needs_confirmation`: wait for the user (`freevideo_wait` without an id), then repeat the identical call once approved. If the user declines, stop and say so; do not retry with a changed reference. + +## Pick the right task + +- One video, up to 60 seconds: `freevideo_generate`. Longer than the device can do in one run is cut into parts and joined automatically. +- Start from or end on a picture: pass it as first or last frame. Make the picture first with `freevideo_generate_image`. +- A film of several shots to choose between: the `freevideo_drama_*` tools. See `references/film.md`. +- Existing footage on a timeline: the editing project tools. See `references/edit.md`. +- A picture to change: `freevideo_edit_image` for a quick change, the `freevideo_image_*` tools for layered edits. See `references/image.md`. +- Several independent jobs: `freevideo_submit_batch`. + +## Running jobs + +- A job takes minutes. Call `freevideo_wait` with the job id; it returns after at most 25 seconds. If the job is still active, call it again with the same id. Never submit the same job twice. +- A failed or interrupted generation is continued with `freevideo_resume`; only unfinished parts run. +- `freevideo_cancel` only for a job you started and the user wants stopped. +- Look at a result before reporting it: `freevideo_look` (a frame or a contact sheet). +- Report what you made with the saved-video id (`freevideo_library`), not a local path. + +## Rules + +- Do not start a second generation while one for the same request is running; the computer has one GPU. +- Never claim a video is good without looking at it. +- Keep your own chat short; the user is watching progress in FreeVideo. diff --git a/freevideo_engine/agent/skill/freevideo/references/edit.txt b/freevideo_engine/agent/skill/freevideo/references/edit.txt new file mode 100644 index 00000000..18dfcf41 --- /dev/null +++ b/freevideo_engine/agent/skill/freevideo/references/edit.txt @@ -0,0 +1,11 @@ +# Editing projects + +Projects are the timelines shared with the FreeVideo editor page. + +1. `freevideo_projects` lists them; `freevideo_project_create` starts an empty one. +2. Footage from the user's disk: `freevideo_import_media`, then poll `freevideo_imports` until `ready` (`failed` is final). Insert the returned `import:` source. +3. Always `freevideo_project_read` first. Edits carry the revision and clip ids you just read. On a `project_changed` error, read again and recompute; do not retry the old edit. +4. `freevideo_editor_operations` applies 1 to 64 edits atomically. Prefer it to `freevideo_project_write`, which replaces the whole timeline. +5. `freevideo_project_generate` makes new picture and sound for selected clips or a frame range, as the Generate button does. +6. Subtitles: `freevideo_project_listen` with mode `align` times lines you give, `transcribe` writes down what is said. +7. `freevideo_export` renders one saved revision to MP4 in the saved videos. Wait for the returned job. diff --git a/freevideo_engine/agent/skill/freevideo/references/film.txt b/freevideo_engine/agent/skill/freevideo/references/film.txt new file mode 100644 index 00000000..e2a2f758 --- /dev/null +++ b/freevideo_engine/agent/skill/freevideo/references/film.txt @@ -0,0 +1,12 @@ +# Films of several shots + +Use this only when the film needs separately chosen shots. A single video of up to 60 seconds is one `freevideo_generate` job. + +1. Reuse or make cast cards: `freevideo_cast_list` first (reuse a matching card), then `freevideo_cast_create`; `freevideo_cast_generate` makes reference pictures when the user gave none. +2. `freevideo_drama_plan` previews the shots without starting. +3. `freevideo_drama_start` creates the durable film. Read it with `freevideo_drama_get` (use `since` and `wait`). +4. For each shot whose takes are done: `freevideo_drama_sheet` shows timed frames and the previous chosen last frame. Look, then `freevideo_drama_choose`. A continuation can only start after the previous shot is chosen. +5. A take that missed: `freevideo_drama_reshoot` with a short note on what to change, then review the sheet again. +6. When every shot is chosen: `freevideo_drama_finish`, then read the film until its output is ready. + +Only the user may pause or resume a film. If it is paused, stop acting and long-wait. diff --git a/freevideo_engine/agent/skill/freevideo/references/image.txt b/freevideo_engine/agent/skill/freevideo/references/image.txt new file mode 100644 index 00000000..775e9f2c --- /dev/null +++ b/freevideo_engine/agent/skill/freevideo/references/image.txt @@ -0,0 +1,6 @@ +# Pictures + +- Quick change (crop, rotate, flip, pad, adjust, text, shapes, overlays, remove burned-in captions): `freevideo_edit_image`. The result is a new input file usable as a first frame or reference. +- Layered edits like the image editor page: `freevideo_image_create` opens a session, `freevideo_image_read` gives the document and revision, `freevideo_image_operations` applies 1 to 64 edits atomically, `freevideo_image_render` makes a PNG input file. Read the revision and layer ids before each edit; on `image_changed` read again. +- A new picture from a prompt, or combining input pictures, or a first frame with cast members: `freevideo_generate_image`. +- Check the result with `freevideo_look` before using it. diff --git a/freevideo_engine/agent/skill_install.py b/freevideo_engine/agent/skill_install.py new file mode 100644 index 00000000..c68ed439 --- /dev/null +++ b/freevideo_engine/agent/skill_install.py @@ -0,0 +1,38 @@ +"""Copy the bundled FreeVideo skill folder where Agents that support SKILL.md look for it.""" +from pathlib import Path +import shutil +import tempfile + +SOURCE = Path(__file__).resolve().parent / 'skill' / 'freevideo' +# Codex, Gemini CLI and Cursor read ~/.agents/skills; Claude Code reads ~/.claude/skills. +TARGETS = {'agents': ('.agents', 'skills'), 'claude': ('.claude', 'skills')} + + +def target_dir(target, home=None): + return Path(home or Path.home()).joinpath(*TARGETS[target]) / 'freevideo' + + +def install(targets=('agents', 'claude'), home=None): + """Replace the installed copy (only our own freevideo folder) and return the paths written.""" + written = [] + for target in targets: + destination = target_dir(target, home) + destination.parent.mkdir(parents=True, exist_ok=True) + # A fresh folder made exclusively beside the destination: a leftover or a link with a fixed name is never written through. + staging = Path(tempfile.mkdtemp(prefix='.freevideo-installing-', dir=destination.parent)) + try: + for source in SOURCE.rglob('*.txt'): + # Stored as .txt so the update payload only carries file types every launcher accepts; agents read .md. + copy = staging / source.relative_to(SOURCE).with_suffix('.md') + copy.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, copy) + if destination.is_symlink(): + destination.unlink() + elif destination.exists(): + shutil.rmtree(destination) + staging.chmod(0o755) # mkdtemp creates 0700; the installed folder keeps the usual readable mode + staging.rename(destination) + finally: + shutil.rmtree(staging, ignore_errors=True) + written.append(str(destination)) + return written diff --git a/freevideo_engine/agent/whitebox_pick.py b/freevideo_engine/agent/whitebox_pick.py new file mode 100644 index 00000000..098ca1f2 --- /dev/null +++ b/freevideo_engine/agent/whitebox_pick.py @@ -0,0 +1,144 @@ +"""The built-in AI picks a ready-made white-model scene for a shot instead of writing scene code. + +Two short questions go to the local prompt model: first whether any template fits at all (a single 是/否), then, +only on a yes, a five-field JSON choice. A no, an unreadable answer or an invalid choice means no white model: +the video is generated as usual. Scene code is never taken from the model; the chosen template's code runs +with its few parameters applied by a fixed wrapper.""" +import json +import math +import re + +from .whitebox_presets import preset + +# The templates the built-in AI may choose: those in the ready-made scenes whose generated video was measured to +# follow the white model. Adding or retiring one is a change to the scenes and these tables. +ENABLED = ('cat',) +GATE_NAMES = dict(cat='猫在客厅逐层跳上书架') +PICK_NAMES = dict(cat='橘猫跳上书架') +# Camera height offsets (metres) each template tolerates without the camera meeting a wall or leaving the subject. +SAFE_HEIGHT = dict(cat=(-0.5, 0.5)) +LIMITS = (('speed', .7, 1.3), ('camera_height', -1, 3), ('fov_scale', .8, 1.25)) +KEYS = ('template', 'hero_color', 'speed', 'camera_height', 'fov_scale') +NONE = dict(template='none', hero_color=None, speed=1, camera_height=0, fov_scale=1) +GATE_TOKENS = 16 +PICK_TOKENS = 120 +SYSTEM = '你是视频制作助手，严格按要求的格式回答。' + +GATE_PROMPT = """判断镜头能否用以下任一固定模板，只答是或否。 +模板：{NAMES} +镜头与模板的动作基本一致即答是。只调颜色、速度、机位高度、视野无法实现的镜头答否。 +镜头：{SHOT}""" +PICK_PROMPT = """镜头：{SHOT} +模板：{NAMES} +只输出JSON，五字段全齐：template(以上id；都不像选none)，hero_color(主体#RRGGBB；未提颜色填null)，speed(时间倍率0.7~1.3)，camera_height(相机高度偏移-1~3米)，fov_scale(视野倍率0.8~1.25)。未指定数值用1、0、1。""" + +# Applies the parameters to a built template; embedded in the compiled scene as it is. +APPLY = """function applyParameters(base, p) { + const {scene, camera} = base; + if (p.hero_color !== null) { + scene.traverse(object => { + if (!object.isMesh) return; + let hero = false; + for (let node = object; node; node = node.parent) { + if ((node.name || '').startsWith('hero')) { hero = true; break; } + } + if (!hero) return; + const recolor = material => { + const copy = material.clone(); + if (copy.color) copy.color.set(p.hero_color); + return copy; + }; + object.material = Array.isArray(object.material) ? object.material.map(recolor) : recolor(object.material); + }); + } + camera.fov *= p.fov_scale; + camera.updateProjectionMatrix(); + let previousOffset = 0; + return {scene, camera, update(t) { + if (typeof t !== 'number' || !Number.isFinite(t)) throw new Error('invalid_time'); + camera.position.y -= previousOffset; + base.update(Math.min(p.seconds, Math.max(0, t * p.speed))); + camera.position.y += p.camera_height; + previousOffset = p.camera_height; + }}; +}""" + + +def gate_question(shot): + return GATE_PROMPT.format(NAMES='；'.join(GATE_NAMES[i] for i in ENABLED), SHOT=shot) + + +def pick_question(shot): + return PICK_PROMPT.format(NAMES='；'.join(i + ' ' + PICK_NAMES[i] for i in ENABLED) + '。', SHOT=shot) + + +def gate_allows(text): + """True for 是, False for 否; anything else is not an answer.""" + answer = text.strip() if isinstance(text, str) else '' + if answer not in ('是', '否'): + raise ValueError('gate_invalid') + return answer == '是' + + +def parameters(value): + """The five fields, checked strictly; the choice may name none or an enabled template only.""" + if not isinstance(value, dict) or set(value) != set(KEYS): + raise ValueError('params_invalid') + if value['template'] != 'none' and value['template'] not in ENABLED: + raise ValueError('params_invalid') + color = value['hero_color'] + if color is not None and not (isinstance(color, str) and re.fullmatch(r'#[0-9a-fA-F]{6}', color)): + raise ValueError('params_invalid') + for key, low, high in LIMITS: + number = value[key] + if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number) \ + or not low <= number <= high: + raise ValueError('params_invalid') + return {key: value[key] for key in KEYS} + + +def effective(value): + """The parameters as rendered: camera height limited to what the template can hold.""" + result = parameters(value) + if result['template'] != 'none': + low, high = SAFE_HEIGHT[result['template']] + result['camera_height'] = min(high, max(low, result['camera_height'])) + return result + + +def compile_scene(value): + """Scene code for the page: the template's own code plus the fixed parameter wrapper; None for no template.""" + params = effective(value) + if params['template'] == 'none': + return None + row = preset(params['template']) + if row is None: + raise ValueError('template_invalid') + source = row['code'] + seconds = row['seconds'] + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or not 0 < seconds <= 12: + raise ValueError('template_invalid') + head = 'export function build(THREE)' + if source.count(head) != 1: + raise ValueError('template_invalid') + return (source.replace(head, 'function buildTemplate(THREE)') + '\n' + APPLY + '\nexport function build(THREE) {\n' + ' return applyParameters(buildTemplate(THREE), ' + json.dumps(dict(params, seconds=seconds)) + ');\n}\n') + + +def pick(shot, ask): + """Choose a scene for the shot. `ask(question, system, answer_format, max_tokens)` returns the model's answer + row (`text`, and `json` for the JSON format). The result always has `params`; `code` is None, and `reason` + says why, whenever no white model should be made. A failure to ask at all (timeout, connection) is one of those.""" + try: + if not gate_allows(ask(gate_question(shot), SYSTEM, 'text', GATE_TOKENS).get('text')): + return dict(params=dict(NONE), code=None, reason='gate_no') + row = ask(pick_question(shot), SYSTEM, 'json', PICK_TOKENS) + params = effective(row.get('json')) + code = compile_scene(params) + except (OSError, TimeoutError): + # The model could not be asked (timeout, connection): the video is made without a white model. + return dict(params=dict(NONE), code=None, reason='ask_failed') + except (ValueError, AttributeError, TypeError, KeyError) as error: + reason = str(error) + return dict(params=dict(NONE), code=None, reason=reason if reason in ('gate_invalid', 'params_invalid') else 'params_invalid') + return dict(params=params, code=code, reason='none' if code is None else 'template') diff --git a/freevideo_engine/agent/whitebox_presets.py b/freevideo_engine/agent/whitebox_presets.py index 450d1465..7f700d97 100644 --- a/freevideo_engine/agent/whitebox_presets.py +++ b/freevideo_engine/agent/whitebox_presets.py @@ -3,6 +3,7 @@ PRESETS = ( dict(id='cat', name='橘猫跳上书架', shot='一只橘猫在客厅地板上蹲下蓄力，沿着错落的书架层板一级一级跳上去，最后蹲在最高一层，镜头侧面平移并略微上摇跟随。', prompt='A ginger cat crouches on a living room floor, gathers itself and leaps up a staggered wooden bookshelf, jumping from one shelf to the next between rows of books, and finally settles on the top shelf, while the camera pans sideways and tilts up slightly to follow. Warm indoor daylight, realistic fur and wood, soft shadows, smooth camera motion. Soft room tone and the light thud of paws on wood.', + seconds=5, code='export function build(THREE) {\n const scene = new THREE.Scene();\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 300);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name, parent) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || "";\n m.castShadow = true; m.receiveShadow = true; (parent || scene).add(m); return m;\n };\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const clamp01 = x => Math.min(1, Math.max(0, x));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048);\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60; sun.shadow.camera.far = 200;\n scene.add(sun);\n\n scene.background = new THREE.Color(0xe9e2d6);\n scene.add(new THREE.HemisphereLight(0xfff4e6, 0x8a7f72, 1.2)); sun.position.set(6, 10, 8);\n add(new THREE.PlaneGeometry(14, 10).rotateX(-Math.PI / 2), std(0xb88f65), 0, 0, 0, "floor");\n add(new THREE.BoxGeometry(14, 4, 0.2), std(0xe9e2d6), 0, 2, -1.2, "wall");\n // bookshelf: two side boards, staggered shelves\n add(new THREE.BoxGeometry(0.1, 2.6, 0.5), std(0x7a5232), -1.4, 1.3, -0.9, "side");\n add(new THREE.BoxGeometry(0.1, 2.6, 0.5), std(0x7a5232), 1.4, 1.3, -0.9, "side");\n const shelves = [[0.6, -0.9], [1.2, -0.4], [1.8, 0.2], [2.4, 0.6]];\n const stepsX = [-0.9, -0.2, 0.5, 1.0], ys = [0.55, 1.05, 1.55, 2.1];\n for (let i = 0; i < 4; i++) {\n add(new THREE.BoxGeometry(1.0, 0.06, 0.5), std(0x9a6b3f), stepsX[i], ys[i] - 0.03, -0.9, "shelf");\n for (let k = 0; k < 3; k++) add(new THREE.BoxGeometry(0.12, 0.28, 0.3), std([0x3c6ea8, 0xb04a4a, 0x4a8a5a][(i + k) % 3]), stepsX[i] - 0.3 + k * 0.2, ys[i] + 0.14, -1.0, "book");\n }\n add(new THREE.BoxGeometry(0.9, 0.5, 0.5), std(0x6b7b8c), -2.6, 0.25, 0.6, "sofa_block");\n const cat = new THREE.Group(); cat.name = "hero_cat"; scene.add(cat);\n const orange = std(0xe58a2e);\n const body = add(new THREE.BoxGeometry(0.42, 0.22, 0.2), orange, 0, 0.18, 0, "hero_cat_body", cat);\n add(new THREE.SphereGeometry(0.11, 10, 8), orange, 0.26, 0.28, 0, "hero_cat_head", cat);\n add(new THREE.ConeGeometry(0.035, 0.08, 4), orange, 0.28, 0.4, 0.06, "hero_cat_ear", cat);\n add(new THREE.ConeGeometry(0.035, 0.08, 4), orange, 0.28, 0.4, -0.06, "hero_cat_ear", cat);\n const tail = add(new THREE.CylinderGeometry(0.03, 0.03, 0.4, 6), orange, -0.34, 0.3, 0, "hero_cat_tail", cat);\n tail.rotation.z = 0.8;\n // waypoints: floor crouch, jump to each shelf\n const wp = [[-1.9, 0.0, 0.1], [-1.9, 0.0, 0.1], [stepsX[0], ys[0], -0.9], [stepsX[1], ys[1], -0.9], [stepsX[2], ys[2], -0.9], [stepsX[3], ys[3], -0.9], [stepsX[3], ys[3], -0.9]];\n const times = [0, 0.7, 1.4, 2.3, 3.2, 4.1, 5.0];\n const pos = t => { let i = 0; while (i < 5 && t > times[i + 1]) i++;\n const u = clamp01((t - times[i]) / (times[i + 1] - times[i])); const a = wp[i], b = wp[i + 1];\n const arc = (i >= 1 && i <= 4) ? Math.sin(Math.PI * u) * 0.45 : 0;\n const e = ss(u); return new THREE.Vector3(lerp(a[0], b[0], e), lerp(a[1], b[1], u) + arc, lerp(a[2], b[2], e)); };\n return { scene, camera, update(t) {\n const p = pos(t); cat.position.copy(p);\n cat.scale.y = (t < 0.7) ? 1 - 0.35 * Math.sin(Math.PI * clamp01(t / 0.7)) : 1;\n cat.rotation.z = (t > 0.7 && t < 4.1) ? 0.35 : 0;\n camera.position.set(p.x * 0.6, 0.9 + p.y * 0.5, 2.7);\n camera.lookAt(p.x * 0.85, 0.35 + p.y * 0.85, -0.9);\n } };\n}\n'), ) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index b3975267..e9d02978 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -1579,8 +1579,9 @@ def update_progress(*, final=False): if self.ui.verbose or not success else 'Ready') if child.returncode: from .failure_details import setup_command_failure, environment_python_removed + report = args[args.index('--out') + 1] if '--out' in args[:-1] else None raise RuntimeError(environment_python_removed(args, getattr(self, 'pythons', {}).values()) - + setup_command_failure(label, log, child.returncode)) + + setup_command_failure(label, log, child.returncode, report=report)) return log def component_tasks(self, uv): diff --git a/freevideo_engine/capabilities.py b/freevideo_engine/capabilities.py index e7323281..a2c027d2 100644 --- a/freevideo_engine/capabilities.py +++ b/freevideo_engine/capabilities.py @@ -55,7 +55,7 @@ def normalize_references(task, references=None): kind = row.get('kind') allowed = {'kind', 'seconds'} if kind == 'audio' else {'kind', 'width', 'height'} if kind == 'video': - allowed |= {'seconds', 'audio_seconds'} + allowed |= {'seconds', 'audio_seconds', 'guide'} if kind not in ('image', 'video', 'audio') or set(row) - allowed: raise ValueError('Unsupported reference metadata.') width = height = seconds = audio = None @@ -71,6 +71,12 @@ def normalize_references(task, references=None): raise ValueError('Reference audio is empty after normalization.') if kind == 'video' and math.ceil(min(seconds, 15.) * 24) < 22: raise ValueError('The reference video needs at least 22 normalized frames.') + if 'guide' in row: + # A guide clip is packed as a silent video reference (same rows and + # patches; only the time coordinates differ), so it keeps the plain + # video tuple and history matching counts it as reference video tokens. + if type(row['guide']) is not bool or 'audio_seconds' in row: + raise ValueError('A guide clip is a silent video reference.') if 'audio_seconds' in row: audio = row['audio_seconds'] if not positive(audio): diff --git a/freevideo_engine/chain_plan.py b/freevideo_engine/chain_plan.py index 5eb36f56..dfbc0281 100644 --- a/freevideo_engine/chain_plan.py +++ b/freevideo_engine/chain_plan.py @@ -5,20 +5,47 @@ cannot do at once, are cut into blocks that continue one another. A block length given explicitly also cuts a video under 30 s. """ +import json import math +from pathlib import Path from .geometry import geometry FPS, UNIT, OVERLAP = 24, 17, 22 SINGLE_SECONDS, BLOCK_SECONDS, SHORTEST_BLOCK = 30, 15, 5 -# A part that continues another is at least this long (the first part is not held to it). -MIN_PART_SECONDS = 10 -MIN_PART_FRAMES = geometry(seconds=MIN_PART_SECONDS)['frames'] +# The limits on continuing parts, shared with the page (web/chain_rules.js: one JSON object) so that both apply +# the same numbers. +RULES = json.loads((Path(__file__).resolve().parents[1] / 'web' / 'chain_rules.js').read_text(encoding='utf-8') + .partition('=')[2].strip().removesuffix(';')) +# A part that continues another is at least MIN_PART_SECONDS (the first part is not held to it). A video with a +# continuing part shorter than FULL_PART_SECONDS has at most SHORT_MAX_PARTS parts, or MAX_PARTS with the +# first-frame reference; every other video has at most MAX_PARTS. +MIN_PART_SECONDS, FULL_PART_SECONDS = RULES['min_part_seconds'], RULES['full_part_seconds'] +MAX_PARTS, SHORT_MAX_PARTS = RULES['max_parts'], RULES['short_max_parts'] +# The length a new continuation starts with, and its length on devices that cannot continue with a full part. +CONTINUE_SECONDS, SHORT_CONTINUE_SECONDS = RULES['continue_seconds'], RULES['short_continue_seconds'] + + +def part_frames(seconds): + """The fewest frames a continuing block of `seconds` whole seconds asks for. Each block takes the shortest step + that reaches its whole second, so a block after another may ask for up to UNIT - 1 frames fewer than the same + length on its own (5 s alone is 158 frames, after another part 141).""" + return OVERLAP + UNIT * math.ceil((FPS * seconds - (UNIT - 1)) / UNIT) + + +# Continuing blocks below MIN_PART_FRAMES are too short; below FULL_PART_FRAMES they are short parts. +MIN_PART_FRAMES, FULL_PART_FRAMES = part_frames(MIN_PART_SECONDS), part_frames(FULL_PART_SECONDS) SHORTEST_FIRST = 2 # whole seconds: what the first part keeps when the continuing parts take the rest MAX_SECONDS = 7200 MAX_BLOCKS = MAX_SECONDS // SHORTEST_BLOCK +def parts_limit(short, first_frame=False, max_parts=MAX_PARTS): + """The most parts one video may have: fewer when one of its continuing parts is short, unless every + continuing part also sees the first frame.""" + return max_parts if not short or first_frame else min(max_parts, SHORT_MAX_PARTS) + + class PlanError(ValueError): def __init__(self, code, message): super().__init__(message) @@ -35,14 +62,41 @@ def _first_frames(seconds): return geometry(seconds=seconds)['frames'] -def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, block_seconds=None, continued=False, max_parts=None): +def _lengths(total, first, following, least, continued): + """Part lengths in seconds with every continuing part from `least` to `following` seconds, or None.""" + if first < (least if continued else SHORTEST_BLOCK) or following < least: + return None + # The part that starts a new video is not held to `least`; a rest too short for the continuing parts is + # taken from that part. + rest = total if continued else total - min(first, total) + count = math.ceil(round(rest, 6) / following) if rest > 0 else 0 + rest = max(rest, least * count) + whole = math.ceil(round(rest, 6)) + if continued and round(rest - total, 6) > 0 or not continued and total - whole < SHORTEST_FIRST: + return None + if count > MAX_BLOCKS: + raise PlanError('too_long', 'This length needs too many blocks') + lengths = [whole // count + (index < whole % count) for index in range(count)] + if continued: + lengths[0] = round(lengths[0] - (whole - total), 6) # a fractional request keeps its length in the first part + else: + lengths.insert(0, round(total - whole, 6)) + return lengths + + +def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, block_seconds=None, continued=False, + max_parts=None, first_frame=False): """`max_frames`: the most frames this device generates at once for the current settings (unknown: 15 s). `continued`: the video goes on from a saved one, so its first block too spends OVERLAP frames on what came before and adds only the rest. - `max_parts`: the most parts one video may have. A new video that needs more is refused here, before anything - is submitted; a continued one adds to parts this planner does not know, so the server checks it on submission. + `max_parts`: the most parts one video may have; `first_frame`: every continuing part also sees the first + frame, which keeps that limit when a part is short. A video that needs more parts is refused here, before + anything is submitted; a continued one adds to parts this planner does not know, so it counts one for them + and the server checks the rest on submission. + + Continuing parts are FULL_PART_SECONDS or longer whenever the length allows it, else at least MIN_PART_SECONDS. """ _number(seconds, 'Length') if type(single) is not bool: @@ -88,32 +142,31 @@ def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, bloc return alone # a block this short cannot continue another: a short video stays one generation raise PlanError('block_too_short', 'A block that continues another is at least %d s long' % MIN_PART_SECONDS) first, following = min(block, head), min(block, following) - shortest = MIN_PART_SECONDS if continued else SHORTEST_BLOCK - if first < shortest or following < MIN_PART_SECONDS: - if fits and seconds < SINGLE_SECONDS: - return alone # a requested block length never makes a part this device cannot do - raise PlanError('device_limit', 'This device cannot generate this length at this size') # Under 30 s the length stays as asked (it may be fractional); from 30 s it is whole seconds already. total = seconds - # Every part that continues another is at least MIN_PART_SECONDS and at most `following`. The part that - # starts a new video is not held to it; a rest too short for the continuing parts is taken from that part. - rest = total if continued else total - min(first, total) - count = math.ceil(round(rest, 6) / following) if rest > 0 else 0 - rest = max(rest, MIN_PART_SECONDS * count) - whole = math.ceil(round(rest, 6)) - if continued and whole > math.ceil(round(total, 6)) or not continued and total - whole < SHORTEST_FIRST: + lengths = None + for least in (FULL_PART_SECONDS, MIN_PART_SECONDS): + if least < FULL_PART_SECONDS and block >= FULL_PART_SECONDS and fits and seconds < SINGLE_SECONDS: + break # a video that fits at once stays one generation rather than take short parts + lengths = _lengths(total, first, following, least, continued) + if lengths is not None: + break + if lengths is None: if fits and seconds < SINGLE_SECONDS: - return alone + return alone # a requested block length never makes a part this device cannot do + if first < (MIN_PART_SECONDS if continued else SHORTEST_BLOCK) or following < MIN_PART_SECONDS: + raise PlanError('device_limit', 'This device cannot generate this length at this size') raise PlanError('length_not_splittable', 'This length cannot be cut into parts of the requested length') - if count > MAX_BLOCKS: - raise PlanError('too_long', 'This length needs too many blocks') - if max_parts is not None and not continued and count + 1 > max_parts: - raise PlanError('too_many_parts', 'One continuous shot can have up to %d parts. Make it shorter.' % max_parts) - lengths = [whole // count + (index < whole % count) for index in range(count)] - if continued: - lengths[0] = round(lengths[0] - (whole - total), 6) # a fractional request keeps its length in the first part - else: - lengths.insert(0, round(total - whole, 6)) + if max_parts is not None: + parts = len(lengths) + continued # a continued video has at least one part before these + short = any(length < FULL_PART_SECONDS for length in lengths[0 if continued else 1:]) + allowed = parts_limit(short, first_frame, max_parts) + if parts > allowed and allowed < max_parts: + raise PlanError('too_many_short_parts', 'With a part shorter than %d seconds, one continuous shot can have up to ' + '%d parts. Make every part at least %d seconds long, or use fewer parts.' + % (FULL_PART_SECONDS, allowed, FULL_PART_SECONDS)) + if parts > allowed: + raise PlanError('too_many_parts', 'One continuous shot can have up to %d parts. Make it shorter.' % max_parts) frames_all = int(round(FPS * total)) blocks, shown = [], 0 for index, length in enumerate(lengths): diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 37439109..401ee784 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -5,6 +5,7 @@ import math from pathlib import Path, PurePosixPath import random +import re import threading import time import uuid @@ -12,6 +13,30 @@ from . import chain_leads, chain_plan from . import chain_store as store +REPORT_ID = re.compile(r'[a-f0-9]{32}') + + +def _split_value(record, text): + """What the prompt model sorts into parts: the whole description and each part's new seconds.""" + first = next(row for row in record['blocks'] if row['index'] == 0) + return dict(text=text, continued=first['after'] is not None, + seconds=[row['new_frames'] / chain_plan.FPS for row in record['blocks']]) + + +def _ordered_texts(record, value): + """Part texts when the description could not be split: its sentences in order, with the LoRA trigger + words every part carries, or None for a description without sentences.""" + from .lora_triggers import apply_chain_triggers + from .prompt_vlm.rules import order_split + if not isinstance(value['text'], str): + return None + try: + probe = dict(snapshot=deepcopy(record['snapshot']), node=record['node'], + blocks=[dict(inputs=dict(text=text)) for text in order_split(value)]) + apply_chain_triggers(probe) + except Exception: + return None + return [row['inputs']['text'] for row in probe['blocks']] class ChainBusy(ValueError): pass @@ -27,6 +52,18 @@ def __init__(self, parts, limit): self.message_zh = '一个连续镜头最多 %d 段，请缩短。' % limit +class ChainShortPartsTooMany(ChainTooLong): + """A video with a short continuing part, continued through more parts than such a video may have.""" + code = 'chain_short_parts' + + def __init__(self, parts, limit): + full = chain_plan.FULL_PART_SECONDS + ValueError.__init__(self, 'With a part shorter than %d seconds, one continuous shot can have up to %d parts. ' + 'Make every part at least %d seconds long, or use fewer parts.' % (full, limit, full)) + self.parts, self.limit = parts, limit + self.message_zh = '有短于 %d 秒的段时，一个连续镜头最多 %d 段。请把每段加长到至少 %d 秒，或减少段数。' % (full, limit, full) + + class ChainPartTooShort(ValueError): """A continuing part shorter than a continued video may have.""" code = 'chain_part_too_short' @@ -34,7 +71,7 @@ class ChainPartTooShort(ValueError): def __init__(self, seconds, limit): super().__init__('Each continued part must be at least %d seconds. Make it longer.' % limit) self.seconds, self.limit = seconds, limit - self.message_zh = '接着生成的每一段至少 %d 秒，请加长一些。' % limit + self.message_zh = '接着生成的每一段至少 %d 秒，请加长这一段。' % limit def _work_key(path): @@ -69,11 +106,15 @@ def node_prompt(prompt, node): class ChainRunner: def __init__(self, store_dir, host, clock=time.monotonic, wall=time.time, estimate=None, poll=0.5, inputs=None, - max_parts=None, min_part_frames=None, seeds=None): + max_parts=None, min_part_frames=None, seeds=None, full_part_frames=None): self.store_dir, self.host = store_dir, host self.seeds = seeds or random.SystemRandom() self.max_parts = max_parts or store.MAX_PARTS self.min_part_frames = chain_plan.MIN_PART_FRAMES if min_part_frames is None else min_part_frames + # Continuing parts below this are short and hold their video to fewer parts. A runner given a lower + # shortest part (a test host) lowers this with it unless it is given too. + self.full_part_frames = (full_part_frames if full_part_frames is not None else chain_plan.FULL_PART_FRAMES + if min_part_frames is None else min(min_part_frames, chain_plan.FULL_PART_FRAMES)) self.clock, self.wall, self.estimate, self.poll = clock, wall, estimate, poll self.inputs = inputs or host.inputs self.lock = threading.RLock() @@ -219,33 +260,45 @@ def _depths(self, record, seen=()): """Parts each block of `record` continues through, itself included (a block with no source is the first part). A source that is an existing video counts the parts that video was made of, as far as this runner's records know them; one they do not know counts as one part.""" + return {name: parts for name, (parts, _) in self._lineage(record, seen).items()} + + def _lineage(self, record, seen=()): + """For each block of `record`: its parts as `_depths` counts them, and whether any continuing part + among them is short.""" rows = {row['id']: row for row in record['blocks']} - depth, pending = {}, list(rows) + lineage, pending = {}, list(rows) while pending: rest = [] for identity in pending: row = rows[identity] sources = [row[key] for key in ('after', 'before') if row[key] is not None] - if any(isinstance(value, str) and value not in depth for value in sources): + if any(isinstance(value, str) and value not in lineage for value in sources): rest.append(identity) continue - depth[identity] = 1 + sum(depth[value] if isinstance(value, str) - else self._source_parts(value, seen + (record['id'],)) - for value in sources) + found = [lineage[value] if isinstance(value, str) else self._source_lineage(value, seen + (record['id'],)) + for value in sources] + short = bool(sources) and row['inputs']['frames'] < self.full_part_frames + lineage[identity] = (1 + sum(parts for parts, _ in found), short or any(flag for _, flag in found)) if len(rest) == len(pending): raise ValueError('Block references cannot contain a cycle or self reference') pending = rest - return depth + return lineage def _source_parts(self, value, seen): + return self._source_lineage(value, seen)[0] + + def _source_lineage(self, value, seen): if 'chain' not in value: - return self._work_parts(value['work'], seen) + return self._work_lineage(value['work'], seen) self._referenced_block(value) if value['chain'] in seen: raise ValueError('Chain references cannot contain a cycle') - return self._depths(self.records[value['chain']], seen)[value['block']] + return self._lineage(self.records[value['chain']], seen)[value['block']] def _work_parts(self, work, seen): + return self._work_lineage(work, seen)[0] + + def _work_lineage(self, work, seen): key = _work_key(work) for record in list(self.records.values()): if record['id'] in seen: @@ -255,9 +308,27 @@ def _work_parts(self, work, seen): else: names = [row['id'] for row in record['blocks'] if row['output'] and _work_key(row['output']) == key] if names: - depth = self._depths(record, seen) - return max(depth[name] for name in names) - return 1 + lineage = self._lineage(record, seen) + return max(lineage[name][0] for name in names), any(lineage[name][1] for name in names) + return 1, False + + def _check_parts(self, record): + """Refuses, before anything is written or queued, a chain with a continuing part too short or a video + continued through too many parts.""" + short = [row['inputs']['frames'] for row in record['blocks'] + if (row['after'] is not None or row['before'] is not None) + and row['inputs']['frames'] < self.min_part_frames] + if short: + raise ChainPartTooShort(min(short) / chain_plan.FPS, chain_plan.MIN_PART_SECONDS) + lineage = self._lineage(record).values() + parts = max(parts for parts, _ in lineage) + if parts > self.max_parts: + raise ChainTooLong(parts, self.max_parts) + inputs = record['snapshot']['prompt'][record['node']].get('inputs') or {} + limit = chain_plan.parts_limit(True, inputs.get('first_frame_reference') is True, self.max_parts) + parts = max((parts for parts, flag in lineage if flag), default=0) + if parts > limit: + raise ChainShortPartsTooMany(parts, limit) def create(self, payload): from .lora_triggers import apply_chain_triggers @@ -265,14 +336,7 @@ def create(self, payload): apply_chain_triggers(record) with self.lock: # Before anything is written or queued: a refused chain leaves every record and file as it was. - short = [row['inputs']['frames'] for row in record['blocks'] - if (row['after'] is not None or row['before'] is not None) - and row['inputs']['frames'] < self.min_part_frames] - if short: - raise ChainPartTooShort(min(short) / chain_plan.FPS, chain_plan.MIN_PART_SECONDS) - parts = max(self._depths(record).values()) - if parts > self.max_parts: - raise ChainTooLong(parts, self.max_parts) + self._check_parts(record) record['created'] = self.wall() self._save(record) self.records[record['id']] = record @@ -334,10 +398,14 @@ def resume(self, chain_id, client_id=None): fraction=0., remaining_seconds=None, continuation=None, skip_frames=0, skip_audio_samples=0) split = record.get('split') + whole = record['snapshot']['prompt'][record['node']]['inputs'].get('text') if (split is not None and split['state'] == 'failed' and split['reason'] in ('busy', 'memory') and all(row['state'] != 'done' for row in record['blocks']) - and len({row['inputs']['text'] for row in record['blocks']}) == 1): + and [row['inputs']['text'] for row in record['blocks']] in ( + [whole] * len(record['blocks']), _ordered_texts(record, _split_value(record, whole)))): # Busy and memory pass; nothing has started and no part was rewritten, so split again. + for row in record['blocks']: + row['inputs']['text'] = whole split.update(state='pending', reason=None) if client_id is not None: record['client_id'] = client_id @@ -410,14 +478,7 @@ def redo(self, chain_id, block_id, text=None, client_id=None): except Exception: raise ValueError('Text must be at most 20000 characters with LoRA triggers') from None record['blocks'][start['index']]['inputs']['text'] = probe['blocks'][0]['inputs']['text'] - short = [row['inputs']['frames'] for row in record['blocks'] - if (row['after'] is not None or row['before'] is not None) - and row['inputs']['frames'] < self.min_part_frames] - if short: - raise ChainPartTooShort(min(short) / chain_plan.FPS, chain_plan.MIN_PART_SECONDS) - parts = max(self._depths(record).values()) - if parts > self.max_parts: - raise ChainTooLong(parts, self.max_parts) + self._check_parts(record) # The kept seams are repaired with the sound the original chain kept for them. Copies left by # a failed save have no record, so the next start prunes them. try: @@ -690,8 +751,7 @@ def _step_chain(self, identity): split['state'] = 'running' record['state'] = 'running' self._save(record) - value = dict(text=block['inputs']['text'], continued=block['after'] is not None, - seconds=[row['new_frames'] / chain_plan.FPS for row in record['blocks']]) + value = _split_value(record, block['inputs']['text']) action = 'split' elif prompt_id is None: self.progress_messages.pop(identity, None) @@ -819,9 +879,13 @@ def cancelled(): row['inputs']['text'] = value['text'] record['split'].update(state='failed', reason='failed') else: - # The whole description goes to every part, as before; the page says why. + # Without the prompt model the sentences go to the parts in order, so no part plays the whole + # story again; the page says why. A description without sentences goes to every part. if parts is not None: reason = 'unusable' + texts = _ordered_texts(record, value) + for row, text in zip(record['blocks'], texts or [value['text']] * len(record['blocks'])): + row['inputs']['text'] = text record['split'].update(state='failed', reason=reason if reason in store.SPLIT_REASONS else 'failed') self._save(record) @@ -898,6 +962,12 @@ def progress(self, node, message): block = self._current(record) if block is None or block['state'] != 'running': return None + token = message.get('report_id') + if isinstance(token, str) and REPORT_ID.fullmatch(token) and block['report_id'] != token: + # A restart ends this block before `finished` stores its report, and the + # report of the interrupted run is the one a failure needs. + block['report_id'] = record['last_report_id'] = token + self._save(record) overall = message.get('overall') or {} used = _number(overall.get('elapsed_seconds')) used = self._elapsed(block) if used is None else used diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py index d7e536f4..cd4bb63f 100644 --- a/freevideo_engine/chain_store.py +++ b/freevideo_engine/chain_store.py @@ -19,15 +19,15 @@ # pending: split before the first part; given: the request gave one text per part; none: nothing to split. SPLIT_STATES = ('pending', 'running', 'done', 'failed', 'given', 'none') SPLIT_REASONS = { - 'model_missing': 'The prompt model is not installed, so every part used the whole description.', - 'busy': 'The prompt model was busy, so every part used the whole description.', - 'memory': 'There was not enough memory to split the description, so every part used the whole description.', - 'unusable': 'The description could not be split into parts, so every part used the whole description.', - 'unavailable': 'Splitting is not available here, so every part used the whole description.', - 'failed': 'Splitting the description failed, so every part used the whole description.', + 'model_missing': 'The prompt model is not installed, so the parts took the sentences of the description in order.', + 'busy': 'The prompt model was busy, so the parts took the sentences of the description in order.', + 'memory': 'There was not enough memory to split the description, so the parts took the sentences of the description in order.', + 'unusable': 'The description could not be split into parts, so the parts took the sentences of the description in order.', + 'unavailable': 'Splitting is not available here, so the parts took the sentences of the description in order.', + 'failed': 'Splitting the description failed, so the parts took the sentences of the description in order.', } -# Parts one video may be continued through: the first part and seven continuations. -MAX_PARTS = 8 +# Parts one video may be continued through: the first part and seven continuations (fewer with a short part). +MAX_PARTS = chain_plan.MAX_PARTS def reference(value, queue=False): diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 8d4f8a7a..0d085fa3 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -75,8 +75,14 @@ def resolve_assets(serialized, input_directory): root = Path(input_directory).resolve() result = {} for row in rows: - if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound'}: + if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth'}: raise ValueError('Invalid media item') + if 'depth' in row: + if type(row['depth']) is not bool: + raise ValueError('Media depth must be a boolean') + if (row.get('role') != 'reference' or not isinstance(row.get('file'), str) + or media_kind(row['file']) != 'video'): + raise ValueError('depth is only allowed on video references') # The character an @ mention added the row for; a continued part keeps one picture per character. if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) != {'id'} or not isinstance(row['cast']['id'], str) or not 0 < len(row['cast']['id']) <= 128): @@ -106,6 +112,8 @@ def resolve_assets(serialized, input_directory): result[role] = str(path) elif role == 'reference': reference = {'kind': kind, 'path': str(path)} + if row.get('depth') is True: + reference.update(depth=True, early_steps=3) if cast_id(row) is not None: reference['cast'] = cast_id(row) # only a continued part reads it (comfy_media.continued) if 'early_steps' in row: @@ -117,6 +125,16 @@ def resolve_assets(serialized, input_directory): validate_sound(dict(row, kind=kind)) if not row['sound']: reference['sound'] = False + if 'guide' in row: + from .media_request import validate_guide + validate_guide(dict(row, kind=kind)) + if row['guide']: + reference['guide'] = True + if 'keep_source_audio' in row: + from .media_request import validate_keep_source_audio + validate_keep_source_audio(dict(row, kind=kind), path) + if row['keep_source_audio']: + reference['keep_source_audio'] = True result.setdefault('references', []).append(reference) else: raise ValueError('Media role must be first, last or reference') @@ -222,6 +240,8 @@ def output_summary(report, relative_video): video=relative_video.as_posix(), report=relative_video.with_suffix( '.debug.json' if report.get('diagnostic_file') == relative_video.with_suffix('.debug.json').name else '.request.json').as_posix()) + if report.get('keep_source_audio_failed'): + result['keep_source_audio_failed'] = True if type(report.get('output_frames')) is int and report['output_frames'] > 0: # Resource reports keep the generated canvas; result cards show only saved frames. result['geometry'].update(frames=report['output_frames'], diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 96010987..76475544 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1392,6 +1392,19 @@ def poll(): elif export_inputs: send_progress({'label': 'Retaining and checking input media', 'timing_phase': 'encoding'}) extra = export_inputs(run, canvas) or {} + if any(row.get('depth') is True for row in (extra.get('media') or {}).get('references', [])): + from .depth_reference import prepare_media + def release_depth_models(): + if release_models: + release_models() + from .encoder_prewarm import IDLE + from .resident_process import OWNER, ENV + IDLE.stop() + OWNER.release('depth_reference') + environment.pop(ENV, None) + extra = dict(extra, media=prepare_media(extra['media'], root=root, machine=machine, + environ=environment, progress=send_progress, interrupted=interrupted, + release_models=release_depth_models)) if extra: if extra.get('conditioning'): position = command.index('--prompt-file') diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index aa4b5cb2..1b99620d 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -5,6 +5,7 @@ import math import os from pathlib import Path, PurePosixPath +import platform import ssl import time from urllib import error, request @@ -248,6 +249,14 @@ def continuation_supported(): return supported()[0] +def continue_seconds(): + """The length a new continuation starts with where this device sets it: short parts on Apple silicon, where + a full part takes long and often does not fit; None elsewhere, where the page goes by the measured limits.""" + if platform.system() == 'Darwin' and platform.machine().lower() in ('arm64', 'aarch64'): + return chain_plan.SHORT_CONTINUE_SECONDS + return None + + def continuation_available(): return 'after' in continuation_directions() and continuation_supported() @@ -333,7 +342,8 @@ async def plan(request): value = await body(request) result = chain_plan.plan( value.get('seconds'), value.get('single', False), value.get('max_frames'), - value.get('single_limit_seconds'), value.get('block_seconds'), max_parts=runner.max_parts) + value.get('single_limit_seconds'), value.get('block_seconds'), max_parts=runner.max_parts, + first_frame=value.get('first_frame_reference') is True) if not result['single'] and not continuation_supported(): # A device that cannot continue a video has one block to give: never a plan in parts. raise chain_plan.PlanError('device_limit', 'This device cannot generate this length at this size') @@ -376,7 +386,8 @@ def asking(request): @server.routes.get('/freevideo/chains') async def listing(request): - value = {'chains': runner.list(asking(request)), 'continuation': continuation_available()} + value = {'chains': runner.list(asking(request)), 'continuation': continuation_available(), + 'continue_seconds': continue_seconds()} if fake: value['fake'] = True return response(value) diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index fd1b6740..f2006644 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -687,6 +687,8 @@ def __init__(self, source, *, child_environment=None): self.server_address = None self.access_proxy = None self.access_configuration = None + # Shared only by the ComfyUI this controller starts and the access proxy in front of it. + self.hop_secret = secrets.token_urlsafe(32) self.access_links = [] self.access_polled = 0 self._server_lock = threading.RLock() @@ -1242,7 +1244,8 @@ def _start_access_proxy(self): if self._closed or self.cancelled.is_set(): raise RuntimeError('LAN access startup cancelled') self.access_proxy = start_proxy(selected, listen='0.0.0.0', port=port, - upstream=loopback_url(selected['url']), token_file=folder(selected['engine']) / TOKEN, tls=None) + upstream=loopback_url(selected['url']), token_file=folder(selected['engine']) / TOKEN, tls=None, + hop_secret=self.hop_secret if self.owns_server() else None) try: check_proxy(self.access_proxy, scheme='http', port=port, secret=secret) if self._closed or self.cancelled.is_set(): @@ -1372,6 +1375,9 @@ def _connect_comfy(self): if not loopback_host(parsed.hostname): env = bypass_local(env, parsed.hostname) env['FREEVIDEO_COMFY_LOG'] = str(self.server_log) + from .proxy_identity import ENV as HOP + self.hop_secret = env.get(HOP) or self.hop_secret + env[HOP] = self.hop_secret from .launcher_bridge import ENV as BRIDGE env.pop(BRIDGE, None) if self.update_bridge: diff --git a/freevideo_engine/comfy_library.py b/freevideo_engine/comfy_library.py index 3f2ece89..f8469bf7 100644 --- a/freevideo_engine/comfy_library.py +++ b/freevideo_engine/comfy_library.py @@ -19,6 +19,8 @@ from .system import windows _ID = re.compile(r'\d{4}-\d{2}-\d{2}/[0-9a-f]{32}') +# The prompt box takes 20000 characters; a character is at most 4 bytes. +_PROMPT_BYTES = 4 * 20000 def _video(root, identity): @@ -142,6 +144,23 @@ def list_videos(output_directory, *, before=None, limit=24): return dict(items=rows, next=next_cursor, unreadable=sum(last is None or key > last for key in unreadable)) +def prompt_text(output_directory, identity): + """The text a saved video was made from, or None for a run that kept none.""" + path = _video(output_directory, identity) + root = Path(output_directory).resolve() / 'FreeVideo' + saved = path.with_name('prompt.txt') + if saved.is_symlink() or not saved.resolve().is_relative_to(root): + raise ValueError('Saved prompt is outside the library or is a symbolic link') + _report(path) + try: + with os.fdopen(os.open(saved, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_BINARY', 0)), 'rb') as stream: + data = stream.read(_PROMPT_BYTES + 1) + except FileNotFoundError: + return None + # Longer than the prompt box takes: its beginning alone is not the prompt. + return data.decode('utf-8').strip() if len(data) <= _PROMPT_BYTES else None + + def thumbnail(output_directory, identity): path = _video(output_directory, identity) saved = path.with_suffix('.thumbnail.jpg') @@ -1162,6 +1181,14 @@ async def library(request): except ValueError as error: raise web.HTTPBadRequest(text=str(error)) from error + @server.routes.get('/freevideo/library/prompt') + async def prompt(request): + try: + text = await asyncio.to_thread(prompt_text, folder_paths.get_output_directory(), request.query.get('id')) + except (OSError, ValueError, UnicodeDecodeError): + raise web.HTTPNotFound(text='Prompt unavailable') from None + return web.json_response(dict(prompt=text), headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/library/thumbnail') async def preview(request): async with thumbnails: diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index d51268d0..572be5c8 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -236,6 +236,8 @@ def _recall(self, token): return None def snapshot(self, token): + with self.lock: + restarted = token not in self.runs output = self._lookup(token) if output is None: raise KeyError('Unknown report') @@ -244,7 +246,7 @@ def snapshot(self, token): target = output.with_suffix('.live.debug.json') if is_link(output.parent) or target.is_symlink() or target.exists() and is_link(target): raise KeyError('Report unavailable') - path = write(output, live=True) + path = write(output, live=True, restarted=restarted) if path is None: raise RuntimeError('Report not available yet') return path.read_bytes() diff --git a/freevideo_engine/comfy_residency.py b/freevideo_engine/comfy_residency.py index b2ee8158..aa329952 100644 --- a/freevideo_engine/comfy_residency.py +++ b/freevideo_engine/comfy_residency.py @@ -2,10 +2,14 @@ from contextlib import contextmanager from functools import wraps import threading +import time from .resident_process import OWNER _local = threading.local() +# Set while release_native() waits: ComfyUI's prompt thread runs that unload, so the thread-local +# keep_engine_cache() cannot reach it. Our own unload frees native models only, as a video's does. +_releasing = threading.Event() @contextmanager @@ -18,6 +22,60 @@ def keep_engine_cache(): _local.keep = old +def _native(): + """ComfyUI's memory manager in this process, or None outside ComfyUI.""" + import sys + memory = sys.modules.get('comfy.model_management') + return memory if memory is not None and getattr(memory, 'current_loaded_models', None) is not None else None + + +def _prompt_running(server): + try: + return bool(server.prompt_queue.get_current_queue()[0]) + except (AttributeError, TypeError, IndexError): + return True + + +def native_reclaimable(server): + """GPU bytes that ComfyUI's own idle models hold in this process; release_native() returns them. + + 0 when nothing is loaded, or while a prompt is running (its models are in use).""" + memory = _native() + if memory is None or not memory.current_loaded_models or server is None or _prompt_running(server): + return 0 + try: + import torch + return int(torch.cuda.memory_reserved(memory.get_torch_device())) + except Exception: + return 0 + + +def release_native(server, *, timeout=10.): + """Before an image or prompt-model worker starts, unload ComfyUI's own idle models, as a video does. + + ComfyUI's prompt worker does it between prompts, through the same unload command its own + "free memory" button sends. Nothing happens when no native model is loaded, or while a prompt + is running (its models are in use). Returns what was released.""" + memory = _native() + if memory is None or not memory.current_loaded_models or server is None or _prompt_running(server): + return None + import torch + device = memory.get_torch_device() + before = dict(models=len(memory.current_loaded_models), reserved_bytes=int(torch.cuda.memory_reserved(device))) + _releasing.set() + try: + server.prompt_queue.set_flag('unload_models', True) + deadline = time.monotonic() + timeout + while memory.current_loaded_models and time.monotonic() < deadline: + time.sleep(.05) + finally: + _releasing.clear() + if not memory.current_loaded_models: + memory.soft_empty_cache() + return dict(before, released=not memory.current_loaded_models, + reserved_after_bytes=int(torch.cuda.memory_reserved(device))) + + def install(memory): if getattr(memory.free_memory, '_freevideo_resident_cooperation', False) is True: return @@ -28,7 +86,7 @@ def free_memory(memory_required, device, *args, **kwargs): # FreeVideo's own handoff releases native Comfy models only. Other # nodes, and Comfy's explicit unload command, can reclaim our idle # process before choosing a lower-memory native loading strategy. - if (not getattr(_local, 'keep', False) and not OWNER.active and + if (not getattr(_local, 'keep', False) and not _releasing.is_set() and not OWNER.active and OWNER.process is not None and OWNER.process.poll() is None): if device is None or memory_required > memory.get_free_memory(device): OWNER.release('comfy_model') diff --git a/freevideo_engine/comfy_upload.py b/freevideo_engine/comfy_upload.py index cc7b408c..175afa8e 100644 --- a/freevideo_engine/comfy_upload.py +++ b/freevideo_engine/comfy_upload.py @@ -55,7 +55,15 @@ async def receive(reader, input_directory): except BaseException: shutil.rmtree(folder, ignore_errors=True) raise - return {'file': destination.relative_to(base).as_posix(), 'bytes': size} + result = {'file': destination.relative_to(base).as_posix(), 'bytes': size} + if media_kind(name) == 'video': + # Whether the clip has a sound track; the Replace person option is greyed out without one. + try: + from .guide_audio import has_audio + result['audio'] = await asyncio.to_thread(has_audio, destination) + except Exception: + pass + return result def register(): diff --git a/freevideo_engine/component_licenses.py b/freevideo_engine/component_licenses.py index 6bc54f06..6dc6990e 100644 --- a/freevideo_engine/component_licenses.py +++ b/freevideo_engine/component_licenses.py @@ -35,6 +35,13 @@ def row(key, name, licence, link, text=None, terms=None): ('MiniMax H3 Community License Agreement, with the Qwen3-VL Apache License 2.0 notice', 'MiniMax H3 社区许可协议，附 Qwen3-VL 的 Apache 2.0 声明'), 'https://huggingface.co/t8star/Vdn-Minimax-H3-Comfy/blob/main/LICENSE') +DEPTH = row('depth-reference', ('Depth Anything V2 Small code and weights', 'Depth Anything V2 Small 代码和权重'), + ('Apache License 2.0', 'Apache 许可证 2.0'), + 'https://github.com/DepthAnything/Depth-Anything-V2/blob/a561b849ebae10a6f5ef49e26c83cbbcd36c71bf/LICENSE') + + +def depth_license(): + return local(DEPTH, HERE / 'licenses/Depth-Anything-V2-Apache-2.0.txt') # Libraries a runtime may carry, found by file name: versioned Windows names and Linux # lib*.so.* alike. A CUDA build of Torch counts for CUDA, so a runtime whose CUDA files # are named otherwise still lists it. @@ -113,12 +120,17 @@ def bundle(root, value): # The rest of what the Windows runtime carries (Triton's tools, TinyCC) and where its source is. rows.append(row('notices', ('Third-party notices', '第三方组件说明'), ('Copyrights and licenses of the other components', '其余组件的版权和许可'), TEXTS / NOTICES)) + if any('depth_anything_v2/' in entry['path'].replace('\\', '/') + or entry['path'].endswith('depth_anything_v2_vits.pth') for entry in value['files']): + rows.append(depth_license()) return rows def named(url): """A row for a licence link from an installation plan; one not listed here shows its site.""" from urllib.parse import urlsplit + if url.startswith('https://huggingface.co/depth-anything/Depth-Anything-V2-Small/'): + return depth_license() prefixes = ((MODEL, 'https://huggingface.co/OpenVDN/'), (ENCODER, 'https://huggingface.co/t8star/'), *((value, value['link'].rsplit('/', 1)[0] + '/') for value, _ in RUNTIME)) known = next((value for value, prefix in prefixes if url.startswith(prefix)), None) diff --git a/freevideo_engine/decode.py b/freevideo_engine/decode.py index 7ed5a3e5..50832b08 100644 --- a/freevideo_engine/decode.py +++ b/freevideo_engine/decode.py @@ -26,7 +26,7 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, pre preload=False, pin_weights=False, artifacts_dir=None, phase_callback=None, stream_output=False, stream_weights=False, resident_blocks=0, model_cache=None, linear_compute_cache=False, continuation=None, audio_lead_in_callback=None, output_size=None, - hide_tail_frames=0): + hide_tail_frames=0, ram_budget_bytes=None): from .continuation import validate_hide_tail validate_hide_tail(hide_tail_frames) def phase(name): @@ -70,9 +70,22 @@ def phase(name): count = resident_blocks if offload else layers if stream_weights: from .streamed_weights import SafetensorLayers + room = None + if ram_budget_bytes is not None: + # Converted layers are private memory: keep them inside the request's own + # working-memory budget, so they can neither trip an explicit budget nor + # lift the peak that later plans take as demonstrated. + def room(): + import os + from .ram import ProcessMemory + from .system import memory_sample + used = memory_sample(ProcessMemory().sample(os.getpid())) + return ram_budget_bytes - used if type(used) is int and used >= 0 else None weight_source = SafetensorLayers((Path(base)/'vae').glob('*.safetensors'), [f'decoder.transformer_blocks.{index}.' for index in range(count, layers)], - intermediate_dtype=torch.float32 if linear_compute_cache else None) + intermediate_dtype=torch.float32 if linear_compute_cache else None, + keep_converted=True, converted_room=room, + converted_store=getattr(model_cache, 'converted_store', None)) vae, compute_cache_metrics = load_video_decoder(base, resident_blocks=count, weight_source=weight_source, linear_fp16=linear_compute_cache) vae.eval().requires_grad_(False) diff --git a/freevideo_engine/depth_anything_v2/__init__.py b/freevideo_engine/depth_anything_v2/__init__.py new file mode 100644 index 00000000..4a686853 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/__init__.py @@ -0,0 +1,4 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. diff --git a/freevideo_engine/depth_anything_v2/dinov2.py b/freevideo_engine/depth_anything_v2/dinov2.py new file mode 100644 index 00000000..102e3798 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2.py @@ -0,0 +1,308 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +from functools import partial +import math +import logging +from typing import Sequence, Tuple, Union, Callable + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn.init import trunc_normal_ + +from .dinov2_layers import Mlp, PatchEmbed, Attention, Block + + +logger = logging.getLogger("dinov2") + + +def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module: + if not depth_first and include_root: + fn(module=module, name=name) + for child_name, child_module in module.named_children(): + child_name = ".".join((name, child_name)) if name else child_name + named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True) + if depth_first and include_root: + fn(module=module, name=name) + return module + + +class BlockChunk(nn.ModuleList): + def forward(self, x): + for b in self: + x = b(x) + return x + + +class DinoVisionTransformer(nn.Module): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + ffn_bias=True, + proj_bias=True, + drop_path_rate=0.0, + drop_path_uniform=False, + init_values=None, # for layerscale: None or 0 => no layerscale + embed_layer=PatchEmbed, + act_layer=nn.GELU, + block_fn=Block, + ffn_layer="mlp", + block_chunks=1, + num_register_tokens=0, + interpolate_antialias=False, + interpolate_offset=0.1, + ): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + proj_bias (bool): enable bias for proj in attn if True + ffn_bias (bool): enable bias for ffn if True + drop_path_rate (float): stochastic depth rate + drop_path_uniform (bool): apply uniform drop rate across blocks + weight_init (str): weight init scheme + init_values (float): layer-scale init values + embed_layer (nn.Module): patch embedding layer + act_layer (nn.Module): MLP activation layer + block_fn (nn.Module): transformer block class + ffn_layer (str): "mlp" (Small only) + block_chunks: (int) split block sequence into block_chunks units for FSDP wrap + num_register_tokens: (int) number of extra cls tokens (so-called "registers") + interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings + interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings + """ + super().__init__() + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 1 + self.n_blocks = depth + self.num_heads = num_heads + self.patch_size = patch_size + self.num_register_tokens = num_register_tokens + self.interpolate_antialias = interpolate_antialias + self.interpolate_offset = interpolate_offset + + self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) + assert num_register_tokens >= 0 + self.register_tokens = ( + nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None + ) + + if drop_path_uniform is True: + dpr = [drop_path_rate] * depth + else: + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + if ffn_layer == "mlp": + logger.info("using MLP layer as FFN") + ffn_layer = Mlp + else: + raise NotImplementedError + + blocks_list = [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + ffn_layer=ffn_layer, + init_values=init_values, + ) + for i in range(depth) + ] + if block_chunks > 0: + self.chunked_blocks = True + chunked_blocks = [] + chunksize = depth // block_chunks + for i in range(0, depth, chunksize): + # this is to keep the block index consistent if we chunk the block list + chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize]) + self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks]) + else: + self.chunked_blocks = False + self.blocks = nn.ModuleList(blocks_list) + + self.norm = norm_layer(embed_dim) + self.head = nn.Identity() + + self.mask_token = nn.Parameter(torch.zeros(1, embed_dim)) + + self.init_weights() + + def init_weights(self): + trunc_normal_(self.pos_embed, std=0.02) + nn.init.normal_(self.cls_token, std=1e-6) + if self.register_tokens is not None: + nn.init.normal_(self.register_tokens, std=1e-6) + named_apply(init_weights_vit_timm, self) + + def interpolate_pos_encoding(self, x, w, h): + previous_dtype = x.dtype + npatch = x.shape[1] - 1 + N = self.pos_embed.shape[1] - 1 + if npatch == N and w == h: + return self.pos_embed + pos_embed = self.pos_embed.float() + class_pos_embed = pos_embed[:, 0] + patch_pos_embed = pos_embed[:, 1:] + dim = x.shape[-1] + w0 = w // self.patch_size + h0 = h // self.patch_size + # we add a small number to avoid floating point error in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0 + w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset + # w0, h0 = w0 + 0.1, h0 + 0.1 + + sqrt_N = math.sqrt(N) + sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2), + scale_factor=(sx, sy), + # (int(w0), int(h0)), # to solve the upsampling shape issue + mode="bicubic", + antialias=self.interpolate_antialias + ) + + assert int(w0) == patch_pos_embed.shape[-2] + assert int(h0) == patch_pos_embed.shape[-1] + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype) + + def prepare_tokens_with_masks(self, x, masks=None): + B, nc, w, h = x.shape + x = self.patch_embed(x) + if masks is not None: + x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x) + + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.interpolate_pos_encoding(x, w, h) + + if self.register_tokens is not None: + x = torch.cat( + ( + x[:, :1], + self.register_tokens.expand(x.shape[0], -1, -1), + x[:, 1:], + ), + dim=1, + ) + + return x + + def _get_intermediate_layers_not_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + # If n is an int, take the n last blocks. If it's a list, take them + output, total_block_len = [], len(self.blocks) + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in blocks_to_take: + output.append(x) + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def _get_intermediate_layers_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + output, i, total_block_len = [], 0, len(self.blocks[-1]) + # If n is an int, take the n last blocks. If it's a list, take them + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for block_chunk in self.blocks: + for blk in block_chunk[i:]: # Passing the nn.Identity() + x = blk(x) + if i in blocks_to_take: + output.append(x) + i += 1 + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, # Layers or n last layers to take + reshape: bool = False, + return_class_token: bool = False, + norm=True + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + if self.chunked_blocks: + outputs = self._get_intermediate_layers_chunked(x, n) + else: + outputs = self._get_intermediate_layers_not_chunked(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0] for out in outputs] + outputs = [out[:, 1 + self.num_register_tokens:] for out in outputs] + if reshape: + B, _, w, h = x.shape + outputs = [ + out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + +def init_weights_vit_timm(module: nn.Module, name: str = ""): + """ViT weight initialization, original timm impl (for reproducibility)""" + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=0.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + + +def vit_small(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=384, + depth=12, + num_heads=6, + mlp_ratio=4, + block_fn=partial(Block, attn_class=Attention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def DINOv2(model_name): + if model_name != "vits": + raise ValueError("Only DINOv2 Small is bundled") + return vit_small(img_size=518, patch_size=14, init_values=1.0, ffn_layer="mlp", + block_chunks=0, num_register_tokens=0, + interpolate_antialias=False, interpolate_offset=0.1) diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/__init__.py b/freevideo_engine/depth_anything_v2/dinov2_layers/__init__.py new file mode 100644 index 00000000..d78af092 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/__init__.py @@ -0,0 +1,9 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +from .mlp import Mlp +from .patch_embed import PatchEmbed +from .block import Block +from .attention import Attention diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/attention.py b/freevideo_engine/depth_anything_v2/dinov2_layers/attention.py new file mode 100644 index 00000000..b9c68c3a --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/attention.py @@ -0,0 +1,58 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +import logging + +from torch import Tensor +from torch import nn + + +logger = logging.getLogger("dinov2") + + +class Attention(nn.Module): + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + proj_bias: bool = True, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + ) -> None: + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x: Tensor) -> Tensor: + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + + q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] + attn = q @ k.transpose(-2, -1) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/block.py b/freevideo_engine/depth_anything_v2/dinov2_layers/block.py new file mode 100644 index 00000000..7f268c5e --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/block.py @@ -0,0 +1,126 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +import logging +from typing import Callable, List, Any, Tuple, Dict + +import torch +from torch import nn, Tensor + +from .attention import Attention +from .drop_path import DropPath +from .layer_scale import LayerScale +from .mlp import Mlp + + +logger = logging.getLogger("dinov2") + + +class Block(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = False, + proj_bias: bool = True, + ffn_bias: bool = True, + drop: float = 0.0, + attn_drop: float = 0.0, + init_values=None, + drop_path: float = 0.0, + act_layer: Callable[..., nn.Module] = nn.GELU, + norm_layer: Callable[..., nn.Module] = nn.LayerNorm, + attn_class: Callable[..., nn.Module] = Attention, + ffn_layer: Callable[..., nn.Module] = Mlp, + ) -> None: + super().__init__() + # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}") + self.norm1 = norm_layer(dim) + self.attn = attn_class( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + attn_drop=attn_drop, + proj_drop=drop, + ) + self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = ffn_layer( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + bias=ffn_bias, + ) + self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.sample_drop_ratio = drop_path + + def forward(self, x: Tensor) -> Tensor: + def attn_residual_func(x: Tensor) -> Tensor: + return self.ls1(self.attn(self.norm1(x))) + + def ffn_residual_func(x: Tensor) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + if self.training and self.sample_drop_ratio > 0.1: + # the overhead is compensated only for a drop path rate larger than 0.1 + x = drop_add_residual_stochastic_depth( + x, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + x = drop_add_residual_stochastic_depth( + x, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + elif self.training and self.sample_drop_ratio > 0.0: + x = x + self.drop_path1(attn_residual_func(x)) + x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2 + else: + x = x + attn_residual_func(x) + x = x + ffn_residual_func(x) + return x + + +def drop_add_residual_stochastic_depth( + x: Tensor, + residual_func: Callable[[Tensor], Tensor], + sample_drop_ratio: float = 0.0, +) -> Tensor: + # 1) extract subset using permutation + b, n, d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + x_subset = x[brange] + + # 2) apply residual_func to get residual + residual = residual_func(x_subset) + + x_flat = x.flatten(1) + residual = residual.flatten(1) + + residual_scale_factor = b / sample_subset_size + + # 3) add the residual + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + return x_plus_residual.view_as(x) diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/drop_path.py b/freevideo_engine/depth_anything_v2/dinov2_layers/drop_path.py new file mode 100644 index 00000000..a26bc04b --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/drop_path.py @@ -0,0 +1,40 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py + + +from torch import nn + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0: + random_tensor.div_(keep_prob) + output = x * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/layer_scale.py b/freevideo_engine/depth_anything_v2/dinov2_layers/layer_scale.py new file mode 100644 index 00000000..4b69a9ba --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/layer_scale.py @@ -0,0 +1,33 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 + +from typing import Union + +import torch +from torch import Tensor +from torch import nn + + +class LayerScale(nn.Module): + def __init__( + self, + dim: int, + init_values: Union[float, Tensor] = 1e-5, + inplace: bool = False, + ) -> None: + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x: Tensor) -> Tensor: + return x.mul_(self.gamma) if self.inplace else x * self.gamma diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/mlp.py b/freevideo_engine/depth_anything_v2/dinov2_layers/mlp.py new file mode 100644 index 00000000..66cc9d7b --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/mlp.py @@ -0,0 +1,46 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py + + +from typing import Callable, Optional + +from torch import Tensor, nn + + +class Mlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = nn.GELU, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x: Tensor) -> Tensor: + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/freevideo_engine/depth_anything_v2/dinov2_layers/patch_embed.py b/freevideo_engine/depth_anything_v2/dinov2_layers/patch_embed.py new file mode 100644 index 00000000..642712c3 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dinov2_layers/patch_embed.py @@ -0,0 +1,94 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +from typing import Callable, Optional, Tuple, Union + +from torch import Tensor +import torch.nn as nn + + +def make_2tuple(x): + if isinstance(x, tuple): + assert len(x) == 2 + return x + + assert isinstance(x, int) + return (x, x) + + +class PatchEmbed(nn.Module): + """ + 2D image to patch embedding: (B,C,H,W) -> (B,N,D) + + Args: + img_size: Image size. + patch_size: Patch token size. + in_chans: Number of input image channels. + embed_dim: Number of linear projection output channels. + norm_layer: Normalization layer. + """ + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + embed_dim: int = 768, + norm_layer: Optional[Callable] = None, + flatten_embedding: bool = True, + ) -> None: + super().__init__() + + image_HW = make_2tuple(img_size) + patch_HW = make_2tuple(patch_size) + patch_grid_size = ( + image_HW[0] // patch_HW[0], + image_HW[1] // patch_HW[1], + ) + + self.img_size = image_HW + self.patch_size = patch_HW + self.patches_resolution = patch_grid_size + self.num_patches = patch_grid_size[0] * patch_grid_size[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.flatten_embedding = flatten_embedding + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x: Tensor) -> Tensor: + _, _, H, W = x.shape + patch_H, patch_W = self.patch_size + + assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" + assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" + + x = self.proj(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) # B HW C + x = self.norm(x) + if not self.flatten_embedding: + x = x.reshape(-1, H, W, self.embed_dim) # B H W C + return x + + def flops(self) -> float: + Ho, Wo = self.patches_resolution + flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) + if self.norm is not None: + flops += Ho * Wo * self.embed_dim + return flops diff --git a/freevideo_engine/depth_anything_v2/dpt.py b/freevideo_engine/depth_anything_v2/dpt.py new file mode 100644 index 00000000..89088e64 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/dpt.py @@ -0,0 +1,209 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, torch resize, no torchvision. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .dinov2 import DINOv2 +from .util.blocks import FeatureFusionBlock, _make_scratch +from .util.transform import Resize, NormalizeImage, PrepareForNet + + +def _make_fusion_block(features, use_bn, size=None): + return FeatureFusionBlock( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + size=size, + ) + + +class DPTHead(nn.Module): + def __init__( + self, + in_channels, + features=64, + use_bn=False, + out_channels=[48, 96, 192, 384], + use_clstoken=False + ): + super(DPTHead, self).__init__() + + self.use_clstoken = use_clstoken + + self.projects = nn.ModuleList([ + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channel, + kernel_size=1, + stride=1, + padding=0, + ) for out_channel in out_channels + ]) + + self.resize_layers = nn.ModuleList([ + nn.ConvTranspose2d( + in_channels=out_channels[0], + out_channels=out_channels[0], + kernel_size=4, + stride=4, + padding=0), + nn.ConvTranspose2d( + in_channels=out_channels[1], + out_channels=out_channels[1], + kernel_size=2, + stride=2, + padding=0), + nn.Identity(), + nn.Conv2d( + in_channels=out_channels[3], + out_channels=out_channels[3], + kernel_size=3, + stride=2, + padding=1) + ]) + + if use_clstoken: + self.readout_projects = nn.ModuleList() + for _ in range(len(self.projects)): + self.readout_projects.append( + nn.Sequential( + nn.Linear(2 * in_channels, in_channels), + nn.GELU())) + + self.scratch = _make_scratch( + out_channels, + features, + groups=1, + expand=False, + ) + + self.scratch.stem_transpose = None + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + head_features_1 = features + head_features_2 = 32 + + self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1) + self.scratch.output_conv2 = nn.Sequential( + nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True), + nn.Identity(), + ) + + def forward(self, out_features, patch_h, patch_w): + out = [] + for i, x in enumerate(out_features): + if self.use_clstoken: + x, cls_token = x[0], x[1] + readout = cls_token.unsqueeze(1).expand_as(x) + x = self.readout_projects[i](torch.cat((x, readout), -1)) + else: + x = x[0] + + x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)) + + x = self.projects[i](x) + x = self.resize_layers[i](x) + + out.append(x) + + layer_1, layer_2, layer_3, layer_4 = out + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv1(path_1) + out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) + out = self.scratch.output_conv2(out) + + return out + + +class DepthAnythingV2(nn.Module): + def __init__( + self, + encoder='vits', + features=64, + out_channels=[48, 96, 192, 384], + use_bn=False, + use_clstoken=False + ): + super(DepthAnythingV2, self).__init__() + + if encoder != 'vits': + raise ValueError('Only Depth Anything V2 Small is bundled') + self.intermediate_layer_idx = {'vits': [2, 5, 8, 11]} + + self.encoder = encoder + self.pretrained = DINOv2(model_name=encoder) + + self.depth_head = DPTHead(self.pretrained.embed_dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken) + + def forward(self, x): + patch_h, patch_w = x.shape[-2] // 14, x.shape[-1] // 14 + + features = self.pretrained.get_intermediate_layers(x, self.intermediate_layer_idx[self.encoder], return_class_token=True) + + depth = self.depth_head(features, patch_h, patch_w) + depth = F.relu(depth) + + return depth.squeeze(1) + + @torch.no_grad() + def infer_image(self, raw_image, input_size=518): + image, (h, w) = self.image2tensor(raw_image, input_size) + + depth = self.forward(image) + + depth = F.interpolate(depth[:, None], (h, w), mode="bilinear", align_corners=True)[0, 0] + + return depth.cpu().numpy() + + def image2tensor(self, raw_image, input_size=518): + transforms = [ + Resize( + width=input_size, + height=input_size, + resize_target=False, + keep_aspect_ratio=True, + ensure_multiple_of=14, + resize_method='lower_bound', + image_interpolation_method='bicubic', + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + ] + + h, w = raw_image.shape[:2] + + image = raw_image[..., ::-1] / 255.0 + + sample = {'image': image} + for transform in transforms: + sample = transform(sample) + image = sample['image'] + image = torch.from_numpy(image).unsqueeze(0) + + image = image.to(next(self.parameters()).device) + + return image, (h, w) diff --git a/freevideo_engine/depth_anything_v2/util/__init__.py b/freevideo_engine/depth_anything_v2/util/__init__.py new file mode 100644 index 00000000..4a686853 --- /dev/null +++ b/freevideo_engine/depth_anything_v2/util/__init__.py @@ -0,0 +1,4 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. diff --git a/freevideo_engine/depth_anything_v2/util/blocks.py b/freevideo_engine/depth_anything_v2/util/blocks.py new file mode 100644 index 00000000..cff1779c --- /dev/null +++ b/freevideo_engine/depth_anything_v2/util/blocks.py @@ -0,0 +1,153 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, no torchvision. + +import torch.nn as nn + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + if len(in_shape) >= 4: + out_shape4 = out_shape + + if expand: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + if len(in_shape) >= 4: + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d(in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups) + scratch.layer2_rn = nn.Conv2d(in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups) + scratch.layer3_rn = nn.Conv2d(in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups) + if len(in_shape) >= 4: + scratch.layer4_rn = nn.Conv2d(in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups) + + return scratch + + +class ResidualConvUnit(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups=1 + + self.conv1 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) + + self.conv2 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) + + if self.bn == True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn == True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn == True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block. + """ + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + size=None + ): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups=1 + + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1) + + self.resConfUnit1 = ResidualConvUnit(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + self.size=size + + def forward(self, *xs, size=None): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + + output = self.resConfUnit2(output) + + if (size is None) and (self.size is None): + modifier = {"scale_factor": 2} + elif size is None: + modifier = {"size": self.size} + else: + modifier = {"size": size} + + output = nn.functional.interpolate(output, **modifier, mode="bilinear", align_corners=self.align_corners) + + output = self.out_conv(output) + + return output diff --git a/freevideo_engine/depth_anything_v2/util/transform.py b/freevideo_engine/depth_anything_v2/util/transform.py new file mode 100644 index 00000000..e90bce2d --- /dev/null +++ b/freevideo_engine/depth_anything_v2/util/transform.py @@ -0,0 +1,173 @@ +# Depth Anything V2, Copyright (c) 2024 Depth Anything authors. +# Adapted from a561b849ebae10a6f5ef49e26c83cbbcd36c71bf for Small inference. +# SPDX-License-Identifier: Apache-2.0; see freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt. +# Changes: Small only, ordinary torch attention, explicit device, torch resize, no torchvision. + +import numpy as np +import torch +import torch.nn.functional as F + + +def _resize(image, width, height, mode): + channels = image.ndim == 3 + value = np.ascontiguousarray(image.transpose(2, 0, 1) if channels else image[None]) + tensor = torch.from_numpy(value).unsqueeze(0) + options = dict(align_corners=False) if mode == 'bicubic' else {} + resized = F.interpolate(tensor, (height, width), mode=mode, **options)[0].numpy() + return resized.transpose(1, 2, 0) if channels else resized[0] + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method='bicubic', + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of(scale_height * height, min_val=self.__height) + new_width = self.constrain_to_multiple_of(scale_width * width, min_val=self.__width) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of(scale_height * height, max_val=self.__height) + new_width = self.constrain_to_multiple_of(scale_width * width, max_val=self.__width) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size(sample["image"].shape[1], sample["image"].shape[0]) + + # resize sample + sample["image"] = _resize(sample["image"], width, height, self.__image_interpolation_method) + + if self.__resize_target: + if "depth" in sample: + sample["depth"] = _resize(sample["depth"], width, height, 'nearest') + + if "mask" in sample: + sample["mask"] = _resize(sample["mask"].astype(np.float32), width, height, 'nearest') + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + return sample diff --git a/freevideo_engine/depth_reference.py b/freevideo_engine/depth_reference.py new file mode 100644 index 00000000..3d4e92c9 --- /dev/null +++ b/freevideo_engine/depth_reference.py @@ -0,0 +1,487 @@ +"""Optional, file-backed depth references. Torch lives only in a disposable worker. + +Small's vendored Apache-2.0 network and preprocessing follow Quinn's fixed +DAV2 experiment. Cache identity deliberately excludes prompt, seed and K. +""" +from contextlib import contextmanager +from fractions import Fraction +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +from . import processes +from .paths import data_root + +PREPROCESSING_VERSION = 'dav2-small-pyav-torch-bgr-imagenet-lower-bound-14-clay-relief-v1' +CODE_VERSION = 'freevideo-depth-reference-v1' +STAGE = 'depth_reference' + + +def catalog(): + return json.loads(Path(__file__).with_name('depth_reference_model.json').read_text(encoding='utf-8')) + + +def canonical_json(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False) + + +def digest(path, check=lambda: None): + value = hashlib.sha256() + with Path(path).open('rb') as stream: + while block := stream.read(1024 * 1024): + check() + value.update(block) + return value.hexdigest() + + +def cache_directory(root=None): + # download_cache is OS page-cache advice, not an application cache folder. + return Path(root or data_root()) / 'input-cache' / 'depth-cache' + + +@contextmanager +def _lock(path, check, *, inherit=False): + from .locking import runtime_lock + while True: + check() + try: + lease = runtime_lock(path, inherit=inherit) + descriptor = lease.__enter__() + break + except BlockingIOError: + time.sleep(.1) + try: + yield descriptor + finally: + lease.__exit__(None, None, None) + + +def ensure_weights(root, emit, check): + """The same optional-model catalog/transfer mechanism as FreeToken and ASR.""" + from . import network + from .adaln_assets import _download_plan + from .prompt_vlm.assets import remote + spec = catalog() + row = spec['files'][0] + folder = Path(root) / 'models' / 'depth-anything-v2-small' + path = folder / row['file'] + with _lock(folder / 'download.lock', check): + if path.is_file() and path.stat().st_size == row['bytes'] and digest(path, check) == row['sha256']: + return path + if path.exists(): + network.retain_partial(path, 'rejected') + plan = _download_plan(Path(root)) + sources = plan['sources'] = dict(plan.get('sources', {})) + if not sources.get('image-models'): + sources['image-models'] = [dict(row) for row in sources.get('models', []) + if row['id'] in ('official', 'hf-mirror')] + sources['image-models'].append({'id': 'modelscope'}) + plan.update(quiet=True, resource_check=check) + # remote() describes HF pins and a hash-locked ModelScope fallback; + # network retains its measured routes and the user's connection mode. + def moved(current, total, speed, **_): + emit(phase='dependencies', stage='depth_reference_download', label='Downloading depth reference model', + done=current, total=row['bytes'], unit='bytes', bytes_per_second=speed) + check() + try: + network.download(network.model_urls(plan, remote(spec, row)), path, row['sha256'], moved, + network=plan, size=row['bytes'], category='image-models', keep_partial=True, + stall_seconds=30, slow_seconds=15, low_speed_limit=64 * 1024) + except Exception as error: + from .sampling_assets import _cancel_or_disk + if _cancel_or_disk(error): + raise + check() + raise RuntimeError('Depth reference weights are unavailable. Retry with a connection, or put the ' + 'official depth_anything_v2_vits.pth (99,218,434 bytes, SHA-256 ' + + row['sha256'] + ') in ' + str(folder)) from error + if path.stat().st_size != row['bytes'] or digest(path, check) != row['sha256']: + raise ValueError('Depth reference weight size/SHA-256 verification failed: ' + str(path)) + return path + + +def _rotation(frame): + # PyAV exposes the display matrix as counterclockwise degrees, as in agent.media. + try: + return round(frame.rotation or 0) % 360 + except (AttributeError, ValueError): + return 0 + + +def _upright_bgr(frame): + import numpy as np + return np.ascontiguousarray(np.rot90(frame.to_ndarray(format='bgr24'), round(_rotation(frame) / 90) % 4)) + + +def inspect_video(path, check=lambda: None, *, depth=False): + """Decode every frame, rather than trusting container frame-count headers.""" + import av + pts, dimensions, rotation = [], None, None + with av.open(str(path)) as container: + if not container.streams.video: + raise ValueError('Depth reference needs a decodable video') + if depth and container.streams.audio: + raise ValueError('Depth reference cache unexpectedly contains audio') + stream = container.streams.video[0] + rate = stream.average_rate + if rate is None or rate <= 0: + raise ValueError('Depth reference video has no valid frame rate') + time_base = stream.time_base + for frame in container.decode(stream): + check() + current_rotation = _rotation(frame) + if rotation is not None and current_rotation != rotation: + raise ValueError('Depth reference video changes display rotation') + rotation = current_rotation + size = (frame.width, frame.height) + if dimensions is not None and dimensions != size: + raise ValueError('Depth reference video changes dimensions') + dimensions = size + pts.append(frame.pts) + if not pts: + raise ValueError('Depth reference video has no decodable frames') + if depth and rotation: + raise ValueError('Depth reference cache unexpectedly contains display rotation') + if round(rotation / 90) % 2: + dimensions = dimensions[::-1] + return dict(width=dimensions[0], height=dimensions[1], fps_num=rate.numerator, fps_den=rate.denominator, + frames=len(pts), pts=pts, time_base=str(time_base), rotation=rotation) + + +def identity(source, video, device, check=lambda: None): + spec = catalog() + output = {key: video[key] for key in ('width', 'height', 'fps_num', 'fps_den', 'frames')} + output.update(fit='source', audio=False) + pix_fmt = 'yuv420p' if video['width'] % 2 == video['height'] % 2 == 0 else 'yuv444p' + return dict(source_sha256=digest(source, check), source_size=Path(source).stat().st_size, + model_id=spec['repo'], weights_sha256=spec['files'][0]['sha256'], source_commit=spec['source_commit'], + code_version=CODE_VERSION, preprocessing_version=PREPROCESSING_VERSION, + selection_spec=dict(source_start_pts=video['pts'][0], source_end_pts=video['pts'][-1], + source_time_base=video['time_base'], crop_rect=None, + rotation=video['rotation'], tail_policy='none'), + output_spec=output, + inference_spec=dict(input_size=518, precision='fp16' if device == 'cuda' else 'fp32', + normalization='whole_clip_p1_p99_from_240x136_near_white_v1', + render='clay-relief-v1', + encoder='rgb-to-'+pix_fmt+'/libx264/crf16/threads2', runtime_backend='torch-'+device)) + + +def cache_key(config): + return hashlib.sha256(canonical_json(config).encode('utf-8')).hexdigest() + + +def _validate(path, expected, check): + observed = inspect_video(path, check, depth=True) + if any(observed[key] != expected[key] for key in ('width', 'height', 'fps_num', 'fps_den', 'frames')): + raise ValueError('Depth reference output dimensions, frame rate or frame count differ from the input') + return observed + + +def _cached(folder, config, check): + try: + manifest = json.loads((folder / 'manifest.json').read_text(encoding='utf-8')) + output = folder / 'depth.mp4' + if (manifest['config'] != config or manifest['output_sha256'] != digest(output, check) + or manifest['output_bytes'] != output.stat().st_size): + return None + _validate(output, config['output_spec'], check) + return output + except (OSError, ValueError, KeyError, TypeError) as error: + if isinstance(error, InterruptedError): + raise + return None + + +def _load_model(weights, device): + import torch + from .depth_anything_v2.dpt import DepthAnythingV2 + model = DepthAnythingV2() + model.load_state_dict(torch.load(weights, map_location='cpu', weights_only=True), strict=True) + return model.to(device).eval() + + +def _clay_relief(gray): + """Render quantized, near-bright depth with Quinn's fixed shade.py recipe. + + SciPy's default Gaussian uses sigma=3, radius=12 and half-sample symmetric + reflection (including the edge pixel). Torch's reflect pad excludes that + pixel, so use reflected indices; this also works on frames below 13 pixels. + """ + import numpy as np + import torch + from torch.nn.functional import conv2d + + d = torch.from_numpy(gray.astype(np.float32) / 255)[None, None] + offsets = torch.arange(-12, 13, dtype=torch.float64) + kernel = torch.exp(-0.5 * offsets.square() / 3.0**2) + kernel = (kernel / kernel.sum()).float() + # Match gaussian_filter's axis order and float32 intermediate images. + for axis in (2, 3): + size = d.shape[axis] + indices = torch.arange(-12, size + 12) % (2 * size) + indices = torch.where(indices < size, indices, 2 * size - 1 - indices) + weight = kernel.reshape(1, 1, 25, 1) if axis == 2 else kernel.reshape(1, 1, 1, 25) + d = conv2d(d.index_select(axis, indices), weight) + d = d[0, 0].numpy() + height, width = d.shape + # numpy.gradient uses central differences inside and one-sided edges. + gy = np.gradient(d, axis=0) if height > 1 else np.zeros_like(d) + gx = np.gradient(d, axis=1) if width > 1 else np.zeros_like(d) + k = 5.0 + nx, ny, nz = -gx*k*width/64, -gy*k*height/64, np.ones_like(d) + n = np.sqrt(nx**2 + ny**2 + nz**2) + nx, ny, nz = nx/n, ny/n, nz/n + light = np.array([-0.5, -0.6, 0.62]) + light /= np.linalg.norm(light) + lam = np.clip(nx*light[0] + ny*light[1] + nz*light[2], 0, 1) + shade = (0.25 + 0.75*lam) * (0.75 + 0.25*d) + base = np.array([0.82, 0.80, 0.77]) + return (np.clip(shade, 0, 1)[..., None] * base * 255).astype(np.uint8) + + +def _infer(source, stage, video, weights, device, emit, check): + import av + import numpy as np + import psutil + import torch + # Reserve system RAM and account for live free memory before loading. + if psutil.virtual_memory().available < 2 * 2**30: + raise MemoryError('Depth reference needs 1 GiB working RAM plus 1 GiB system reserve; free memory and retry') + if device == 'cuda': + if not torch.cuda.is_available(): + raise RuntimeError('CUDA is unavailable for depth reference; select the CPU backend to use FP32') + free, total = torch.cuda.mem_get_info() + reserve = max(512 * 2**20, total // 10) + if free < reserve + 512 * 2**20: + raise MemoryError('Not enough free GPU memory for depth reference and the GPU reserve') + torch.set_num_threads(min(8, os.cpu_count() or 1)) + shape = (video['frames'], video['height'], video['width']) + from .disk_space import free_bytes + if free_bytes(stage) < (shape[0] * (shape[1] * shape[2] + 240 * 136) * 4 + 256 * 2**20): + raise OSError('Not enough disk space to stage depth frames; free space and retry') + model = _load_model(weights, device) + raw_path, small_path = stage / 'raw.f32', stage / 'small.f32' + depths = np.memmap(raw_path, mode='w+', dtype=np.float32, shape=shape) + small = np.memmap(small_path, mode='w+', dtype=np.float32, shape=(video['frames'], 136, 240)) + try: + with av.open(str(source)) as container: + count = 0 + for frame in container.decode(video=0): + check() + bgr = _upright_bgr(frame) + if (count >= video['frames'] or bgr.shape[:2] != shape[1:] + or _rotation(frame) != video['rotation']): + raise ValueError('Source video changed during depth conversion') + if psutil.virtual_memory().available < 2**30: + raise MemoryError('Depth reference stopped to preserve 1 GiB of free system RAM') + with torch.inference_mode(), torch.autocast('cuda', dtype=torch.float16, enabled=device == 'cuda'): + depth = np.asarray(model.infer_image(bgr, 518), dtype=np.float32) + if depth.shape != shape[1:] or not np.isfinite(depth).all(): + raise ValueError('Depth model returned invalid/nonfinite pixels') + depths[count] = depth + small[count] = torch.nn.functional.interpolate(torch.from_numpy(depth)[None, None], + (136, 240), mode='area')[0, 0].numpy() + count += 1 + emit(done=count, total=video['frames'], unit='frames', phase='depth_reference', + detail='%d/%d frames' % (count, video['frames'])) + if count != video['frames']: + raise ValueError('Source video frame count changed during depth conversion') + # Whole-clip p1/p99, in-place selection on disk-backed samples, bounded RAM. + small.flush() + lo, hi = np.percentile(small.reshape(-1), [1, 99], overwrite_input=True) + span = max(float(hi - lo), 1e-6) + output = stage / 'depth.mp4' + emit(phase='depth_reference', detail='Encoding depth video') + with av.open(str(output), 'w') as container: + stream = container.add_stream('libx264', rate=Fraction(video['fps_num'], video['fps_den'])) + stream.width, stream.height = video['width'], video['height'] + stream.pix_fmt = 'yuv420p' if stream.width % 2 == stream.height % 2 == 0 else 'yuv444p' + stream.options = {'crf': '16', 'threads': '2'} + for index, depth in enumerate(depths): + check() + gray = (np.clip((depth - lo) / span, 0, 1) * 255).round().astype(np.uint8) + frame = av.VideoFrame.from_ndarray(_clay_relief(gray), format='rgb24') + frame.pts, frame.time_base = index, Fraction(video['fps_den'], video['fps_num']) + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(None): + container.mux(packet) + return dict(low=float(lo), high=float(hi), near='white', torch=torch.__version__, + av=av.__version__, ffmpeg=av.library_versions, + attention='ordinary-torch-matmul-softmax', device=device) + finally: + # Close mappings before deleting or moving files (Windows). + depths._mmap.close() + small._mmap.close() + del depths, small + raw_path.unlink(missing_ok=True) + small_path.unlink(missing_ok=True) + + +def _convert(request, emit, check): + source, root, work = Path(request['source']), Path(request['root']), Path(request['work']) + device = request.get('device', 'auto') + if device == 'auto': + import torch + device = 'cuda' if torch.cuda.is_available() else 'cpu' + if device not in ('cpu', 'cuda'): + raise ValueError('Depth reference device must be cpu, cuda or auto') + emit(phase='depth_reference', detail='Checking depth reference cache') + initial_hash, initial_size = digest(source, check), source.stat().st_size + video = inspect_video(source, check) + config = identity(source, video, device, check) + if config['source_sha256'] != initial_hash or config['source_size'] != initial_size: + raise ValueError('Source video changed during depth inspection') + cache = Path(request['cache']) + key = cache_key(config) + target = cache / key + with _lock(cache / '.locks' / (key + '.lock'), check): + found = _cached(target, config, check) + if found is not None: + return dict(path=str(found), cache_hit=True, key=key) + weights = ensure_weights(root, emit, check) + # Parent releases idle Comfy/engine/encoder models only on a cache miss. + emit(phase=STAGE, stage=STAGE, label='Extracting depth from the reference video', + done=0, total=video['frames'], unit='frames', detail='Loading depth model', waiting_for_device=True) + while not (work / 'device-ready').is_file(): + check() + time.sleep(.05) + stage = work / 'result' + stage.mkdir() + with _lock(Path(os.environ.get('FREEVIDEO_LOCK_PATH', root / 'engine.lock')), check, inherit=True): + metrics = _infer(source, stage, video, weights, device, emit, check) + observed = _validate(stage / 'depth.mp4', video, check) + if digest(source, check) != config['source_sha256'] or source.stat().st_size != config['source_size']: + raise ValueError('Source video changed during depth conversion') + manifest = dict(config=config, source_pts=video['pts'], source_time_base=video['time_base'], output=observed, + output_sha256=digest(stage / 'depth.mp4', check), output_bytes=(stage / 'depth.mp4').stat().st_size, + normalization=metrics, render='clay-relief-v1', license='Apache-2.0', + license_sources=['https://github.com/DepthAnything/Depth-Anything-V2/tree/'+catalog()['source_commit'], + 'https://huggingface.co/'+catalog()['repo']+'/tree/'+catalog()['revision']]) + (stage / 'manifest.json').write_text(canonical_json(manifest), encoding='utf-8') + check() + if target.exists(): + # Retain invalid old artifacts separately; never accept a partial directory. + target.rename(cache / (key + '.rejected-' + uuid.uuid4().hex)) + os.replace(stage, target) + return dict(path=str(target / 'depth.mp4'), cache_hit=False, key=key) + + +def convert(source, *, root=None, cache=None, device='auto', python=None, environ=None, + progress=None, interrupted=None, release_models=None): + """Convert in a supervised child; return a validated cache path after it exits. + + Progress carries stage=depth_reference and done/total in frames. Cancellation + stops the whole child tree and removes only this request's temporary files. + """ + root = Path(root or data_root()).resolve() + cache = Path(cache or cache_directory(root)).resolve() + cache.mkdir(parents=True, exist_ok=True) + def check(): + if interrupted: + interrupted() + check() + with tempfile.TemporaryDirectory(prefix='.depth-', dir=cache) as directory: + work = Path(directory) + request = dict(source=str(Path(source).resolve()), root=str(root), cache=str(cache), device=device, work=str(work)) + (work / 'request.json').write_text(canonical_json(request), encoding='utf-8') + command = processes.module_command('freevideo_engine.depth_reference', '--request', work / 'request.json') + if python is not None: + command[0] = str(python) + with (work / 'worker.log').open('wb') as log: + child = processes.popen(command, env=environ, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True, supervise=True, pass_fds=processes.descriptors()) + last, released = None, False + def poll(): + nonlocal last, released + check() + try: + value = json.loads((work / 'progress.json').read_text(encoding='utf-8')) + except (OSError, ValueError): + return + if value.get('waiting_for_device') and not released: + if release_models: + release_models() + check() + (work / 'device-ready').touch() + released = True + if progress and value != last: + progress({key: val for key, val in value.items() if key != 'waiting_for_device'}) + last = value + try: + while child.poll() is None: + poll() + try: + child.wait(timeout=.1) + except subprocess.TimeoutExpired: + pass + poll() + if child.returncode: + try: + error = json.loads((work / 'error.json').read_text(encoding='utf-8'))['error'] + except (OSError, ValueError, KeyError): + error = 'Depth reference worker exited with code %s' % child.returncode + raise RuntimeError(error) + result = json.loads((work / 'result.json').read_text(encoding='utf-8')) + check() + return Path(result['path']) + except BaseException: + (work / 'cancelled').touch() + processes.stop(child) + raise + + +def prepare_media(media, *, root, machine, environ, progress, interrupted, release_models): + """Execution only. With depth off, return the original object unchanged.""" + if not any(row.get('depth') is True for row in media.get('references', [])): + return media + result = dict(media, references=[]) + for row in media.get('references', []): + if 'depth' in row and (type(row['depth']) is not bool or row.get('kind') != 'video'): + raise ValueError('depth must be a boolean on a video reference') + if row.get('depth') is not True: + result['references'].append(row) + continue + converted = convert(row['path'], root=root, python=machine['python'], environ=environ, + device='cpu' if machine.get('device_backend') in ('cpu', 'mps') else 'cuda', + progress=progress, interrupted=interrupted, release_models=release_models) + value = {key: val for key, val in row.items() if key != 'depth'} + value['path'] = str(converted) + value.setdefault('early_steps', 3) + result['references'].append(value) + return result + + +def main(): + import argparse + from .monitoring import save + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--request', required=True, type=Path) + args = parser.parse_args() + request = json.loads(args.request.read_text(encoding='utf-8')) + work = Path(request['work']) + def check(): + if (work / 'cancelled').exists(): + raise InterruptedError('Depth reference conversion cancelled') + state = dict(stage=STAGE, phase=STAGE, timing_phase='encoding', label='Extracting depth from the reference video', + done=0, total=None, unit='frames', references=True) + def emit(**values): + state.update(values) + save(work / 'progress.json', state) + try: + with processes.worker_signals(): + result = _convert(request, emit, check) + save(work / 'result.json', result) + except Exception as error: + save(work / 'error.json', dict(error=type(error).__name__ + ': ' + str(error))) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/freevideo_engine/depth_reference_model.json b/freevideo_engine/depth_reference_model.json new file mode 100644 index 00000000..54553f38 --- /dev/null +++ b/freevideo_engine/depth_reference_model.json @@ -0,0 +1,12 @@ +{ + "repo": "depth-anything/Depth-Anything-V2-Small", + "revision": "03876f8651c73a60fe4c2c48294e09fcb6838fcf", + "source_commit": "a561b849ebae10a6f5ef49e26c83cbbcd36c71bf", + "license": "Apache-2.0", + "modelscope": {"repo": "depth-anything/Depth-Anything-V2-Small", "revision": "master"}, + "files": [{ + "file": "depth_anything_v2_vits.pth", + "bytes": 99218434, + "sha256": "715fade13be8f229f8a70cc02066f656f2423a59effd0579197bbf57860e1378" + }] +} diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index a8c0c37e..6f173580 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -8,59 +8,11 @@ import json import os from pathlib import Path -import stat import sys import time from .monitoring import save from .processes import worker_signals - -PACKAGE = Path(__file__).resolve().parent - - -def resolve_encoder(folder_paths, name): - files = json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')) - expected = next((row['bytes'] for row in files if row['file'] == 'text_encoders/' + name), None) - path = None - copies = {} - if expected is not None: - folders = folder_paths.get_folder_paths('text_encoders') - models_dir = getattr(folder_paths, 'models_dir', None) - if models_dir is not None: - models_dir = os.path.normcase(os.path.realpath(models_dir)) - freevideo, bundled = [], [] - # FreeVideo paths stay preferred after a resident worker registers them. - for folder in folders: - resolved = os.path.normcase(os.path.realpath(folder)) - try: - in_models = os.path.commonpath([models_dir, resolved]) == models_dir - except ValueError: - in_models = False # Different Windows drives cannot share a model root. - (bundled if in_models else freevideo).append(folder) - folders = freevideo + bundled - for folder in folders: - candidate = os.path.realpath(os.path.join(folder, name)) - if candidate in copies: - continue - try: - info = Path(candidate).stat() - except OSError: - continue - if not stat.S_ISREG(info.st_mode): - continue - copies[candidate] = info.st_size - if path is None and info.st_size == expected: - path = candidate - if path is None: - path = folder_paths.get_full_path_or_raise('text_encoders', name) - checkpoint = dict(state='read_only_mmap' if sys.platform == 'win32' else 'native_mmap', - bytes=Path(path).stat().st_size) - if expected is not None: - checkpoint.update(expected_bytes=expected, tested_size=checkpoint['bytes'] == expected) - selected = os.path.realpath(path) - mismatched = sum(size != expected for candidate, size in copies.items() if candidate != selected) - if mismatched: - checkpoint['mismatched_copies'] = mismatched - return path, checkpoint +from .encoder_checkpoint import resolve_encoder def encoder_on_gpu(clip): @@ -311,6 +263,8 @@ def phase(name, **metrics): return dict(state='skipped', resource_decision=decision, reason='Foreground work or insufficient live memory for idle preloading') phase('encoder_options') + from .encoder_source import ensure + ensure(args.comfy_root) sys.path.insert(0, str(args.comfy_root.resolve())) import comfy.options comfy.options.enable_args_parsing() diff --git a/freevideo_engine/encoder_checkpoint.py b/freevideo_engine/encoder_checkpoint.py index 0e9bf9ec..90b64040 100644 --- a/freevideo_engine/encoder_checkpoint.py +++ b/freevideo_engine/encoder_checkpoint.py @@ -3,12 +3,82 @@ import math import mmap import os +from pathlib import Path +import stat import struct import sys import warnings import weakref +PACKAGE = Path(__file__).resolve().parent + + +def resolve_encoder(folder_paths, name): + files = json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')) + expected = next((row['bytes'] for row in files if row['file'] == 'text_encoders/' + name), None) + path = None + copies = {} + if expected is not None: + folders = folder_paths.get_folder_paths('text_encoders') + models_dir = getattr(folder_paths, 'models_dir', None) + if models_dir is not None: + models_dir = os.path.normcase(os.path.realpath(models_dir)) + freevideo, bundled = [], [] + # FreeVideo paths stay preferred after a resident worker registers them. + for folder in folders: + resolved = os.path.normcase(os.path.realpath(folder)) + try: + in_models = os.path.commonpath([models_dir, resolved]) == models_dir + except ValueError: + in_models = False # Different Windows drives cannot share a model root. + (bundled if in_models else freevideo).append(folder) + folders = freevideo + bundled + for folder in folders: + candidate = os.path.realpath(os.path.join(folder, name)) + if candidate in copies: + continue + try: + info = Path(candidate).stat() + except OSError: + continue + if not stat.S_ISREG(info.st_mode): + continue + copies[candidate] = info.st_size + if path is None and info.st_size == expected: + path = candidate + if path is None: + path = folder_paths.get_full_path_or_raise('text_encoders', name) + checkpoint = dict(state='read_only_mmap' if sys.platform == 'win32' else 'native_mmap', + bytes=Path(path).stat().st_size) + if expected is not None: + checkpoint.update(expected_bytes=expected, tested_size=checkpoint['bytes'] == expected) + selected = os.path.realpath(path) + mismatched = sum(size != expected for candidate, size in copies.items() if candidate != selected) + if mismatched: + checkpoint['mismatched_copies'] = mismatched + return path, checkpoint + + +def preflight_encoder(root, name, model_paths=None): + """Use the worker's native path configuration without importing its tensor runtime.""" + previous = list(sys.path) + sys.path.insert(0, str(Path(root).resolve())) + try: + import folder_paths + from utils.extra_config import load_extra_path_config + if model_paths: + load_extra_path_config(str(model_paths)) + try: + return resolve_encoder(folder_paths, name) + except FileNotFoundError as error: + raise FileNotFoundError( + "ENCODER_MODEL_MISSING: Model in folder 'text_encoders' with filename '%s' not found. " + 'Please repair the installation. 请修复安装。' % name) from error + finally: + sys.path[:] = previous + + def read_only_state_dict(path, dtypes): """Only for immutable inference weights; tensors retain their mapping. diff --git a/freevideo_engine/encoder_source.py b/freevideo_engine/encoder_source.py new file mode 100644 index 00000000..2a9ea46c --- /dev/null +++ b/freevideo_engine/encoder_source.py @@ -0,0 +1,132 @@ +"""Keep the encoder's pinned ComfyUI source tree whole. + +The tree is a sparse Git checkout of one commit, so a file that disappears after +installation (security software, a cleanup tool) is still in the local object +store: put it back from there instead of downloading anything. +""" +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess + +from . import subprocess_output +from .long_paths import io_path +from .paths import data_root +from .windows_ux import hidden_console + +# Read when the native Qwen tokenizer is built. Without vocab.json or merges.txt +# the tokenizers library fails with an error that names neither file. +REQUIRED = ('comfy/text_encoders/qwen25_tokenizer/vocab.json', + 'comfy/text_encoders/qwen25_tokenizer/merges.txt', + 'comfy/text_encoders/qwen25_tokenizer/tokenizer_config.json') + + +# A Windows command line holds 32767 characters; a deleted tree can name more paths than that. +BATCH = 8000 + + +class EncoderSourceError(RuntimeError): + """Text encoder files are missing and could not be restored.""" + + +def executable(env=None): + """The Git to use: the one the installer exported, one on PATH, else the one it recorded. + + Windows installs keep their own MinGit under the installation folder and + note it only in machine.json, so a machine without a system Git has neither + of the first two. + """ + found = (env or os.environ).get('FREEVIDEO_GIT') or shutil.which('git') + if found: + return found + try: + recorded = json.loads((data_root() / 'machine.json').read_text(encoding='utf-8')).get('git') + if isinstance(recorded, str) and recorded.strip() and Path(recorded).is_file(): + return recorded + except (OSError, ValueError, AttributeError): + pass + return None + + +def managed(root, env=None): + """Only the installer's vendor checkout, with no linked path or Git directory.""" + environ = os.environ if env is None else env + home = Path(environ['FREEVIDEO_HOME']).expanduser() if environ.get('FREEVIDEO_HOME') else data_root() + root = Path(root).expanduser().absolute() + expected = (home / 'vendor' / 'h3-text-encoder').absolute() + if root != expected or '..' in root.parts: + return False + try: + # Do not resolve first: that would hide symlinks, junctions and other + # reparse points, including one in an ancestor of the installation. + for path in (root / '.git', root, *root.parents): + info = io_path(path).lstat() + if (not stat.S_ISDIR(info.st_mode) or + getattr(info, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT): + return False + except (OSError, ValueError): + return False + return True + + +def missing(root): + return [name for name in REQUIRED if not (Path(root) / name).is_file()] + + +def deleted(root, env=None): + """Tracked files that are gone from the work tree; None if Git cannot say.""" + git = executable(env) + if not git or not (Path(root) / '.git').exists(): + return None + try: + result = subprocess_output.run_external([git, '--literal-pathspecs', '-c', 'core.longpaths=true', + '-C', str(root), 'ls-files', '-d', '-z'], + env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, **hidden_console()) + except (OSError, ValueError, subprocess.SubprocessError): + return None + if result.returncode: + return None + return [name for name in (result.stdout or '').split('\0') if name] + + +def restore(root, names, env=None): + """Check the named tracked files out again; True if every command succeeded.""" + git = executable(env) + if not git or not names: + return False + batches, used = [[]], 0 + for name in names: + if batches[-1] and used + len(name) + 1 > BATCH: + batches.append([]) + used = 0 + batches[-1].append(name) + used += len(name) + 1 + for batch in batches: + try: + result = subprocess_output.run_external([git, '--literal-pathspecs', '-c', 'core.longpaths=true', + '-C', str(root), 'checkout', '--', *batch], + env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120, **hidden_console()) + except (OSError, ValueError, subprocess.SubprocessError): + return False + if result.returncode: + return False + return True + + +def ensure(root, env=None): + """Restore missing vendor files; custom ComfyUI checkouts stay read-only.""" + absent = missing(root) + if not absent or not managed(root, env): + return [] + gone = deleted(root, env) + # Another ComfyUI layout, or no Git to ask: leave it to the library. + lost = [name for name in absent if name in (gone or ())] + if not lost: + return [] + restored = restore(root, gone, env) + lost = missing(root) if restored else [name for name in gone if not io_path(Path(root) / name).exists()] + if not restored or lost: + raise EncoderSourceError('FreeVideo text encoder files are missing: %s' % ', '.join(lost[:10])) + return gone diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 6797aeee..fec07b67 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -161,7 +161,7 @@ def display_launcher_error(value, *, zh=False, replacements=()): return text -def setup_command_failure(label, log, exit_code): +def setup_command_failure(label, log, exit_code, report=None): """Keep the complete failed step, with required GPU failures before metadata.""" log = Path(log) try: @@ -171,7 +171,7 @@ def setup_command_failure(label, log, exit_code): lines = ['Failed step: %s (exit code %s)' % (label, exit_code)] if label == 'kernels': try: - report = json.loads((log.parent / 'kernel-capabilities.json').read_text(encoding='utf-8')) + report = json.loads(Path(report or log.parent / 'kernel-capabilities.json').read_text(encoding='utf-8')) except (OSError, ValueError): report = {} if isinstance(report, dict): @@ -375,6 +375,7 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep return dict(title='', detail='', action='', kind='') text = _without_ready_curl_diagnostics(str(value)) # Match only the cause position, never an echoed input or a log line below it. + # Holder diagnostics follow this exact cause; they are not part of its signature. causes = [line.strip() for line in text.splitlines() if line.strip()] index = 1 if causes and re.fullmatch(r'FreeVideo generation failed(?: \(exit -?\d+\))?', causes[0]) else 0 cause = re.sub(r'^(?:[A-Za-z_][\w.]*(?:Error|Exception)):\s*', '', causes[index]) if len(causes) > index else '' @@ -586,7 +587,7 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep ('A video is still generating', '还有视频正在生成'), ('The update is installed. Click Connect again after the job finishes to restart ComfyUI with it.', '更新已装好。等当前任务完成后点“重新连接”，ComfyUI 会重启并使用新版。')), - (r'WinError 206|The filename or extension is too long', 'path-too-long', + (r'WinError 206|The filename or extension is too long|is too long for Git to read', 'path-too-long', ("The installation folder's path is too long", '安装文件夹的路径太长'), ('Install FreeVideo in a folder with a shorter path, such as D:\\FreeVideo, then click "Retry installation".', '请把 FreeVideo 安装到路径较短的文件夹，例如 D:\\FreeVideo，然后点击“重试安装”。')), @@ -677,6 +678,12 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep model, capability = found.groups() detail = ('检测到：%s，计算能力 %s' if zh else 'Detected: %s, compute capability %s') % (model, capability) return dict(title=title[zh], detail=detail + '\n' + action[zh], action='', kind=kind) + if kind == 'path-too-long': + # A dependency folder Git cannot read is named, above the same next step. + found = re.search(r'The path of the dependency folder (.+?) is too long for Git to read', text) + if found: + detail = ('文件夹：%s' if zh else 'Folder: %s') % found[1] + return dict(title=title[zh], detail=detail + '\n' + action[zh], action='', kind=kind) return dict(title=title[zh], detail=detail, action=action[zh], kind=kind) # Validation messages are already actionable. Show the cause, not a stack # trace or a guessed hardware diagnosis, and leave long details expandable. @@ -872,6 +879,11 @@ def _mac_run(report, run, redactor): return 'Run: ' + '; '.join(parts) +def missing_encoder(text): + return re.search(r"ENCODER_MODEL_MISSING:|Model in folder ['\"]text_encoders['\"] " + r"with filename ['\"][^\r\n]+?['\"] not found\.", text, re.I) is not None + + def generation_failure(run, exit_code=None, *, repair=None): """Surface the cause, including a child traceback, even without a report. @@ -886,10 +898,12 @@ def generation_failure(run, exit_code=None, *, repair=None): report = _report(run) redactor.structured(report) # Collect prompt values without showing the request. # Read the whole bounded log, not a tail that could hide other output. - # A legacy managed failure has exactly this one nonempty line and exit 1. - if exit_code == 1 and [line for line in _read(run / 'generate.log', 8192).splitlines() - if line.strip()] == [ENGINE_BUSY]: - lines.append(ENGINE_BUSY) + # Managed failures contain only the exact busy sentence and optional holder + # diagnostics. Other output must never acquire a busy classification. + busy_lines = [line for line in _read(run / 'generate.log', 8192).splitlines() if line.strip()] + if exit_code == 1 and busy_lines[:1] == [ENGINE_BUSY] \ + and all(line.startswith('Lock file open:') for line in busy_lines[1:]): + lines.extend(busy_lines) # What the worker found when cuDNN refused its own files, ahead of the trace. explained = _runtime_explanation(report, REPAIR_MARK) if explained: @@ -961,11 +975,14 @@ def generation_failure(run, exit_code=None, *, repair=None): promoted = bool(causes) and causes[-1] not in known if promoted: lines.insert(1, 'Cause: ' + causes[-1][:300]) + encoder_missing = missing_encoder('\n'.join(lines)) + if encoder_missing: + lines.insert(1, 'ENCODER_MODEL_MISSING: The encoder model file is missing. ' + REPAIR_MARK) if len(lines) == 1: lines.append('The worker ended without a readable error report.') summary = _mac_run(report, run, redactor) if summary: - lines.insert(2 if promoted else 1, summary) + lines.insert(1 + bool(promoted) + bool(encoder_missing), summary) lines.append('\nRetained outputs: ' + str(run)) # The table failure can be in a retained child log rather than the report. # Keep its numeric metadata ahead of technical output for all interactive callers. @@ -977,9 +994,9 @@ def generation_failure(run, exit_code=None, *, repair=None): card = sampling_table_failure(marker) first, _, rest = value.partition('\n') value = '\n'.join((first, marker, card['title'], card['detail'], rest)) - if explained: + if explained or encoder_missing: from .runtime_libraries import REINSTALL - value = value.replace(REPAIR_MARK, repair or REINSTALL, 1) + value = value.replace(REPAIR_MARK, repair or REINSTALL) value = re.sub(r'\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\))', '', value) value = re.sub(r'[\x00-\x08\x0b-\x1f\x7f]', '', value) return value diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 3355497f..d5815e69 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -20,6 +20,7 @@ from .locking import runtime_lock, LOCK_ENV from .paths import vdn_root, comfy_root from .monitoring import save +from .encoder_checkpoint import preflight_encoder from . import tuning from . import processes from .system import (inference_headroom, inference_memory_sample, inference_environment, memory_sample, @@ -771,7 +772,8 @@ def interrupted(signum, frame): 'metrics': str(destination.with_suffix('.encoding.json')), 'diagnostic_output': str(destination)} if media: - encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'), + from .paths import base_path + encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base') or str(base_path()), reference_latent_cache=str(input_cache.root.parent.parent) if input_cache else None) # Lets the encoder child read the video model while it computes, on a slow drive. from .model_read_ahead import request_fields @@ -800,6 +802,13 @@ def interrupted(signum, frame): for trimmed in cached.get('reference_trims') or []: print(json.dumps(dict(event='reference_trimmed', **trimmed)), flush=True) else: + try: + preflight_encoder(args.comfy_root or comfy_root(), args.encoder, encoding['model_paths']) + except Exception as error: + # Only a missing encoder stops the run; the worker stays the judge of anything else. + if str(error).startswith('ENCODER_MODEL_MISSING:'): + raise + print(json.dumps(dict(event='encoder_preflight_skipped', error=repr(error)[:300])), flush=True) encoder_env = dict(env, PYTHONPATH=str(repo)) print(json.dumps({'event': 'encoding_start'}), flush=True) encoder_attempt = history.begin(resource_identity, {'encoder': args.encoder}, @@ -1156,6 +1165,9 @@ def retry(index, row, selected, decision): shutil.copyfile(destination.with_suffix('.engine.json'), artifacts / 'first-pass.engine.json') from .preview import upscale_reads report['preview']['upscale_reads'] = upscale_reads(request['cache'], report['video'], upscaler_checkpoint) + if not preview and any(row.get('keep_source_audio') for row in media.get('references', [])): + from .guide_audio import apply_to_report as keep_source_audio + keep_source_audio(destination, media, report) report['success'] = True except BaseException as error: from .diagnostic_resources import exception_details diff --git a/freevideo_engine/gpu_budget.py b/freevideo_engine/gpu_budget.py index 83a7420b..e04733e0 100644 --- a/freevideo_engine/gpu_budget.py +++ b/freevideo_engine/gpu_budget.py @@ -305,13 +305,22 @@ def _settle(self, workspace): return min(seen), samples return None, samples - def refresh(self, stage, reclaim=None, workspace=None): + def refresh(self, stage, reclaim=None, workspace=None, confirm=False): cuda = self.torch.cuda free, reserved, local, decision, known, candidate = self._observe() target, action = self.limit, 'hold' if candidate < self.limit: target, action, self.pending = candidate, 'shrink', None elif candidate >= self.limit + 64 * 2**20: + if self.pending is None and confirm: + # A pass boundary has no earlier step to have seen the larger + # budget, and the next pass allocates against this limit at + # once. Confirm with a second reading here instead of starting + # it under a cap left over from the last pass's pressure. + self.sleep(SETTLE_INTERVAL) + *_, again_known, again = self._observe() + if again_known and again >= self.limit + 64 * 2**20: + self.pending = candidate = min(candidate, again) if self.pending is not None: target, action = min(candidate, self.pending), 'grow' self.pending = None diff --git a/freevideo_engine/guide_audio.py b/freevideo_engine/guide_audio.py new file mode 100644 index 00000000..4633534d --- /dev/null +++ b/freevideo_engine/guide_audio.py @@ -0,0 +1,108 @@ +"""Keep the guide clip's own sound on a finished guide-based result. + +The video stream is copied packet by packet; only the audio stream is replaced, with +the guide's audio trimmed or padded with silence to the video length. Requests +without "keep_source_audio" never reach this module. +""" +from fractions import Fraction +from pathlib import Path + +import av +import numpy as np + + +def has_audio(path): + try: + with av.open(str(path)) as container: + return bool(container.streams.audio) + except av.error.FFmpegError as error: + raise ValueError('Cannot read the guide clip: %s' % error) from error + + +def _video_seconds(container, stream): + end = 0 + for packet in container.demux(stream): + if packet.pts is not None and packet.duration is not None: + end = max(end, packet.pts + packet.duration) + if not end: + raise ValueError('Cannot measure the video length') + return float(end * stream.time_base) + + +def _guide_samples(path, rate, layout): + """Decode the guide's first audio stream to float planar samples at the output format.""" + blocks = [] + with av.open(str(path)) as container: + stream = container.streams.audio[0] + resampler = av.AudioResampler(format='fltp', layout=layout, rate=rate) + for frame in container.decode(stream): + for out in resampler.resample(frame): + blocks.append(out.to_ndarray()) + for out in resampler.resample(None): + blocks.append(out.to_ndarray()) + channels = len(av.AudioLayout(layout).channels) + return np.concatenate(blocks, axis=1) if blocks else np.zeros((channels, 0), np.float32) + + +def replace_audio(video, guide, output, *, rate=48000, layout='stereo'): + """Write ``output``: ``video``'s picture untouched plus ``guide``'s audio fitted to its length.""" + video, guide, output = Path(video), Path(guide), Path(output) + if not has_audio(guide): + raise ValueError('The guide clip has no audio stream') + with av.open(str(video)) as source: + picture = source.streams.video[0] + seconds = _video_seconds(source, picture) + wanted = round(seconds * rate) + samples = _guide_samples(guide, rate, layout)[:, :wanted] + if samples.shape[1] < wanted: + samples = np.pad(samples, ((0, 0), (0, wanted - samples.shape[1]))) + with av.open(str(video)) as source, av.open(str(output), 'w', format='mp4') as result: + picture = source.streams.video[0] + copied = result.add_stream_from_template(picture) + sound = result.add_stream('aac', rate=rate, layout=layout) + for packet in source.demux(picture): + if packet.dts is None: + continue + packet.stream = copied + result.mux(packet) + position = 0 + while position < wanted: + block = np.ascontiguousarray(samples[:, position:position + 1024]) + frame = av.AudioFrame.from_ndarray(block, format='fltp', layout=layout) + frame.sample_rate = rate + frame.pts = position + frame.time_base = Fraction(1, rate) + for packet in sound.encode(frame): + result.mux(packet) + position += block.shape[1] + for packet in sound.encode(None): + result.mux(packet) + return seconds + + +def apply(video, media): + """Replace ``video`` in place when the request's guide asks to keep its audio. Returns True if replaced.""" + guide = next((row for row in media.get('references', []) if row.get('guide') and row.get('keep_source_audio')), None) + if guide is None: + return False + video = Path(video) + temporary = video.with_name(video.stem + '.remux.mp4') + try: + replace_audio(video, guide['path'], temporary) + temporary.replace(video) + finally: + temporary.unlink(missing_ok=True) + return True + + +def apply_to_report(video, media, report): + """Run ``apply`` after a successful generation; a failed remux leaves the finished video as it is. + + The result is recorded in ``report``: ``keep_source_audio`` is True when the audio was replaced, + and ``keep_source_audio_failed`` holds a short reason when it could not be. + """ + try: + report['keep_source_audio'] = apply(video, media) + except Exception as error: # the video is already complete and valid; only the audio swap failed + report['keep_source_audio'] = False + report['keep_source_audio_failed'] = ('%s: %s' % (type(error).__name__, error))[:300] diff --git a/freevideo_engine/guide_layout.py b/freevideo_engine/guide_layout.py new file mode 100644 index 00000000..d7b2be51 --- /dev/null +++ b/freevideo_engine/guide_layout.py @@ -0,0 +1,46 @@ +"""Guide layout: a guide clip shares the generated clip's time origin. + +The native H3 AddGuide node packs [text | guide rows | references | target] with the guide rows +at time origin cursor + 5/3 * frame_index, the same origin as the target rows. The Ref2VA layout +builder gives every video reference its own time slot instead. This wrapper builds the ordinary +Ref2VA layout with the guides as the LAST references, then rewinds the target rows' time +coordinate by the slot the first guide occupied so that guides and target share one origin. +Guides after the first are moved onto the same origin, and a guide with a frame index is offset +by 5/3 per frame (the native frame-to-time rescale). + +Requests without a guide return the builder's layout object unchanged. +""" + +FRAME_RESCALE = 5. / 3. + + +def build(builder, text_token_tags, references, condition_latents, audio_conditions, *args): + layout = builder(text_token_tags, references, condition_latents, audio_conditions, *args) + guides = [i for i, ref in enumerate(references) if getattr(ref, 'guide', False)] + if not guides: + return layout + count = len(guides) + if guides != list(range(len(references) - count, len(references))): + raise ValueError('Guide clips must come after all other references') + positions, tags, video_ids, audio_ids, text_ids, rows, audio_rows = layout + patch_h, patch_w = args[-4][1], args[-4][2] + # Guides are silent, so their latents are the last `count` entries of condition_latents. + sizes = [] + for latent in condition_latents[-count:]: + sizes.append(latent.shape[2] * (latent.shape[3] // patch_h) * (latent.shape[4] // patch_w)) + end = rows + spans = [] + for n in reversed(sizes): + spans.append(video_ids[:end][-n:]) + end -= n + spans.reverse() + audio_start = int(audio_ids[audio_rows:].min()) + first = float(positions[int(spans[0][0]), 0]) + shift = float(positions[int(video_ids[rows]), 0]) - first + positions[audio_start:, 0] -= shift + for span, reference in zip(spans, references[-count:]): + offset = first - float(positions[int(span[0]), 0]) + index = getattr(reference, 'guide_index', 0) + if offset or index: + positions[span, 0] += offset + FRAME_RESCALE * index + return positions, tags, video_ids, audio_ids, text_ids, rows, audio_rows diff --git a/freevideo_engine/image_gen/failure.py b/freevideo_engine/image_gen/failure.py index 9055e01b..6aeb6ef9 100644 --- a/freevideo_engine/image_gen/failure.py +++ b/freevideo_engine/image_gen/failure.py @@ -199,6 +199,8 @@ def __init__(self, action, value): self.plan = None self.pack = None self.start_memory = None + # ComfyUI's own models this request unloaded before its worker started (comfy_residency.release_native). + self.native_release = None self.update_request(value) def note_start(self): @@ -301,6 +303,10 @@ def service(self, video_busy, worker_restarts, error): service = dict(video_busy=bool(video_busy), worker_restarts=worker_restarts) if self.start_memory: service['memory_at_request_start'] = clean_sample(self.start_memory) + released = self.native_release + if isinstance(released, dict): + service['comfy_models_released'] = {key: released[key] for key in + ('models', 'reserved_bytes', 'reserved_after_bytes', 'released') if type(released.get(key)) in (int, bool)} # Numbers and fixed words only, taken by the service when it saw the worker end. reading = clean_reading(getattr(error, 'exit_reading', None)) if reading: diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index a7b9e681..3ff148ff 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -4,6 +4,7 @@ from collections import OrderedDict from copy import deepcopy import json +import logging import math import os from pathlib import Path @@ -36,6 +37,15 @@ 'invalid_media', 'invalid_points', 'too_many_images')) +def comfy_server(): + """ComfyUI's server in this process, or None (tests, the engine's own command line).""" + try: + from server import PromptServer + return PromptServer.instance + except (ImportError, AttributeError): + return None + + class WorkerIdleExit(RuntimeError): """The worker closed itself after its idle timeout before it answered anything.""" @@ -658,6 +668,9 @@ def memory_budget(self, machine): from ..background_compile import COMPILE from ..gpu_budget import open_bounded from ..resident_process import OWNER + from ..comfy_residency import native_reclaimable + # ComfyUI's own idle models in this process: work() unloads them before the worker starts. + native = native_reclaimable(comfy_server()) def read(): gpu = Gpu(machine.get('gpu_uuid')) @@ -681,7 +694,7 @@ def read(): # The worker still checks live capacity before inference. return total credit = sum(size for pid, size in rows.items() if pid in owned) - return min(total, total - used + credit) + return min(total, total - used + credit + native) finally: gpu.close() @@ -878,6 +891,17 @@ def work(self, root, machine): raise BlockingIOError('busy_video') resident_process.OWNER.stop_prewarm() IDLE.stop() + from ..comfy_residency import release_native + try: + released = release_native(comfy_server()) + except Exception: # only frees memory early; the picture's own budget still decides + logging.warning('[FreeVideo] Could not unload ComfyUI models before the picture', exc_info=True) + released = None + if released is not None: + logging.info('[FreeVideo] Unloaded ComfyUI models before the picture: %s', released) + with self.mutex: + if self.context is not None: + self.context.native_release = released with runtime_lock(Path(root) / 'engine.lock', inherit=False) as descriptor: try: if self.session_stop.is_set(): diff --git a/freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt b/freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt new file mode 100644 index 00000000..d3e3fc83 --- /dev/null +++ b/freevideo_engine/licenses/Depth-Anything-V2-Apache-2.0.txt @@ -0,0 +1,217 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +FreeVideo attribution and modification notice +------------------------------------------- +Depth Anything V2 Small code and weights: Depth Anything authors (2024). +DINOv2: Copyright (c) Meta Platforms, Inc. and affiliates. +Source: https://github.com/DepthAnything/Depth-Anything-V2 +Source commit: a561b849ebae10a6f5ef49e26c83cbbcd36c71bf +Small model: https://huggingface.co/depth-anything/Depth-Anything-V2-Small +Weight revision: 03876f8651c73a60fe4c2c48294e09fcb6838fcf +SHA-256: 715fade13be8f229f8a70cc02066f656f2423a59effd0579197bbf57860e1378 +Only Small is included. The Apache-2.0 permission for Small does not apply to +Base/Large/Giant weights, which are not bundled or downloaded by this feature. +FreeVideo changes: retain Small DPT and DINOv2 only; remove xformers, nested-tensor +and SwiGLU paths; use ordinary torch matmul/softmax attention on every machine; +replace torchvision Compose with the same sequential transforms; follow the +model's explicit device, and add bounded, cached video conversion in a worker. diff --git a/freevideo_engine/locking.py b/freevideo_engine/locking.py index 9524bc6f..2acd4a47 100644 --- a/freevideo_engine/locking.py +++ b/freevideo_engine/locking.py @@ -11,6 +11,7 @@ from pathlib import Path import re import sys +import threading import time @@ -22,23 +23,31 @@ ENGINE_BUSY = 'Setup, testing or generation is already using this installation/GPU lock. Retry after it finishes.' -def lock_holders(paths): +def lock_holders(paths, *, deadline=None): """Read-only Linux diagnostics, including orphans retaining inherited FDs.""" if windows() or not Path('/proc/self/fd').is_dir(): return [] identities = set() for path in paths: + if deadline is not None and time.monotonic() >= deadline: + break try: item = Path(path).stat() identities.add((item.st_dev, item.st_ino)) except OSError: pass holders = [] + if not identities: + return [] for directory in Path('/proc').iterdir(): + if deadline is not None and time.monotonic() >= deadline: + break if not directory.name.isdigit(): continue try: for fd in (directory / 'fd').iterdir(): + if deadline is not None and time.monotonic() >= deadline: + break try: item = fd.stat() if (item.st_dev, item.st_ino) in identities: @@ -51,6 +60,193 @@ def lock_holders(paths): return sorted(holders) +# Only these small, public command identifiers may leave process diagnostics. +_COMMAND_WORD = re.compile(r'[a-z][a-z0-9_.-]{0,40}\Z') +_EXECUTABLE = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,80}\Z') +_HOLDER_TIMEOUT = 2.0 + + +def _executable_basename(value): + name = value.replace('\\', '/').rsplit('/', 1)[-1] + return name if _EXECUTABLE.fullmatch(name) else 'unknown' + + +def _lock_command(cmdline): + """Pure allowlist parser: no scripts, option values, prompts or environment.""" + arguments = cmdline.decode('utf-8', errors='replace').rstrip('\0').split('\0') + executable = _executable_basename(arguments[0]) + index = 1 + # Do not mistake a script/-c argument or an option's value for Python's -m. + while index < len(arguments) and arguments[index] in ( + '-B', '-u', '-E', '-s', '-S', '-I', '-O', '-OO', '-q', '-P'): + index += 1 + words = [] + if index < len(arguments) and arguments[index] == '-m': + # Words after the module are that program's own arguments. Only our own + # modules have subcommands known not to carry private values. + own = arguments[index + 1].startswith('freevideo_engine') if index + 1 < len(arguments) else False + for word in arguments[index + 1:index + (4 if own else 2)]: + if not _COMMAND_WORD.fullmatch(word): + break # Never skip an option/path to expose its value as a word. + words.append(word) + return executable, ' '.join(words) + + +def _holder_line(pid, executable, command='', seconds=None): + line = f'Lock file open: pid {int(pid)}, {_executable_basename(executable)}' + if command: + line += f' ({command})' + if seconds is not None and math.isfinite(seconds) and seconds >= 0: + line += f', running {int(seconds)} s' + return line + + +def _linux_holder(pid): + executable, command, seconds = 'unknown', '', None + try: + with (Path('/proc') / str(pid) / 'cmdline').open('rb') as stream: + executable, command = _lock_command(stream.read(65536)) + except Exception: + pass + try: + # comm may itself contain spaces and parentheses; starttime is field 22. + stat = (Path('/proc') / str(pid) / 'stat').read_text() + ticks = int(stat.rsplit(')', 1)[1].split()[19]) + uptime = float(Path('/proc/uptime').read_text().split()[0]) + seconds = uptime - ticks / os.sysconf('SC_CLK_TCK') + except Exception: + pass + return _holder_line(pid, executable, command, seconds) + + +def _windows_lock_holders(paths, deadline, lines): + """Restart Manager is read-only here: never request a shutdown or restart.""" + import ctypes + from ctypes import POINTER, c_int32, c_uint32, c_void_p, c_wchar, c_wchar_p + + class FileTime(ctypes.Structure): + _fields_ = [('low', c_uint32), ('high', c_uint32)] + + class UniqueProcess(ctypes.Structure): + _fields_ = [('pid', c_uint32), ('started', FileTime)] + + class ProcessInfo(ctypes.Structure): + _fields_ = [('process', UniqueProcess), ('app', c_wchar * 256), + ('service', c_wchar * 64), ('kind', c_int32), + ('status', c_uint32), ('session', c_uint32), ('restartable', c_int32)] + + manager = ctypes.WinDLL('rstrtmgr.dll', use_last_error=True) + kernel = ctypes.WinDLL('kernel32.dll', use_last_error=True) + signatures = ( + (manager.RmStartSession, [POINTER(c_uint32), c_uint32, c_wchar_p], c_uint32), + (manager.RmRegisterResources, [c_uint32, c_uint32, POINTER(c_wchar_p), c_uint32, + POINTER(UniqueProcess), c_uint32, POINTER(c_wchar_p)], c_uint32), + (manager.RmGetList, [c_uint32, POINTER(c_uint32), POINTER(c_uint32), + POINTER(ProcessInfo), POINTER(c_uint32)], c_uint32), + (manager.RmEndSession, [c_uint32], c_uint32), + (kernel.OpenProcess, [c_uint32, c_int32, c_uint32], c_void_p), + (kernel.QueryFullProcessImageNameW, [c_void_p, c_uint32, c_wchar_p, POINTER(c_uint32)], c_int32), + (kernel.CloseHandle, [c_void_p], c_int32), + ) + for function, arguments, result in signatures: + function.argtypes, function.restype = arguments, result + session = c_uint32() + if time.monotonic() >= deadline or manager.RmStartSession(ctypes.byref(session), 0, ctypes.create_unicode_buffer(33)): + return + try: + resources = (c_wchar_p * len(paths))(*(str(Path(path).absolute()) for path in paths)) + if not resources or time.monotonic() >= deadline: + return + if manager.RmRegisterResources(session, len(resources), resources, 0, None, 0, None): + return + needed, count, reason = c_uint32(), c_uint32(), c_uint32() + items = None + # The list can grow between calls. Bound both allocation and retries. + for _ in range(3): + if time.monotonic() >= deadline: + return + status = manager.RmGetList(session, ctypes.byref(needed), ctypes.byref(count), items, ctypes.byref(reason)) + if status == 0: + break + if status != 234 or not 0 < needed.value <= 4096: # ERROR_MORE_DATA + return + count.value = needed.value + items = (ProcessInfo * count.value)() + else: + return + holders = {item.process.pid: item for item in (items or ())[:count.value] + if item.process.pid and item.process.pid != os.getpid()} + for pid in sorted(holders)[:5]: + if time.monotonic() >= deadline: + return + item = holders[pid] + executable = _executable_basename(item.app) + seconds = None + try: + ticks = (item.process.started.high << 32) | item.process.started.low + if ticks: + seconds = time.time() - (ticks / 10_000_000 - 11_644_473_600) + except Exception: + pass + # RM's app name is the fallback when the process has exited or access + # is denied. QueryFullProcessImageNameW returns no command arguments. + position = len(lines) + lines.append(_holder_line(pid, executable, seconds=seconds)) + handle = None + try: + handle = kernel.OpenProcess(0x1000, False, pid) # QUERY_LIMITED_INFORMATION + if handle and time.monotonic() < deadline: + image = ctypes.create_unicode_buffer(32768) + size = c_uint32(len(image)) + if kernel.QueryFullProcessImageNameW(handle, 0, image, ctypes.byref(size)): + lines[position] = _holder_line(pid, image.value, seconds=seconds) + except Exception: + pass + finally: + if handle: + kernel.CloseHandle(handle) + if len(holders) > 5: + lines.append(f'Lock file open: and {len(holders) - 5} more') + finally: + manager.RmEndSession(session) + + +def describe_lock_holders(paths) -> list[str]: + """Best-effort, redacted diagnostics; even a stalled OS call gets two seconds. + + A daemon thread is deliberate: native calls have no timeout, and an attempt + to cancel Restart Manager must not delay the busy exit or interpreter exit. + Completed rows remain available if a later process/API query fails or hangs. + """ + lines = [] + + def query(): + try: + if windows(): + _windows_lock_holders(paths, deadline, lines) + else: + holders = sorted(set(lock_holders(paths, deadline=deadline)) - {os.getpid()}) + for pid in holders[:5]: + if time.monotonic() >= deadline: + return + lines.append(_linux_holder(pid)) + if len(holders) > 5: + lines.append(f'Lock file open: and {len(holders) - 5} more') + except Exception: + pass + finally: + done.set() + + try: + deadline = time.monotonic() + _HOLDER_TIMEOUT + done = threading.Event() + threading.Thread(target=query, daemon=True, name='lock-holder-diagnostics').start() + done.wait(max(0, deadline - time.monotonic())) + except Exception: + pass + return lines[:] + + def device_lock_path(device, path=None): """The lease for one device, beside the machine lease it shares a root with.""" base = Path(path or os.environ.get('FREEVIDEO_LOCK_PATH', data_root() / 'engine.lock')) diff --git a/freevideo_engine/managed.py b/freevideo_engine/managed.py index 25ef5a8a..a19e17e8 100644 --- a/freevideo_engine/managed.py +++ b/freevideo_engine/managed.py @@ -9,7 +9,7 @@ from .agent import enabled as agent_enabled from .bootstrap import DEFAULT_ROOT from . import __version__ -from .locking import runtime_lock, LOCK_ENV, ENGINE_BUSY +from .locking import runtime_lock, describe_lock_holders, LOCK_ENV, ENGINE_BUSY from . import processes from .system import windows @@ -39,7 +39,7 @@ def command_for(machine, command, arguments): return [machine['python'], '-m', 'freevideo_engine', command, *extra, *arguments] -def _main(argv=None): +def _main(argv=None, *, lock_paths=None): argv = list(sys.argv[1:] if argv is None else argv) parser = argparse.ArgumentParser(prog='./freevideo', add_help=False) parser.add_argument('--root', type=Path, default=Path(os.environ.get('FREEVIDEO_HOME', DEFAULT_ROOT))) @@ -117,6 +117,8 @@ def _main(argv=None): or (command == 'doctor' and '--probe' not in tail) or '--help' in tail or '-h' in tail) snapshot = json.loads(config.read_text(encoding='utf-8')) lock_path = os.environ.get('FREEVIDEO_LOCK_PATH', str(Path(snapshot['root']) / 'engine.lock')) + if lock_paths is not None: + lock_paths.append(lock_path) with nullcontext() if readonly else runtime_lock(lock_path, wait_seconds=3 if command == 'generate' else 0) as descriptor: machine = json.loads(config.read_text(encoding='utf-8')) if machine != snapshot: @@ -137,10 +139,31 @@ def _main(argv=None): def main(argv=None): + lock_paths = [] try: - return _main(argv) + return _main(argv, lock_paths=lock_paths) except BlockingIOError: print(ENGINE_BUSY, file=sys.stderr) + try: + if not lock_paths: + # Normally _main supplies the configured path. Keep failures + # before that point best-effort, without reading configuration. + path = os.environ.get('FREEVIDEO_LOCK_PATH') + if path: + lock_paths.append(path) + else: + arguments = list(sys.argv[1:] if argv is None else argv) + root = os.environ.get('FREEVIDEO_HOME', DEFAULT_ROOT) + for index, argument in enumerate(arguments): + if argument == '--root': + root = arguments[index + 1] + elif argument.startswith('--root='): + root = argument.partition('=')[2] + lock_paths.append(Path(root) / 'engine.lock') + for line in describe_lock_holders(lock_paths): + print(line, file=sys.stderr) + except Exception: + pass return 1 except KeyboardInterrupt: print('Cancelled. Workers stopped; existing outputs and reports retained.', file=sys.stderr) diff --git a/freevideo_engine/media_conditioning.py b/freevideo_engine/media_conditioning.py index 09e60cb6..2a067f28 100644 --- a/freevideo_engine/media_conditioning.py +++ b/freevideo_engine/media_conditioning.py @@ -98,9 +98,22 @@ def _describe(value, width, height, frames): raise ValueError('Ref2VA needs reference latents and no keyframe anchors') # Reuse the shared prompt/token validation without pretending references are anchors. to_cache([[prompt[None], {'minimax_token_tags': tags}]], task='ref2va') - video_rows = audio_rows = 0 + video_rows = audio_rows = seen_rows = 0 + guides = 0 for ref in refs: kind = ref.get('kind') + if ref.get('guide'): + # A guide is an ordinary silent video latent whose rows only differ in + # time coordinates (guide_layout), so the shape checks below apply + # unchanged. It must come after the other references and has no audio. + if kind not in ('image', 'video') or ref.get('audio_latent') is not None or 'early_steps' in ref: + raise ValueError('A guide must be a silent video or still with no early_steps') + guides += 1 + index = ref.get('guide_index', 0) + if type(index) is not int or not 0 <= index < frames: + raise ValueError('Guide frame index is outside the clip') + elif guides: + raise ValueError('Guide references must come after all other references') if 'early_steps' in ref: from .media_request import validate_early_steps validate_early_steps(ref) @@ -111,12 +124,16 @@ def _describe(value, width, height, frames): if latent.shape[2] < 1 or any(n < 2 or n % 2 for n in latent.shape[-2:]): raise ValueError('Reference latent dimensions must fit H3 patches') video_rows += math.prod(latent.shape[2:]) // 4 + if not ref.get('guide'): + seen_rows += math.prod(latent.shape[2:]) // 4 if kind == 'audio' or ref.get('audio_latent') is not None: audio = _tensor(ref.get('audio_latent'), (None, 32), 'reference audio rows') if audio.shape[0] % 2: raise ValueError('Reference audio must contain both channels') audio_rows += audio.shape[0] - if video_rows and not bool((tags == 0).any()): + # Guide rows are never shown to the vision-language encoder, so only the + # other visual references need jointly encoded vision tokens. + if seen_rows and not bool((tags == 0).any()): raise ValueError('Visual references need jointly encoded vision tokens') task = 'ref2va_av' if video_rows and audio_rows else 'ref2va_audio' if audio_rows else 'ref2va' value['task'] = task @@ -134,11 +151,19 @@ def from_comfy(conditioning, width, height, frames): if unknown: raise ValueError('Unsupported conditioning modifiers: ' + ', '.join(sorted(unknown))) keyframes, refs = extra.get('minimax_keyframes', []), extra.get('minimax_refs', []) - if keyframes and refs: - raise ValueError('Combined keyframe/reference conditioning is not an official VDN layout') anchors = [] + guides = [] for item in keyframes: index = item.get('resolved_frame_index') + if refs: + # Keyframes next to references are packed as guides that share the clip's + # time origin (same as the native AddGuide node), so any in-clip index + # is a valid position; the latent shape guard in describe() still applies. + if type(index) is not int or not 0 <= index < frames: + raise ValueError('Guide frame index is outside the clip') + guides.append(dict(kind='video', guide=True, guide_index=index, + latent=_tensor(item.get('latent'), (1, 24, None, None, None), 'guide video'))) + continue if index not in (0, frames - 1): raise ValueError('Only first and last keyframe anchors are supported') anchors.append('first' if index == 0 else 'last') @@ -160,4 +185,5 @@ def from_comfy(conditioning, width, height, frames): audio = _tensor(row.get('audio_latent'), (1, 32, 2, None), 'native reference audio') ref['audio_latent'] = audio[0].permute(1, 2, 0).reshape(-1, 32).contiguous() value['references'].append(ref) + value['references'].extend(guides) return value, describe(value, width, height, frames) diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index 4544a681..cacccd8e 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -135,6 +135,8 @@ def prepare(media, canvas, directory): ref = {'kind': kind} if 'early_steps' in row: ref['early_steps'] = row['early_steps'] + if row.get('guide'): + ref['guide'] = True numbers[kind] += 1 longer, length, used = False, None, limit / 24. target = directory / ('reference-%02d.npy' % index) @@ -183,7 +185,7 @@ def prepare(media, canvas, directory): ref.update(pixels=str(target), frames=count, normalized_fps=24) used = count / 24. # The clip's own sound is a reference too, unless the request turned it off. - soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False else \ + soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False or row.get('guide') else \ _audio(path, directory / ('reference-%02d-audio.npy' % index), limit / 24.) longer = longer or longer_audio if soundtrack: @@ -191,7 +193,10 @@ def prepare(media, canvas, directory): ref_items.append({'type': 'audio'}) selected = list(range(0, count, 12)) sampled = torch.from_numpy(np.array(pixels[selected])).float().div_(255) - ref_items.append({'type': 'video', 'data': sampled, 'timestamps': [i / 2. for i in range(len(selected))]}) + if not row.get('guide'): + # A guide only supplies motion latents; it is not described to the + # vision-language encoder, like the native AddGuide node. + ref_items.append({'type': 'video', 'data': sampled, 'timestamps': [i / 2. for i in range(len(selected))]}) del pixels else: ref['audio'], longer, length = _audio(path, target, limit / 24.) @@ -212,7 +217,7 @@ def reference_options(normalized, options, step): """Filter whole references, including a video's optional audio presentation.""" items, offset = [], 0 for row in normalized: - count = 1 + int(row['kind'] == 'video' and bool(row.get('audio'))) + count = 0 if row.get('guide') else 1 + int(row['kind'] == 'video' and bool(row.get('audio'))) if 'early_steps' not in row or step < row['early_steps']: items.extend(options['minimax_ref_items'][offset:offset + count]) offset += count @@ -289,7 +294,7 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu if torch.device(device).type not in ('cuda', 'mps'): raise ValueError('Media encoding requires a CUDA or MPS device') empty_cache = torch.mps.empty_cache if torch.device(device).type == 'mps' else torch.cuda.empty_cache - refs = [{key: row[key] for key in ('kind', 'early_steps') if key in row} for row in normalized] + refs = [{key: row[key] for key in ('kind', 'early_steps', 'guide') if key in row} for row in normalized] keys, cache, hits = {}, None, set() if cache_root and any(row.get('pixels') for row in normalized): # The same picture or clip with another prompt reuses its latent and may skip loading the encoder. diff --git a/freevideo_engine/media_request.py b/freevideo_engine/media_request.py index be8d0ec2..56571c79 100644 --- a/freevideo_engine/media_request.py +++ b/freevideo_engine/media_request.py @@ -55,7 +55,7 @@ def file_info(name): raise ValueError('Invalid ' + field + ' entry') if field != 'references' and 'early_steps' in entry: raise ValueError('early_steps is only allowed on video references') - allowed = {'path', 'kind', 'early_steps', 'sound'} if field == 'references' else {'path', 'strength', 'alpha'} + allowed = {'path', 'kind', 'early_steps', 'sound', 'guide', 'keep_source_audio'} if field == 'references' else {'path', 'strength', 'alpha'} if set(entry) - allowed: raise ValueError('Unknown ' + field + ' fields') row = file_info(entry.get('path')) @@ -70,6 +70,16 @@ def file_info(name): validate_sound(entry) if not entry['sound']: row['sound'] = False + if 'guide' in entry: + validate_guide(entry) + # A false marker is not stored, so requests without a guide + # keep exactly the same rows and cache key as before. + if entry['guide']: + row['guide'] = True + if 'keep_source_audio' in entry: + validate_keep_source_audio(entry, row['path']) + if entry['keep_source_audio']: + row['keep_source_audio'] = True else: if Path(row['path']).suffix.lower() != '.safetensors': raise ValueError('LoRA requires safetensors; pickle adapters are not loaded') @@ -81,7 +91,13 @@ def file_info(name): row[name] = number result[field].append(row) if result['references'] and any(result.get(k) for k in ('first', 'last')): - raise ValueError('Reference and keyframe layouts cannot be mixed; use separate requests') + raise ValueError('Reference and first/last keyframe layouts cannot be mixed in one request; ' + 'mark the clip that carries the motion with "guide": true instead') + guides = [row for row in result['references'] if row.get('guide')] + if len(guides) > 1: + raise ValueError('At most one guide clip per request') + # The packed layout needs the guide after the other references. + result['references'].sort(key=lambda row: bool(row.get('guide'))) if value.get('conditioning_info') is not None: info = value['conditioning_info'] if not isinstance(info, dict) or info.get('task') not in TASKS: @@ -114,6 +130,33 @@ def validate_sound(row): raise ValueError('Video reference sound must be true or false') +def validate_guide(row): + """A guide is a video reference that shares the generated clip's time origin. + + It is encoded like a silent video reference, so it cannot carry its own + audio or a delayed start. + """ + if row.get('kind') != 'video': + raise ValueError('guide is only allowed on video references') + if type(row['guide']) is not bool: + raise ValueError('Video reference guide must be true or false') + if row['guide'] and 'early_steps' in row: + raise ValueError('A guide clip cannot use early_steps') + + +def validate_keep_source_audio(row, path): + """Keeping the guide's sound needs a guide clip that has an audio stream.""" + if type(row['keep_source_audio']) is not bool: + raise ValueError('keep_source_audio must be true or false') + if not row['keep_source_audio']: + return + if row.get('guide') is not True: + raise ValueError('keep_source_audio is only allowed on a guide clip') + from .guide_audio import has_audio + if not has_audio(path): + raise ValueError('keep_source_audio needs a guide clip with an audio stream') + + def reference_stages(media): """Active references at each boundary; zero-step inputs never reach an encoder.""" refs = media.get('references', []) @@ -134,7 +177,7 @@ def cache_key(prompt, media, canvas): for key in ('first', 'last', 'references'): value = media.get(key) if value: - identity[key] = [{k: v for k, v in row.items() if k != 'path'} + identity[key] = [{k: v for k, v in row.items() if k not in ('path', 'keep_source_audio')} for row in value] if isinstance(value, list) else value['sha256'] return 'multimodal:' + json.dumps(identity, sort_keys=True, ensure_ascii=False) diff --git a/freevideo_engine/model_repair.py b/freevideo_engine/model_repair.py index c0b84f01..97382730 100644 --- a/freevideo_engine/model_repair.py +++ b/freevideo_engine/model_repair.py @@ -242,3 +242,18 @@ def retain(root, path, approved): del values[str(path)] save(Path(root) / 'logs/model-mismatches.json', values) return target + + +def restore(root, path, retained, approved, prior=None): + """Undo retention after a failed download, preserving any newly present target.""" + if os.path.lexists(path): + return + check(retained, approved) + try: + rename_noreplace(retained, path) + except FileExistsError: + return + if prior is not None and same_identity(identity(path), prior): + values = read(root) + values[str(path)] = prior + save(Path(root) / 'logs/model-mismatches.json', values) diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index b17120c7..79131f5b 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -1323,22 +1323,90 @@ def relocate(source, destination): remove_tree(source) +# Checks compare content only: a copy, restore or drive that changes permission +# bits, or a global line-ending setting, must not make a pinned checkout look +# edited. These settings apply to that one Git command and are never stored. +CONTENT_ONLY = ('-c', 'core.filemode=false', '-c', 'core.autocrlf=false') + + +def checkout_environment(path, env): + """Git environment that reads the repository at `path` itself, never one around it.""" + env = dict(os.environ if env is None else env) + for name in ('GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR', 'GIT_OBJECT_DIRECTORY'): + env.pop(name, None) + # Without its own .git/refs (a backup that drops empty folders), Git would + # otherwise climb to an enclosing repository and report that one's HEAD. + env['GIT_CEILING_DIRECTORIES'] = os.path.dirname(os.path.abspath(str(path))) + return env + + +def sparse_pathspecs(patterns): + """Pathspecs equal to anchored, literal sparse patterns, or None when they cannot be expressed.""" + specs = [] + for pattern in patterns: + pattern = pattern.strip() + if not pattern or pattern.startswith('#'): + continue + if not pattern.startswith('/') or any(c in pattern for c in '!*?[\\'): + return None + name = pattern.strip('/') + if not name: + return None + specs.append(':(top,literal)' + name) + return specs or None + + +def sparse_enabled(path, git, env): + value = config_value(path, 'core.sparseCheckout', git, env) or '' + return value.lower() in ('true', 'yes', 'on', '1') + + +def sparse_patterns(path, git, env): + """The sparse checkout patterns the checkout at `path` uses, or None.""" + if not sparse_enabled(path, git, env): + return None + try: + text = io_path(Path(path) / '.git' / 'info' / 'sparse-checkout').read_text(encoding='utf-8') + except OSError: + return None + return [line.strip() for line in text.splitlines() if line.strip()] + + +def checkout_status(path, git, env, hidden, *, filemode=False): + """`git status` lines for tracked files, limited to the sparse checkout when there is one. + + Git 2.37 and newer report a file that appears at a path outside the sparse + patterns; such a file is not part of what FreeVideo uses. + """ + specs = sparse_pathspecs(sparse_patterns(path, git, env) or []) + settings = CONTENT_ONLY if not filemode else ('-c', 'core.filemode=true', '-c', 'core.autocrlf=false') + return subprocess_output.output_external([git, '-c', 'core.longpaths=true', '-C', str(path), *settings, + 'status', '--porcelain', '--untracked-files=no', + *(['--', *specs] if specs else [])], env=env, **hidden) + + def repository_state(path, git, env, revision='HEAD', **options): - """What `revision` is and what local changes the checkout at `path` has.""" + """What `revision` is and what content changes the checkout at `path` has.""" path = Path(path) prefix = [git, '-c', 'core.longpaths=true'] hidden = dict(hidden_console(), **options) if not git_folder_too_deep(path): - head = subprocess_output.output_external([*prefix, '-C', str(path), 'rev-parse', revision], env=env, **hidden).strip() - dirty = subprocess_output.output_external([*prefix, '-C', str(path), 'status', '--porcelain', '--untracked-files=no'], env=env, **hidden) + env = checkout_environment(path, env) + # A folder Git cannot read is reported by the caller, not as Git's own message in the log. + hidden.setdefault('stderr', subprocess.PIPE) + head = subprocess_output.output_external([*prefix, '-C', str(path), *CONTENT_ONLY, 'rev-parse', revision], + env=env, **hidden).strip() + dirty = checkout_status(path, git, env, hidden) return head, dirty # Name the repository instead of entering it. If Git cannot read it this # way, HEAD is read from the file and local changes cannot be checked. git_dir = '--git-dir=' + str(path / '.git') folder = tempfile.gettempdir() try: - head = subprocess_output.output_external([*prefix, git_dir, 'rev-parse', revision], cwd=folder, env=env, **hidden).strip() - dirty = subprocess_output.output_external([*prefix, git_dir, 'diff-index', '--cached', '--name-only', 'HEAD'], cwd=folder, env=env, **hidden) + head = subprocess_output.output_external([*prefix, git_dir, *CONTENT_ONLY, 'rev-parse', revision], + cwd=folder, env=env, **hidden).strip() + dirty = subprocess_output.output_external([*prefix, git_dir, *CONTENT_ONLY, 'diff-index', '--cached', + '--name-only', 'HEAD'], cwd=folder, env=env, **hidden) except (subprocess.CalledProcessError, OSError): # Git also refuses a --git-dir this long, so read what is asked from the files. head = stored_head(path) @@ -1351,8 +1419,29 @@ def repository_state(path, git, env, revision='HEAD', **options): return head, dirty +def stored_reference(path, name): + """The commit a reference such as refs/heads/main names, from its file or packed-refs, or None.""" + try: + return io_path(path / '.git' / name).read_text(encoding='utf-8').strip() + except OSError: + pass + try: + packed = io_path(path / '.git' / 'packed-refs').read_text(encoding='utf-8') + except OSError: + return None + for line in packed.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1] == name and not line.startswith(('#', '^')): + return parts[0] + return None + + def stored_head(path): - return io_path(path / '.git' / 'HEAD').read_text(encoding='utf-8').strip() + """The commit .git/HEAD names, following a symbolic reference such as `ref: refs/heads/main`.""" + head = io_path(path / '.git' / 'HEAD').read_text(encoding='utf-8').strip() + if head.startswith('ref:'): + return stored_reference(path, head.split(None, 1)[1].strip()) or head + return head def stored_tree(path, commit): @@ -1360,10 +1449,9 @@ def stored_tree(path, commit): import zlib try: if commit.startswith('ref:'): - name = commit.split(None, 1)[1] - commit = io_path(path / '.git' / name).read_text(encoding='utf-8').strip() + commit = stored_reference(path, commit.split(None, 1)[1].strip()) raw = zlib.decompress(io_path(path / '.git' / 'objects' / commit[:2] / commit[2:]).read_bytes()) - except (OSError, ValueError, IndexError, zlib.error): + except (OSError, ValueError, IndexError, TypeError, zlib.error): return None header, _, body = raw.partition(b'\0') first = body.split(b'\n', 1)[0] @@ -1372,14 +1460,169 @@ def stored_tree(path, commit): return None +def status_paths(lines): + return [line[3:].split(' -> ')[-1] for line in (lines or '').splitlines() if len(line) > 3] + + +def config_value(path, key, git, env): + """A value from the checkout's own config file (readable even where Git cannot open the repository).""" + try: + result = subprocess_output.run_external( + [git, 'config', '--file', str(Path(path) / '.git' / 'config'), '--get', key], env=env, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=30, **hidden_console()) + except (OSError, ValueError, subprocess.SubprocessError): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def same_source(first, second): + def normal(url): + url = url.strip().rstrip('/') + return url[:-4] if url.endswith('.git') else url + return normal(first) == normal(second) + + +def owned_checkout(target, url, sparse, git, env): + """Whether `target` is a checkout FreeVideo made itself and may set aside. + + Only dependency folders directly in a FreeVideo `vendor` folder qualify, and + only when the checkout is set up as clone() leaves it: the same sparse + patterns, or (without them) no origin or the same origin. A ComfyUI the user + works in is never replaced: its custom nodes and models live inside it. + """ + target = Path(target) + if target.parent.name != 'vendor' or not io_path(target / '.git').is_dir(): + return False + enabled = sparse_enabled(target, git, env) + if sparse: + try: + patterns = io_path(target / '.git' / 'info' / 'sparse-checkout').read_text(encoding='utf-8').split() + except OSError: + return False + return enabled and patterns == [p.strip() for p in sparse] + origin = config_value(target, 'remote.origin.url', git, env) + return not enabled and (origin is None or same_source(origin, url)) + + +def mismatch_message(target, git, env, head, commit, dirty): + """A specific description of how an existing checkout differs, naming only its folder and files.""" + details = [] + if head is None: + details.append('Its version could not be read.') + elif head != commit: + details.append('It holds a different version.') + changed = status_paths(dirty) + if changed: + shown = ', '.join(changed[:8]) + (' and %d more' % (len(changed) - 8) if len(changed) > 8 else '') + details.append('Files with changed content (%d): %s.' % (len(changed), shown)) + if head is not None: + try: + with_modes = status_paths(checkout_status(target, git, checkout_environment(target, env), + dict(hidden_console(), timeout=60), filemode=True)) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + with_modes = [] + modes = [name for name in with_modes if name not in set(changed)] + if modes: + details.append('1 other file differs only in permissions, which does not block installation.' + if len(modes) == 1 else + '%d other files differ only in permissions, which do not block installation.' % len(modes)) + return ('A dependency folder differs from the version FreeVideo needs: %s. %s ' + 'Rename or move this folder, then retry installation. FreeVideo will download a fresh copy.' + % (target, ' '.join(details))) + + +# Git reads a checkout's settings from .git\config by its ordinary path, even in +# the deep-folder branch above. Past the path limit it cannot tell whether +# FreeVideo made the folder, and a new copy in the same place could not be read +# either, so such a folder is reported before anything is renamed. +GIT_SETTINGS_SUFFIX = '\\.git\\config' + + +def git_settings_out_of_reach(path): + return is_windows() and path_chars(plain(path)) + len(GIT_SETTINGS_SUFFIX) > GIT_PATH_LIMIT + + +def path_too_long_message(target): + return ('The path of the dependency folder %s is too long for Git to read. Install FreeVideo in a folder ' + 'with a shorter path, such as D:\\FreeVideo, then click "Retry installation". ' + 'The folder was left unchanged.' % plain(target)) + + +REPLACED_MARKER = 'freevideo-replaced' + + +def set_aside(target): + """Rename an outdated checkout to `.replaced-