FreeVideo 0.4.29

THIRD_PARTY_NOTICES.md

@@ -19,7 +19,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th
 | [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) 1.13.8 speech runtime, built by FreeVideo without text-to-speech, ALSA and PortAudio (modified build; `__init__.py` edited; source snapshot 11afbd00 kept by FreeVideo; optional download for automatic subtitles, [build note](freevideo_engine/licenses/sherpa-onnx-build-NOTE.txt)). Includes statically linked kaldi-native-fbank, kaldi-decoder, kaldifst, OpenFst, simple-sentencepiece (Apache-2.0), KissFFT (BSD-3-Clause), pybind11 (BSD-3-Clause), nlohmann/json (MIT), hclust-cpp and Darts-clone (BSD-2-Clause), ThreadPool (Zlib). The Windows build also contains msvcp140.dll and msvcp140_1.dll (Microsoft Visual C++ 2022 runtime 14.44.35211, © Microsoft Corporation), unmodified and redistributed as permitted by Microsoft for its redistributable files. | Apache-2.0 |
 | [ONNX Runtime](https://github.com/microsoft/onnxruntime) 1.28.2 (CPU), © Microsoft Corporation; its third-party notices are shipped inside the sherpa-onnx wheel | MIT |
 | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 |
-| [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download; license file downloaded with the model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) |
+| [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) |
 | [NVIDIA NeMo stt_en_conformer_ctc_small](https://huggingface.co/nvidia/stt_en_conformer_ctc_small), © NVIDIA, exported to ONNX and quantized to int8 by the sherpa-onnx project, changed from the original; not endorsed by NVIDIA (optional download for English subtitles; [notice](freevideo_engine/licenses/NeMo-stt_en_conformer_ctc_small-NOTICE.txt)) | [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) |
 | [OpenAI Whisper tiny](https://github.com/openai/whisper), © 2022 OpenAI, exported to ONNX and quantized to int8 by the sherpa-onnx project (optional download for spoken-language detection) | [MIT](freevideo_engine/licenses/Whisper-MIT.txt) |
 | [SenseVoiceSmall](https://huggingface.co/FunAudioLLM/SenseVoiceSmall) by FunASR / FunAudioLLM (Alibaba Group), converted to ONNX and quantized to int8 by the sherpa-onnx project, used unchanged under its original name (optional download for multilingual subtitles; [notice](freevideo_engine/licenses/SenseVoice-NOTICE.txt)) | [FunASR Model Open Source License Agreement v1.1](freevideo_engine/licenses/FunASR-MODEL_LICENSE-v1.1.txt) |

__init__.py

@@ -45,6 +45,8 @@ async def comfy_entrypoint():
     capabilities_routes()
     from .freevideo_engine.subtitle_ears import register as subtitle_ears_routes
     subtitle_ears_routes()
+    from .freevideo_engine.editor_music import register as editor_music_routes
+    editor_music_routes()
     from .freevideo_engine.subtitle_files import register as subtitle_file_routes
     subtitle_file_routes()
     from .freevideo_engine.comfy_share import register as share_routes

freevideo_engine/agent/cast.py

@@ -57,8 +57,7 @@ def reference_prompt(card, *, face=False):
     """The one place the reference clip prompts are written."""
     who = (card.get('description') or card['name']).strip().rstrip('.')
     if card['kind'] == 'setting':
-        return ('%s. A slow, steady camera pan across the empty place, no people, even daylight, '
-                'sharp focus, no text.' % who)
+        return ('%s. A slow, steady camera pan across the place, no people, sharp focus, no text.' % who)
     if face:
         return ('%s. A close-up of the face and shoulders, looking straight at the camera with a neutral '
                 'expression, in front of a plain light grey studio backdrop. The person stays still. '

freevideo_engine/agent/errors.py

@@ -295,6 +295,7 @@ def safe_details(value):
     'not_found': '找不到这次任务或素材',
     'source_missing': '接续所需的上一段视频已经不在',
     'chain_missing': '接续所需的生成记录已经不在',
+    'not_resumable': '此任务无法继续生成',
     'chain_unavailable': '分段生成服务还没有启动',
     'continuation_unavailable': '当前生成方式不支持接续视频',
     'join_failed': '片段已生成，但拼接成片时出错',
@@ -345,6 +346,7 @@ def safe_details(value):
     'not_found': 'This task or media file cannot be found',
     'source_missing': 'The previous video needed to continue is no longer available',
     'chain_missing': 'The generation record needed to continue is no longer available',
+    'not_resumable': 'This task cannot be continued',
     'chain_unavailable': 'The part-by-part generation service has not started yet',
     'continuation_unavailable': 'The current generation mode cannot continue a video',
     'join_failed': 'The parts were generated, but joining them into one video failed',

freevideo_engine/agent/guide.py

@@ -189,7 +189,7 @@
                        for name, (method, path, summary) in SESSION_OPERATIONS.items()],
     'filming': [
         'Filming is for a film of several shots, each picked from takes. One video of up to 60 seconds is one generate '
-        'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate. '
+        'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate with "prompt": {"from_job": "<enhance id>"} rather than retyping the text. '
         'A film is cut into shots: see the rule "Cut by default" before making one long continued video.',
         'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.',
         'At start tell the user the estimated total time; on over_budget decide whether to ask them; '
@@ -243,6 +243,10 @@
     'edited. When the user tells you to stop, stop: cancel what you started and start nothing new. '
     'When cancelled_by is "user", the user stopped this job on the page and the page already shows it as stopped: '
     'send no message about it, and do not retry or resume it unless they ask.',
+    'A user message with revises=N replaces message N and everything you wrote after it: understand it afresh '
+    'from the new message. The jobs started for the old one are already stopped; do not continue them. '
+    'A control event with action "retried" means the user ran job refs.job again on the page: do not start it '
+    'again yourself; follow it like a job you started.',
     'Cast before you shoot. Search the cast library first. Use the card the user named, or one that clearly '
     'matches; ask if you are unsure. Create a card only when none fits: from the user\'s photos if they gave any, '
     'otherwise make its reference pictures with POST /cast/{id}/generate. '
@@ -257,7 +261,7 @@
     'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should '
     'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a '
     'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and '
-    'continuing again and again is not recommended. Each continued part is at least 5 seconds. One video has at '
+    'continuing again and again is not recommended. Each continued part is at least 5 seconds, and the last one can be as short as 2 seconds. One video has at '
     'most 8 parts (the first and 7 continuations, about 110 seconds in all) when every continued part is at '
     'least 10 seconds; with a continued part shorter than 10 seconds it has at most 4 parts, unless '
     'first_frame_reference is true. Filling that is not recommended. Past it, cut to a new shot.',
@@ -287,6 +291,8 @@
     '只在 FreeVideo 内工作，不要改动本机的其他文件，不要撤销用户做的修改。'
     '用户要求停止时，取消你启动的任务，不再启动新任务。'
     'cancelled_by 为 "user" 时，表示用户在页面上停止了该任务，页面已显示已停止：不要为此发消息；除非用户要求，否则不要重试或恢复。',
+    '带 revises=N 的用户消息取代 seq N 那条及其之后你写的内容，按新消息重新理解；为旧消息开的任务已被停掉，不要续旧的。'
+    'action 为 "retried" 的 control 事件表示用户在页面上重新运行了 refs.job 这个任务：不要自己再提交一次，像你自己提交的任务一样跟进。',
     '先选角，再拍摄。先搜索角色库，使用用户指定的卡片或明确符合要求的卡片；拿不准时询问用户。'
     '只有找不到合适卡片时才新建：用户提供了照片就用照片，否则通过 POST /cast/{id}/generate 生成参考图。'
     '开拍前用一句话告诉用户选角结果。后续镜头使用卡片当前的图片。保留你创建的卡片，方便以后拍片复用。',
@@ -295,7 +301,7 @@
     '先用 people（角色图和各自的位置）做出首帧，再用它生成视频。',
     '默认切镜头。能切镜头的地方就切：新镜头直接用对应的参考图（人物卡、场景图）生成，不接上一段视频。'
     '只有不能切镜的长镜头、必须连贯的动作、必须连贯的声音，才接着上一段生成，而且不建议一段接一段地连续接。'
-    '接着生成的每一段至少 5 秒。每段都不短于 10 秒时，一条视频最多 8 段（第一段加 7 次接着生成，总长约 110 秒）；'
+    '接着生成的每一段至少 5 秒，最后一段可短至 2 秒。每段都不短于 10 秒时，一条视频最多 8 段（第一段加 7 次接着生成，总长约 110 秒）；'
     '有短于 10 秒的段时最多 4 段，first_frame_reference 为 true 时除外。不建议接满，超过就切镜头。',
     '问话不是要求。用户问进度或状态（“在做吗”“怎么了”）时，用一句话照实回答，依据是你所知道的、任务和白模的状态，'
     '什么都不要开始；不要把问句当提示词。某条 inbox 带有 whitebox 时，说明用户刚启动了白模场景，'

freevideo_engine/agent/jobs.py

@@ -187,13 +187,16 @@ def part_prompt_problems(spec, prefix=''):
 
 
 def _continued_part_problem(seconds, chain):
-    """True when a video that continues another has a part the backend refuses, or is under MIN_PART_SECONDS.
+    """True when a video that continues another has a part the backend refuses, or is under MIN_TAIL_SECONDS.
 
     The backend compares each part's node input frames (overlap included); a part that
-    continues is also at least MIN_PART_SECONDS long.
+    another part continues is at least MIN_PART_SECONDS long, the last part MIN_TAIL_SECONDS.
     """
-    from ..chain_plan import MIN_PART_FRAMES, MIN_PART_SECONDS
-    return seconds < MIN_PART_SECONDS or any(block['frames'] < MIN_PART_FRAMES for block in chain['blocks'])
+    from ..chain_plan import MIN_PART_FRAMES, MIN_TAIL_FRAMES, MIN_TAIL_SECONDS
+    blocks = chain['blocks']
+    return seconds < MIN_TAIL_SECONDS or any(
+        block['frames'] < (MIN_TAIL_FRAMES if index == len(blocks) - 1 else MIN_PART_FRAMES)
+        for index, block in enumerate(blocks))
 
 
 def length_problem(seconds, continued, prefix='', *, minimum=None, frames=None):
@@ -232,7 +235,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N
     """
     if spec['kind'] != 'generate' or spec.get('refine_from'):
         return []
-    from ..chain_plan import MIN_PART_SECONDS, PlanError, parts_limit, plan
+    from ..chain_plan import MIN_TAIL_SECONDS, PlanError, parts_limit, plan
     from ..chain_runner import ChainShortPartsTooMany, ChainTooLong
     from ..chain_store import reference
     continued, frames = bool(spec.get('continue_from')), single_frame_limit(spec)
@@ -243,7 +246,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N
         except PlanError:
             return [length_problem(spec['seconds'], continued, prefix, frames=frames)]
         if continued and _continued_part_problem(spec['seconds'], chain):
-            return [length_problem(spec['seconds'], continued, prefix, minimum=MIN_PART_SECONDS, frames=frames)]
+            return [length_problem(spec['seconds'], continued, prefix, minimum=MIN_TAIL_SECONDS, frames=frames)]
     if runner is None:
         return []
     # A video with a continuing part shorter than full_part_frames has fewer parts, unless every part sees the first frame.
@@ -281,7 +284,7 @@ def fits(seconds):
             except ValueError:
                 return False
             return earlier + len(blocks) <= (short_limit if made_short or short_blocks(blocks) else limit)
-        shortest = MIN_PART_SECONDS if continued else 2
+        shortest = MIN_TAIL_SECONDS
         fit = next((s for s in range(int(spec['seconds']), shortest - 1, -1) if fits(s)), None)
         if fit is None:
             problem = schema.diagnostic(prefix + '/continue_from', 'a video of fewer than %d parts' % error.limit, None)
@@ -1098,6 +1101,51 @@ def resume(self, identity, *, key=None, value=None, replay_result=False, client=
             replace(original, row)
             return (original, False) if replay_result else original
 
+    def again(self, row):
+        """How a conversation card offers a finished job again, or None for one that runs or succeeded.
+
+        'continue' resumes a video and keeps its finished parts, 'retry' runs the same request from its start,
+        and 'resend' leaves it to the agent: the page says the user's words again. A video the user stopped only
+        goes on from its finished parts: nothing starts it over."""
+        if row['status'] not in ('failed', 'interrupted', 'cancelled'):
+            return None
+        stopped = row['kind'] in ('generate', 'assemble') and bool(user_stop_notice(row))
+        parent = row.get('parent') or {}
+        if parent.get('production') or parent.get('cast'):
+            return None if stopped else 'resend'
+        if row['kind'] == 'image':
+            return 'retry'
+        if row['kind'] != 'generate' or row['spec'].get('refine_from'):
+            return None if stopped else 'resend'
+        try:
+            blocks = self.chain(row['chain_id'])['blocks'] if row.get('chain_id') else ()
+        except (KeyError, JobError, OSError, ValueError):
+            blocks = ()
+        if any(block.get('state') == 'done' for block in blocks):
+            return 'continue'
+        return None if stopped else 'retry'
+
+    def run_again(self, identity):
+        """The user's Retry on a conversation card: the same job again, in the same card."""
+        row = self.get(identity)
+        way = self.again(row)
+        if way in ('continue', 'retry') and row['kind'] == 'generate':
+            return self.resume(identity)
+        if way != 'retry':
+            raise chain_error('not_resumable', 'Only failed, interrupted or cancelled videos and pictures run again.',
+                              'Say it to the agent again.')
+        with self.lock:
+            if row['status'] not in ('failed', 'interrupted', 'cancelled'):
+                raise chain_error('not_resumable', 'This picture is already running.', 'Wait for it to finish.')
+            # A picture has nothing finished to keep: its request, seed included, starts over.
+            dropped = ('error', 'result', '_cancel', 'cancelled_by', '_image', '_phase', 'waiting_for')
+            candidate = deepcopy({k: v for k, v in row.items() if k not in dropped})
+            candidate.update(status='waiting', started_at=None, finished_at=None, updated_at=self.clock(),
+                             attempt=row.get('attempt', 1) + 1)
+            self.save(candidate)
+            replace(row, candidate)
+            return row
+
     def queue_view(self):
         """Describe ownership/order without exposing studio workflows or prompts."""
         runner = self.port.chain_runner()