diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 66b947ee..5f7e5ba5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -19,7 +19,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th | [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) 1.13.8 speech runtime, built by FreeVideo without text-to-speech, ALSA and PortAudio (modified build; `__init__.py` edited; source snapshot 11afbd00 kept by FreeVideo; optional download for automatic subtitles, [build note](freevideo_engine/licenses/sherpa-onnx-build-NOTE.txt)). Includes statically linked kaldi-native-fbank, kaldi-decoder, kaldifst, OpenFst, simple-sentencepiece (Apache-2.0), KissFFT (BSD-3-Clause), pybind11 (BSD-3-Clause), nlohmann/json (MIT), hclust-cpp and Darts-clone (BSD-2-Clause), ThreadPool (Zlib). The Windows build also contains msvcp140.dll and msvcp140_1.dll (Microsoft Visual C++ 2022 runtime 14.44.35211, © Microsoft Corporation), unmodified and redistributed as permitted by Microsoft for its redistributable files. | Apache-2.0 | | [ONNX Runtime](https://github.com/microsoft/onnxruntime) 1.28.2 (CPU), © Microsoft Corporation; its third-party notices are shipped inside the sherpa-onnx wheel | MIT | | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 | -| [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download; license file downloaded with the model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) | +| [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) | | [NVIDIA NeMo stt_en_conformer_ctc_small](https://huggingface.co/nvidia/stt_en_conformer_ctc_small), © NVIDIA, exported to ONNX and quantized to int8 by the sherpa-onnx project, changed from the original; not endorsed by NVIDIA (optional download for English subtitles; [notice](freevideo_engine/licenses/NeMo-stt_en_conformer_ctc_small-NOTICE.txt)) | [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) | | [OpenAI Whisper tiny](https://github.com/openai/whisper), © 2022 OpenAI, exported to ONNX and quantized to int8 by the sherpa-onnx project (optional download for spoken-language detection) | [MIT](freevideo_engine/licenses/Whisper-MIT.txt) | | [SenseVoiceSmall](https://huggingface.co/FunAudioLLM/SenseVoiceSmall) by FunASR / FunAudioLLM (Alibaba Group), converted to ONNX and quantized to int8 by the sherpa-onnx project, used unchanged under its original name (optional download for multilingual subtitles; [notice](freevideo_engine/licenses/SenseVoice-NOTICE.txt)) | [FunASR Model Open Source License Agreement v1.1](freevideo_engine/licenses/FunASR-MODEL_LICENSE-v1.1.txt) | diff --git a/__init__.py b/__init__.py index 886ebffe..3349199c 100644 --- a/__init__.py +++ b/__init__.py @@ -45,6 +45,8 @@ async def comfy_entrypoint(): capabilities_routes() from .freevideo_engine.subtitle_ears import register as subtitle_ears_routes subtitle_ears_routes() + from .freevideo_engine.editor_music import register as editor_music_routes + editor_music_routes() from .freevideo_engine.subtitle_files import register as subtitle_file_routes subtitle_file_routes() from .freevideo_engine.comfy_share import register as share_routes diff --git a/freevideo_engine/agent/cast.py b/freevideo_engine/agent/cast.py index 51fa615a..6b2a2e21 100644 --- a/freevideo_engine/agent/cast.py +++ b/freevideo_engine/agent/cast.py @@ -57,8 +57,7 @@ def reference_prompt(card, *, face=False): """The one place the reference clip prompts are written.""" who = (card.get('description') or card['name']).strip().rstrip('.') if card['kind'] == 'setting': - return ('%s. A slow, steady camera pan across the empty place, no people, even daylight, ' - 'sharp focus, no text.' % who) + return ('%s. A slow, steady camera pan across the place, no people, sharp focus, no text.' % who) if face: return ('%s. A close-up of the face and shoulders, looking straight at the camera with a neutral ' 'expression, in front of a plain light grey studio backdrop. The person stays still. ' diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index 28ff01d3..97d362e2 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -295,6 +295,7 @@ def safe_details(value): 'not_found': '找不到这次任务或素材', 'source_missing': '接续所需的上一段视频已经不在', 'chain_missing': '接续所需的生成记录已经不在', + 'not_resumable': '此任务无法继续生成', 'chain_unavailable': '分段生成服务还没有启动', 'continuation_unavailable': '当前生成方式不支持接续视频', 'join_failed': '片段已生成，但拼接成片时出错', @@ -345,6 +346,7 @@ def safe_details(value): 'not_found': 'This task or media file cannot be found', 'source_missing': 'The previous video needed to continue is no longer available', 'chain_missing': 'The generation record needed to continue is no longer available', + 'not_resumable': 'This task cannot be continued', 'chain_unavailable': 'The part-by-part generation service has not started yet', 'continuation_unavailable': 'The current generation mode cannot continue a video', 'join_failed': 'The parts were generated, but joining them into one video failed', diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index c6706dac..f0095b93 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -189,7 +189,7 @@ for name, (method, path, summary) in SESSION_OPERATIONS.items()], 'filming': [ 'Filming is for a film of several shots, each picked from takes. One video of up to 60 seconds is one generate ' - 'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate. ' + 'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate with "prompt": {"from_job": ""} rather than retyping the text. ' 'A film is cut into shots: see the rule "Cut by default" before making one long continued video.', 'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.', 'At start tell the user the estimated total time; on over_budget decide whether to ask them; ' @@ -243,6 +243,10 @@ 'edited. When the user tells you to stop, stop: cancel what you started and start nothing new. ' 'When cancelled_by is "user", the user stopped this job on the page and the page already shows it as stopped: ' 'send no message about it, and do not retry or resume it unless they ask.', + 'A user message with revises=N replaces message N and everything you wrote after it: understand it afresh ' + 'from the new message. The jobs started for the old one are already stopped; do not continue them. ' + 'A control event with action "retried" means the user ran job refs.job again on the page: do not start it ' + 'again yourself; follow it like a job you started.', 'Cast before you shoot. Search the cast library first. Use the card the user named, or one that clearly ' 'matches; ask if you are unsure. Create a card only when none fits: from the user\'s photos if they gave any, ' 'otherwise make its reference pictures with POST /cast/{id}/generate. ' @@ -257,7 +261,7 @@ 'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should ' 'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a ' 'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and ' - 'continuing again and again is not recommended. Each continued part is at least 5 seconds. One video has at ' + 'continuing again and again is not recommended. Each continued part is at least 5 seconds, and the last one can be as short as 2 seconds. One video has at ' 'most 8 parts (the first and 7 continuations, about 110 seconds in all) when every continued part is at ' 'least 10 seconds; with a continued part shorter than 10 seconds it has at most 4 parts, unless ' 'first_frame_reference is true. Filling that is not recommended. Past it, cut to a new shot.', @@ -287,6 +291,8 @@ '只在 FreeVideo 内工作，不要改动本机的其他文件，不要撤销用户做的修改。' '用户要求停止时，取消你启动的任务，不再启动新任务。' 'cancelled_by 为 "user" 时，表示用户在页面上停止了该任务，页面已显示已停止：不要为此发消息；除非用户要求，否则不要重试或恢复。', + '带 revises=N 的用户消息取代 seq N 那条及其之后你写的内容，按新消息重新理解；为旧消息开的任务已被停掉，不要续旧的。' + 'action 为 "retried" 的 control 事件表示用户在页面上重新运行了 refs.job 这个任务：不要自己再提交一次，像你自己提交的任务一样跟进。', '先选角，再拍摄。先搜索角色库，使用用户指定的卡片或明确符合要求的卡片；拿不准时询问用户。' '只有找不到合适卡片时才新建：用户提供了照片就用照片，否则通过 POST /cast/{id}/generate 生成参考图。' '开拍前用一句话告诉用户选角结果。后续镜头使用卡片当前的图片。保留你创建的卡片，方便以后拍片复用。', @@ -295,7 +301,7 @@ '先用 people（角色图和各自的位置）做出首帧，再用它生成视频。', '默认切镜头。能切镜头的地方就切：新镜头直接用对应的参考图（人物卡、场景图）生成，不接上一段视频。' '只有不能切镜的长镜头、必须连贯的动作、必须连贯的声音，才接着上一段生成，而且不建议一段接一段地连续接。' - '接着生成的每一段至少 5 秒。每段都不短于 10 秒时，一条视频最多 8 段（第一段加 7 次接着生成，总长约 110 秒）；' + '接着生成的每一段至少 5 秒，最后一段可短至 2 秒。每段都不短于 10 秒时，一条视频最多 8 段（第一段加 7 次接着生成，总长约 110 秒）；' '有短于 10 秒的段时最多 4 段，first_frame_reference 为 true 时除外。不建议接满，超过就切镜头。', '问话不是要求。用户问进度或状态（“在做吗”“怎么了”）时，用一句话照实回答，依据是你所知道的、任务和白模的状态，' '什么都不要开始；不要把问句当提示词。某条 inbox 带有 whitebox 时，说明用户刚启动了白模场景，' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 75130776..51648bbe 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -187,13 +187,16 @@ def part_prompt_problems(spec, prefix=''): def _continued_part_problem(seconds, chain): - """True when a video that continues another has a part the backend refuses, or is under MIN_PART_SECONDS. + """True when a video that continues another has a part the backend refuses, or is under MIN_TAIL_SECONDS. The backend compares each part's node input frames (overlap included); a part that - continues is also at least MIN_PART_SECONDS long. + another part continues is at least MIN_PART_SECONDS long, the last part MIN_TAIL_SECONDS. """ - from ..chain_plan import MIN_PART_FRAMES, MIN_PART_SECONDS - return seconds < MIN_PART_SECONDS or any(block['frames'] < MIN_PART_FRAMES for block in chain['blocks']) + from ..chain_plan import MIN_PART_FRAMES, MIN_TAIL_FRAMES, MIN_TAIL_SECONDS + blocks = chain['blocks'] + return seconds < MIN_TAIL_SECONDS or any( + block['frames'] < (MIN_TAIL_FRAMES if index == len(blocks) - 1 else MIN_PART_FRAMES) + for index, block in enumerate(blocks)) def length_problem(seconds, continued, prefix='', *, minimum=None, frames=None): @@ -232,7 +235,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N """ if spec['kind'] != 'generate' or spec.get('refine_from'): return [] - from ..chain_plan import MIN_PART_SECONDS, PlanError, parts_limit, plan + from ..chain_plan import MIN_TAIL_SECONDS, PlanError, parts_limit, plan from ..chain_runner import ChainShortPartsTooMany, ChainTooLong from ..chain_store import reference continued, frames = bool(spec.get('continue_from')), single_frame_limit(spec) @@ -243,7 +246,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N except PlanError: return [length_problem(spec['seconds'], continued, prefix, frames=frames)] if continued and _continued_part_problem(spec['seconds'], chain): - return [length_problem(spec['seconds'], continued, prefix, minimum=MIN_PART_SECONDS, frames=frames)] + return [length_problem(spec['seconds'], continued, prefix, minimum=MIN_TAIL_SECONDS, frames=frames)] if runner is None: return [] # A video with a continuing part shorter than full_part_frames has fewer parts, unless every part sees the first frame. @@ -281,7 +284,7 @@ def fits(seconds): except ValueError: return False return earlier + len(blocks) <= (short_limit if made_short or short_blocks(blocks) else limit) - shortest = MIN_PART_SECONDS if continued else 2 + shortest = MIN_TAIL_SECONDS fit = next((s for s in range(int(spec['seconds']), shortest - 1, -1) if fits(s)), None) if fit is None: problem = schema.diagnostic(prefix + '/continue_from', 'a video of fewer than %d parts' % error.limit, None) @@ -1098,6 +1101,51 @@ def resume(self, identity, *, key=None, value=None, replay_result=False, client= replace(original, row) return (original, False) if replay_result else original + def again(self, row): + """How a conversation card offers a finished job again, or None for one that runs or succeeded. + + 'continue' resumes a video and keeps its finished parts, 'retry' runs the same request from its start, + and 'resend' leaves it to the agent: the page says the user's words again. A video the user stopped only + goes on from its finished parts: nothing starts it over.""" + if row['status'] not in ('failed', 'interrupted', 'cancelled'): + return None + stopped = row['kind'] in ('generate', 'assemble') and bool(user_stop_notice(row)) + parent = row.get('parent') or {} + if parent.get('production') or parent.get('cast'): + return None if stopped else 'resend' + if row['kind'] == 'image': + return 'retry' + if row['kind'] != 'generate' or row['spec'].get('refine_from'): + return None if stopped else 'resend' + try: + blocks = self.chain(row['chain_id'])['blocks'] if row.get('chain_id') else () + except (KeyError, JobError, OSError, ValueError): + blocks = () + if any(block.get('state') == 'done' for block in blocks): + return 'continue' + return None if stopped else 'retry' + + def run_again(self, identity): + """The user's Retry on a conversation card: the same job again, in the same card.""" + row = self.get(identity) + way = self.again(row) + if way in ('continue', 'retry') and row['kind'] == 'generate': + return self.resume(identity) + if way != 'retry': + raise chain_error('not_resumable', 'Only failed, interrupted or cancelled videos and pictures run again.', + 'Say it to the agent again.') + with self.lock: + if row['status'] not in ('failed', 'interrupted', 'cancelled'): + raise chain_error('not_resumable', 'This picture is already running.', 'Wait for it to finish.') + # A picture has nothing finished to keep: its request, seed included, starts over. + dropped = ('error', 'result', '_cancel', 'cancelled_by', '_image', '_phase', 'waiting_for') + candidate = deepcopy({k: v for k, v in row.items() if k not in dropped}) + candidate.update(status='waiting', started_at=None, finished_at=None, updated_at=self.clock(), + attempt=row.get('attempt', 1) + 1) + self.save(candidate) + replace(row, candidate) + return row + def queue_view(self): """Describe ownership/order without exposing studio workflows or prompts.""" runner = self.port.chain_runner() diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index 13c60877..80ad868d 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -50,7 +50,7 @@ def tool_list(): dict(name='freevideo_prompt_guide', title='H3 prompt guide', description='How to write prompts: sections, shots and cuts, camera motion, dialogue, references, short-drama tips, and a full example.', annotations=read), dict(name='freevideo_schema', title='Job JSON Schemas', description='JSON Schemas of every job kind and of image edit recipes.', annotations=read), dict(name='freevideo_plan', title='Plan a job', description="Validate a job without running it: actual frames/seconds, sampling passes, warnings and a time estimate from this computer's history. Same fields as freevideo_generate.", annotations=read), - dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise the video is cut into as few parts as possible, each continuing part 10 s or longer where the length allows and never under 5 s, and how many parts depends on how long this computer can generate in one run; the parts continue and join automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), + dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise the video is cut into as few parts as possible, each continuing part 10 s or longer where the length allows and never under 5 s, and how many parts depends on how long this computer can generate in one run; the parts continue and join automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt (pass its job id as prompt, {from_job: id}); the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), dict(name='freevideo_generate_image', title='Generate a picture', description='Generate a picture from a prompt; with images, change or combine input pictures; with people, make a first frame with them near the places you say, in the scene from scene_image or the prompt. The result is an input file to use as a first frame, as a reference, or in images again.', annotations=write), dict(name='freevideo_submit_batch', title='Submit a storyboard', description='Submit several jobs at once. Each row is a job object with "kind" (generate default, assemble, vlm, enhance, image) and an optional "ref"; later rows use "@ref" wherever a job id is expected (from_job, after, clips, refine_from). FreeVideo runs them in dependency order.', annotations=write), dict(name='freevideo_job', title='Job status', description='Status, progress, result or error of a job.', annotations=read), @@ -64,7 +64,7 @@ def tool_list(): dict(name='freevideo_look', title='Look at media', description='Return a picture you can see: one frame of a video (at = first, last or seconds) or a contact sheet of timed frames (mode = sheet), or an image file. Use it to review shots.', annotations=read), dict(name='freevideo_edit_image', title='Edit an image', description="Crop/rotate/flip/pad/adjust an image, add text, shapes or image overlays, or remove burned-in captions (remove_text), without the browser. The result is a new input file plus an editor document a person can reopen in FreeVideo's image editor.", annotations=write), dict(name='freevideo_ask', title='Ask the local VLM', description='Ask FreeVideo\'s local vision-language model about images or sampled frames of a video (e.g. "Does this shot match its description? Answer as JSON {score, issues}").', annotations=write), - dict(name='freevideo_enhance_prompt', title='Enhance a prompt', description='Rewrite a rough idea (any language) into a complete H3 prompt with the local VLM. Images become references.', annotations=write), + dict(name='freevideo_enhance_prompt', title='Enhance a prompt', description='Rewrite a rough idea (any language) into a complete H3 prompt with the local VLM. Images become references. To generate from the result, pass the enhance job id as the prompt, {"from_job": id}; do not retype the text.', annotations=write), dict(name='freevideo_assemble', title='Assemble clips', description='Join finished clips (job ids, saved video ids or files) into one video with cuts or crossfades; trims with start/end.', annotations=write), dict(name='freevideo_release_gpu', title='Release the card', description='Free the graphics card and memory the video model holds; call before running another model. released=false while a video runs.', annotations=write), dict(name='freevideo_library', title='Saved videos', description='Saved FreeVideo videos, newest first, with prompts and the agent job that made them.', annotations=read)] diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 0616aab2..c95fcdd1 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -6,6 +6,7 @@ """ import asyncio import base64 +from datetime import datetime import binascii import json import hashlib @@ -27,7 +28,7 @@ from . import spec as specs from .. import tiers as quality_tiers from . import schema as contract, errors, access -from .jobs import ACTIVE, DONE, JobError, Jobs, chain_error, library_view +from .jobs import ACTIVE, DONE, JobError, Jobs, chain_error, library_view, user_stop_notice PREFIX = '/freevideo/agent/v1' from .whitebox import MAX_VIDEO_BYTES as MAX_WHITEBOX_VIDEO @@ -435,6 +436,25 @@ def reply(value, status=200): contracts = contract.contracts() page_sessions = {} connection = dict(connected=False, client=None, via=None, last_seen_at=None) + refused_connects = set() + + def note_connect_refusal(code, request): + # The card only says the address could not be had; the log says why, once per reason and kind of address. + # The kind, never the name: a computer or network name does not belong in a log that may be sent along. + import ipaddress + from urllib.parse import urlsplit + try: + host = urlsplit('//' + request.host).hostname or '' + except ValueError: + host = '' + try: + ipaddress.ip_address(host) + kind = 'IP address' + except ValueError: + kind = 'localhost' if host == 'localhost' else 'name' + if (code, kind) not in refused_connects: + refused_connects.add((code, kind)) + logging.getLogger(__name__).warning('[FreeVideo] Agent connection address refused: %s (opened by %s)', code, kind) # The conversation in the Autopilot view: the page and the connected agent talk in the agent's latest session. chat = dict(client=None, waiting=0, folds={}, told={}) chat_writes = asyncio.Lock() @@ -535,6 +555,8 @@ async def respond(request): return reply({'error': value}, getattr(error, 'status', errors.ERRORS[value['code']][0])) except JobError as error: value = error.view() + if suffix.endswith('/connect'): + note_connect_refusal(error.code, request) if value.get('next') and request.match_info.get('id'): value['next'] = value['next'].replace('{id}', request.match_info['id']) if error.code == 'too_large': @@ -880,6 +902,11 @@ def chat_card(row, fold): estimate = jobs.estimate(row) if estimate: card['progress']['estimate'] = estimate + again = jobs.again(row) + if again: + card['again'] = again + # The user stopped it here: a video card says so, with no report to download. + card.update({k: v for k, v in user_stop_notice(row).items() if k == 'cancelled_by'}) return card def chat_fold(store, sid): @@ -899,7 +926,8 @@ def chat_fold(store, sid): base = dict(seq=row['seq'], at=row['at'], type=row['type']) if row['type'] == 'message': fold['items'].append(dict(base, role='user' if row['by'] == 'user' else 'agent', text=row['text'], - **({'attachments': row['attachments']} if row.get('attachments') else {}))) + **({'attachments': row['attachments']} if row.get('attachments') else {}), + **({'revises': row['revises']} if row.get('revises') else {}))) if row['by'] == 'user': fold['said'] = row['text'] if row.get('mid'): @@ -1097,6 +1125,30 @@ def chat_attachments(values): raise error from None return result + def rewrite_from(store, sid, fold, said): + """The user rewrites their last message: stop what was started after it and decline what it still asks. + + Jobs and films begun before it keep running. Raises when a job cannot be stopped; nothing is said then.""" + since = datetime.fromisoformat(said['at'].replace('Z', '+00:00')).timestamp() + for row in jobs.active(): + # Where the job shows in the conversation; one not shown yet goes by when it was made. + place = fold['jobs'].get(row['id']) + if not (place > said['seq'] if place is not None else row.get('session') == sid and row['created_at'] >= since): + continue + try: + jobs.cancel(row['id'], by='user') + except KeyError: + pass # Gone already. + for item in fold['items']: + if item['seq'] > said['seq'] and item.get('entity') and fold['entities'].get(item['entity']) == 'production': + try: + jobs.productions.pause(item['entity'], by='user') + except JobError: + pass # Finished, stopped or paused already. + for identity, item in list(fold['asked'].items()): + if item['seq'] > said['seq'] and item['state'] == 'asked': + store.append(sid, dict(type='confirm', id=identity, state='declined'), by='user') + @routes.post('/freevideo/agent/chat') @guarded async def chat_say(request): @@ -1116,6 +1168,13 @@ def write(): pictures = chat_attachments(value.get('attachments', [])) sid = sid or store.create(client=client, by='user')['id'] fold = fold or chat_fold(store, sid) + revised = value.get('revises') + if revised is not None: + said = [i for i in fold['items'] if i['type'] == 'message' and i['role'] == 'user'] + if not said or said[-1]['seq'] != revised: + raise specs.SpecError(contract.diagnostic('/revises', 'the seq of the last user message (%s)' % ( + said[-1]['seq'] if said else 'none yet'), revised)) + rewrite_from(store, sid, fold, said[-1]) if store.stopped(sid): # Speaking again continues a conversation the user stopped earlier. store.append(sid, dict(type='control', action='resumed'), by='user') @@ -1124,6 +1183,8 @@ def write(): message['attachments'] = pictures if value.get('whitebox'): message['whitebox'] = True + if revised is not None: + message['revises'] = revised row = store.append(sid, message, by='user') fold['sent'][value['id']] = row['seq'] fold['ids'][row['seq']] = value['id'] @@ -1329,6 +1390,30 @@ async def activity_cancel(request): return problem('not_found', 'No such job', 404) return reply(dict(id=row['id'], status=row['status'])) + @routes.post('/freevideo/agent/activity/resume') + @guarded + async def activity_resume(request): + # The page's Retry: the same job again, keeping a video's finished parts. Only the page may ask. + if request.content_type != 'application/json': + return problem('unsupported_media_type', 'Send JSON', 415) + value = await body(request) + def work(): + row = jobs.run_again(value['id']) + store = work_sessions() + if store is not None and row.get('session'): + try: + # The agent hears it was the user, and does not plan that job again. + store.append(row['session'], dict(type='control', action='retried', refs=dict(job=row['id'])), by='user') + except JobError: + pass # A deleted or full conversation: the job runs again all the same. + return dict(id=row['id'], status=row['status']) + try: + answer = await asyncio.to_thread(work) + except KeyError: + return problem('not_found', 'No such job', 404) + await jobs.tick() + return reply(answer) + @routes.get('/freevideo/agent/connect') @guarded async def connect(request): @@ -1341,6 +1426,7 @@ async def connect(request): root = installation_root() prepare_token(root) except (OSError, ValueError) as error: + note_connect_refusal('not_ready', request) return problem('not_ready', str(error), 503) config = await asyncio.to_thread(client_config, root) if os.name != 'nt' and not linux_wrapper_supported(): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index ca82a58d..57fa1ab4 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -82,7 +82,7 @@ def chat_request(): # Upload metadata supplied by the page is ignored; only the stored name is authoritative. attachment = obj(dict(name=CHAT_ATTACHMENT_NAME, original_name={}, width={}, height={}, bytes={}), ('name',)) return obj(dict(text=dict(type='string', minLength=0, maxLength=8000), id=CHAT_ID, - session=SESSION, whitebox={'type': 'boolean'}, + session=SESSION, whitebox={'type': 'boolean'}, revises=number(1, integer=True), attachments=array(attachment, 0, MAX_CHAT_ATTACHMENTS)), ('text', 'id'), **{'if': {'required': ['attachments'], 'properties': {'attachments': {'minItems': 1}}}, 'else': {'properties': {'text': string(8000)}}}) @@ -491,7 +491,8 @@ def route(summary, body=None, query=None, **kw): 'Only the user may stop or continue a film (local Studio control)', body=obj({'ref': REF}), path=obj({'id': PRODUCTION}, ('id',)), auth=False, local=True, idempotent=True) for suffix, method, body in (('', 'GET', None), ('/session', 'GET', None), ('/activity', 'GET', None), - ('/activity/cancel', 'POST', obj({'id': JOB}, ('id',))), ('/connect', 'GET', None), + ('/activity/cancel', 'POST', obj({'id': JOB}, ('id',))), ('/activity/resume', 'POST', obj({'id': JOB}, ('id',))), + ('/connect', 'GET', None), ('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None), ('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY), ('/chat/delete', 'POST', obj({'session': SESSION, 'stop': {'type': 'boolean'}}, ('session',))), @@ -502,7 +503,7 @@ def route(summary, body=None, query=None, **kw): 'session': SESSION}, ('id', 'approve')))): routes[(method, '/freevideo/agent' + suffix)] = route('Studio Agent controls', body=body, auth=False, local=True, lan=suffix in ('/chat', '/chat/now', '/chat/confirm', '/chat/abandon', '/chat/list', '/chat/new', '/chat/video', - '/activity', '/activity/cancel')) + '/activity', '/activity/cancel', '/activity/resume')) # These existing ComfyUI handlers are registered outside the Agent dispatcher. # /chain is a creation POST; monitoring reads /chains and /chain/{id}. for suffix in ('/freevideo/progress', '/freevideo/chains', '/freevideo/chain/{id}'): @@ -908,6 +909,9 @@ def session_event(*, stored=False, agent_only=False): variants[0]['properties']['mid'] = CHAT_ID variants[0]['properties']['attachments'] = array(CHAT_ATTACHMENT, 1, MAX_CHAT_ATTACHMENTS) variants[0]['properties']['whitebox'] = {'type': 'boolean', 'description': 'The user chose a white-model preview for this request.'} + variants[0]['properties']['revises'] = dict(number(1, integer=True), description=( + 'The user rewrote their message with this seq: this one replaces it and what was written after it. ' + 'The jobs started for it are stopped.')) from .errors import ERRORS variants += [obj(dict(type={'const': 'step'}, id=string(12, pattern=r'^st[0-9a-f]{10}$'), tool=string(200), summary=text, state=enum(('started', 'done', 'failed')), @@ -923,8 +927,10 @@ def session_event(*, stored=False, agent_only=False): digest=string(64, pattern=r'^[0-9a-f]{64}$')), ('type', 'id', 'state'), **{'if': {'properties': {'state': {'const': 'asked'}}}, 'then': {'required': ['question']}}), - obj(dict(type={'const': 'control'}, action=enum(('stopped', 'resumed', 'renamed', 'abandoned')), - title=dict(type='string', maxLength=120)), ('type', 'action'), + obj(dict(type={'const': 'control'}, action=enum(('stopped', 'resumed', 'renamed', 'abandoned', 'retried')), + title=dict(type='string', maxLength=120), + refs=obj(dict(job=JOB), ('job',), description='retried: the job the user ran again on the page.')), + ('type', 'action'), **{'if': {'properties': {'action': {'const': 'renamed'}}}, 'then': {'required': ['title']}, 'else': {'not': {'required': ['title']}}})] if stored: diff --git a/freevideo_engine/agent/sessions.py b/freevideo_engine/agent/sessions.py index a48a481a..ec5eca3b 100644 --- a/freevideo_engine/agent/sessions.py +++ b/freevideo_engine/agent/sessions.py @@ -300,6 +300,8 @@ def append(self, sid, event: dict, *, by, client=None) -> dict: raise SessionError('forbidden', 'Only the user may answer a confirmation') if kind == 'control' and event['action'] != 'renamed' and by != 'user': raise SessionError('forbidden', 'Only the user may stop or continue') + if kind == 'message' and 'revises' in event and by != 'user': + raise SessionError('forbidden', 'Only the user may rewrite a message') result = dict(event, seq=meta['last_seq'] + 1, at=_stamp(self.clock()), by=by) if client is not None: result['client'] = client @@ -516,7 +518,7 @@ def unread(self, sid, reader: str) -> list[dict]: rows = self._events(sid, after, self.max_events) return [row for row in rows if row['by'] == 'user' and (row['type'] == 'message' or row['type'] == 'confirm' and row['state'] in ('approved', 'declined') - or row['type'] == 'control' and row['action'] == 'abandoned')] + or row['type'] == 'control' and row['action'] in ('abandoned', 'retried'))] class Confirmations: diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index c238093f..368106c4 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -372,6 +372,8 @@ def inventory(gpu=None): # Windows reuses a detected Git from 2.35, the oldest tested there, and # otherwise installs its own MinGit. WINDOWS_MIN_GIT = (2, 35) +# Seconds to wait before creating an environment again after another program held a file. +ENVIRONMENT_BUSY_WAITS = (3, 10, 30) def git_floor(system=None): @@ -1599,23 +1601,31 @@ def task(name, run, after=(), writes=()): def environment(folder=venv, executable=python, label=name): if executable.exists() and environment_seeded(folder): return None - command = [uv, 'venv', '--seed', '--python', self.versions['python']] - # The Python file came back (restored from quarantine) without - # the seed packages: keep it and add them. Without the file, the - # folder is a leftover that uv replaces. - if executable.exists(): - command.append('--allow-existing') - elif folder.exists(): - command.append('--clear') - try: - return self.packages(label + '-venv', [*command, folder]) - except RuntimeError as error: - from .failure_details import environment_python_removed - # uv failed while the new interpreter was disappearing. - removed = '' if executable.exists() else environment_python_removed([executable], [executable]) - if not removed: - raise - raise RuntimeError(removed + str(error)) from error + from .failure_details import environment_python_removed, file_in_use + for wait in (*ENVIRONMENT_BUSY_WAITS, None): + command = [uv, 'venv', '--seed', '--python', self.versions['python']] + # The Python file came back (restored from quarantine) without + # the seed packages: keep it and add them. Without the file, the + # folder is a leftover that uv replaces. + if executable.exists(): + command.append('--allow-existing') + elif folder.exists(): + command.append('--clear') + try: + return self.packages(label + '-venv', [*command, folder]) + except RuntimeError as error: + # Another program (usually security software scanning the + # Python file just unpacked) held it open while uv copied it. + if wait is not None and file_in_use(error) and not self.cancel.is_set(): + self.ui.write('A file is in use by another program; trying again in %d s.\n' % wait) + if not self.cancel.wait(wait): + continue + raise + # uv failed while the new interpreter was disappearing. + removed = '' if executable.exists() else environment_python_removed([executable], [executable]) + if not removed: + raise + raise RuntimeError(removed + str(error)) from error task(name + '-venv', environment, ('python',), writer) command = [uv, 'pip', 'install', '--python', python, *spec['torch'], '-c', constraints_file(name, self.system)] family = 'torch-' + spec['cuda'] + '-' + spec['torch'][0].split('==')[1] diff --git a/freevideo_engine/chain_join.py b/freevideo_engine/chain_join.py index e5a7e659..b7e4b1fd 100644 --- a/freevideo_engine/chain_join.py +++ b/freevideo_engine/chain_join.py @@ -195,6 +195,67 @@ def _join_ending(sources, target, cancelled, frames, leads, expected, speaking, speaking=speaking, original_audio=original_audio) +def _difference(number, shape, other): + """Why part `number` cannot join part 1, in words.""" + if shape[1:3] != other[1:3]: + return f'Part {number} is {shape[1]}x{shape[2]} but part 1 is {other[1]}x{other[2]}' + for name, index in (('codec', 0), ('pixel format', 3)): + if shape[index] != other[index]: + return f'Part {number} uses {shape[index]} {name} but part 1 uses {other[index]}' + if shape[4:6] != other[4:6]: + return f'Part {number} does not have the frame rate of part 1' + return f'Part {number} was encoded differently from part 1' + + +def _conform(source, shape, directory, cancelled): + """A copy of `source` with the first block's picture settings, or None when it cannot have them. + + The picture is encoded again with the encoder's defaults; the sound is copied packet by packet. + """ + import av + from .comfy_metadata import _copy_stream + temporary = directory / ('.join-conform-' + uuid.uuid4().hex + '.mp4') + try: + with av.open(str(source)) as container, av.open(str(temporary), 'w', format='mp4') as output: + video = container.streams.video[0] + if abs(float(video.average_rate or 0) - FPS) > .01: + return None + picture = output.add_stream('libx264', rate=FPS) + picture.width, picture.height, picture.pix_fmt = shape[1:4] + picture.time_base = shape[4] + sound = container.streams.audio[0] if container.streams.audio else None + copy = _copy_stream(output, sound) if sound is not None else None + number = 0 + for packet in container.demux(video, *([sound] if sound is not None else [])): + _check(cancelled) + if packet.stream is sound: + if packet.dts is not None: + packet.stream = copy + output.mux(packet) + continue + for frame in packet.decode(): + frame = frame.reformat(width=shape[1], height=shape[2], format=shape[3]) + frame.pts, frame.time_base = number, Fraction(1, FPS) + frame.pict_type = av.video.frame.PictureType.NONE + for encoded in picture.encode(frame): + output.mux(encoded) + number += 1 + for encoded in picture.encode(None): + output.mux(encoded) + with av.open(str(temporary)) as check: + if not number or _shape(check.streams.video[0]) != shape: + temporary.unlink(missing_ok=True) + return None + return temporary + except JoinCancelled: + temporary.unlink(missing_ok=True) + raise + except Exception: + temporary.unlink(missing_ok=True) + logging.warning('[FreeVideo] A part could not be matched to the first part', exc_info=True) + return None + + def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None, original_audio=None): import av @@ -223,6 +284,7 @@ def audio_lead_in(video, previous=None): wanted, frames, samples = frames, 0, 0 used_tail = False blocks_joined = 0 + conformed = [] try: with av.open(str(sources[0])) as first: if len(first.streams.video) != 1: @@ -236,6 +298,27 @@ def audio_lead_in(video, previous=None): if shape[5] != FPS or step.denominator != 1: raise ValueError('The blocks are not 24 frames per second videos') step = int(step) + originals = {} + for number, source in enumerate(sources[1:], 2): + with av.open(str(source)) as container: + if len(container.streams.video) != 1 or bool(container.streams.audio) != (sound is not None): + continue + other = _shape(container.streams.video[0]) + if other == shape: + continue + copy = _conform(source, shape, target.parent, cancelled) + if copy is None: + raise ValueError(_difference(number, other, shape)) + conformed.append(copy) + originals[str(copy)] = str(source) + sources[number - 1] = copy + if originals: + # Seam sound records belong to the blocks as made, not to the matched copies. + lead_reader = audio_lead_in + if lead_reader is not None: + def audio_lead_in(video, previous=None): + return lead_reader(originals.get(str(video), video), + originals.get(str(previous), previous) if previous is not None else None) voice = _voice(sources, speaking, cancelled) if sound is not None else None with av.open(str(temporary), 'w', format='mp4', options={'movflags': 'use_metadata_tags+faststart'}) as output: @@ -367,6 +450,8 @@ def audio_lead_in(video, previous=None): raise finally: temporary.unlink(missing_ok=True) + for copy in conformed: + copy.unlink(missing_ok=True) def _voice(sources, speaking, cancelled): diff --git a/freevideo_engine/chain_plan.py b/freevideo_engine/chain_plan.py index dfbc0281..8e87173d 100644 --- a/freevideo_engine/chain_plan.py +++ b/freevideo_engine/chain_plan.py @@ -24,6 +24,8 @@ MAX_PARTS, SHORT_MAX_PARTS = RULES['max_parts'], RULES['short_max_parts'] # The length a new continuation starts with, and its length on devices that cannot continue with a full part. CONTINUE_SECONDS, SHORT_CONTINUE_SECONDS = RULES['continue_seconds'], RULES['short_continue_seconds'] +# The last part of a video, and nothing that another part continues, may be as short as MIN_TAIL_SECONDS. +MIN_TAIL_SECONDS = RULES['min_tail_seconds'] def part_frames(seconds): @@ -35,11 +37,26 @@ def part_frames(seconds): # Continuing blocks below MIN_PART_FRAMES are too short; below FULL_PART_FRAMES they are short parts. MIN_PART_FRAMES, FULL_PART_FRAMES = part_frames(MIN_PART_SECONDS), part_frames(FULL_PART_SECONDS) +MIN_TAIL_FRAMES = part_frames(MIN_TAIL_SECONDS) SHORTEST_FIRST = 2 # whole seconds: what the first part keeps when the continuing parts take the rest MAX_SECONDS = 7200 MAX_BLOCKS = MAX_SECONDS // SHORTEST_BLOCK +def _tail_lengths(total, following): + """Lengths of a video that continues a saved one when no even split gives every part MIN_PART_SECONDS: parts as + long as the device allows, and a last part of MIN_TAIL_SECONDS or more for the rest. None if no such split.""" + count = math.ceil(round(total, 6) / following) + if count > MAX_BLOCKS or total < MIN_TAIL_SECONDS + MIN_PART_SECONDS * (count - 1): + return None + lengths = [following] * (count - 1) + lengths.append(round(total - sum(lengths), 6)) + if lengths[-1] < MIN_TAIL_SECONDS: # take what the last part lacks from the one before it + lengths[-2] = round(lengths[-2] - (MIN_TAIL_SECONDS - lengths[-1]), 6) + lengths[-1] = MIN_TAIL_SECONDS + return lengths if all(length >= MIN_PART_SECONDS for length in lengths[:-1]) else None + + def parts_limit(short, first_frame=False, max_parts=MAX_PARTS): """The most parts one video may have: fewer when one of its continuing parts is short, unless every continuing part also sees the first frame.""" @@ -154,6 +171,9 @@ def plan(seconds, single=False, max_frames=None, single_limit_seconds=None, bloc if lengths is None: if fits and seconds < SINGLE_SECONDS: return alone # a requested block length never makes a part this device cannot do + if continued and following >= MIN_PART_SECONDS: + lengths = _tail_lengths(total, following) # the last part may be shorter than the others + if lengths is None: if first < (MIN_PART_SECONDS if continued else SHORTEST_BLOCK) or following < MIN_PART_SECONDS: raise PlanError('device_limit', 'This device cannot generate this length at this size') raise PlanError('length_not_splittable', 'This length cannot be cut into parts of the requested length') diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 7011b013..1dd72393 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -69,9 +69,9 @@ class ChainPartTooShort(ValueError): code = 'chain_part_too_short' def __init__(self, seconds, limit): - super().__init__('Each continued part must be at least %d seconds. Make it longer.' % limit) + super().__init__('A part followed by another part must be at least %d seconds. Make it longer.' % limit) self.seconds, self.limit = seconds, limit - self.message_zh = '接着生成的每一段至少 %d 秒，请加长这一段。' % limit + self.message_zh = '后面还接着其他段时，这一段至少 %d 秒，请加长这一段。' % limit def _work_key(path): @@ -121,6 +121,8 @@ def __init__(self, store_dir, host, clock=time.monotonic, wall=time.time, estima self.step_lock = threading.Lock() self.wake = threading.Event() self.closed = threading.Event() + # The listed records stay bounded; which receipt made a saved video is indexed on first use, over every receipt on disk. + self.work_receipts = None self.records = {row['id']: row for row in store.load_all(store_dir)} chain_leads.prune(store_dir, self.records) self.starts, self.success, self.cancel_sent, self.progress_values = {}, {}, set(), {} @@ -147,6 +149,25 @@ def __init__(self, store_dir, host, clock=time.monotonic, wall=time.time, estima def _save(self, record): record['updated'] = self.wall() store.save(self.store_dir, record) + self._index_receipt(record, latest=True) + + def _receipts(self): + if self.work_receipts is None: + self.work_receipts = {} + store.load_all(self.store_dir, visitor=self._index_receipt) # newest first: an older receipt never replaces one + return self.work_receipts + + def _index_receipt(self, record, latest=False): + if self.work_receipts is None: + return # the first lookup reads every receipt, this one included + output = record.get('output') + if output: + target = self.work_receipts.__setitem__ if latest else self.work_receipts.setdefault + target(_work_key(output), (record['id'], 'output')) + for block in record.get('blocks', ()): + if block.get('output'): + target = self.work_receipts.__setitem__ if latest else self.work_receipts.setdefault + target(_work_key(block['output']), (record['id'], 'block')) def _current(self, record): executing = next((row for row in record['blocks'] if row['state'] in ('queued', 'running')), None) @@ -303,24 +324,54 @@ def _work_parts(self, work, seen): def _work_lineage(self, work, seen): key = _work_key(work) - for record in list(self.records.values()): - if record['id'] in seen: - continue - if record.get('output') and _work_key(record['output']) == key: + receipt = self._receipts().get(key) + if receipt and receipt[0] not in seen: + identity, kind = receipt + record = self.records.get(identity) + outside = record is None + if outside: + # A receipt older than the listed ones is only read, never listed or protected as a recent chain. + rows = store.load_all(self.store_dir, identity=identity) + if not rows: + return 1, False + record = rows[0] + if kind == 'output' and record.get('output') and _work_key(record['output']) == key: + if 'output_parts' in record and 'output_short' in record: + return record['output_parts'], record['output_short'] names = [row['id'] for row in store.leading_blocks(record)[:record.get('output_blocks') or None]] else: - names = [row['id'] for row in record['blocks'] if row['output'] and _work_key(row['output']) == key] + names = [row['id'] for row in record['blocks'] if row.get('output') and _work_key(row['output']) == key] if names: - lineage = self._lineage(record, seen) + try: + lineage = self._lineage(record, seen) + except (KeyError, ValueError): + if not outside: + raise + return 1, False # its own sources are older still and cannot be read: counted as before return max(lineage[name][0] for name in names), any(lineage[name][1] for name in names) return 1, False + def work_lineage(self, work): + """Return (parts, short, known) for an output-relative saved work path.""" + if not isinstance(work, str) or not work or '\x00' in work: + raise ValueError('A work path is required') + key = _work_key(work) + with self.lock: + if key not in self._receipts(): + return 1, False, False + parts, short = self._work_lineage(work, ()) + return parts, short, True + def _check_parts(self, record): """Refuses, before anything is written or queued, a chain with a continuing part too short or a video continued through too many parts.""" + # The last part of a video, one no other block continues, may be shorter than the parts before it. + followed = {row['after'] for row in record['blocks'] if isinstance(row['after'], str)} short = [row['inputs']['frames'] for row in record['blocks'] if (row['after'] is not None or row['before'] is not None) - and row['inputs']['frames'] < self.min_part_frames] + and row['inputs']['frames'] < (min(self.min_part_frames, chain_plan.MIN_TAIL_FRAMES) + if row['before'] is None and row['id'] not in followed + else self.min_part_frames)] if short: raise ChainPartTooShort(min(short) / chain_plan.FPS, chain_plan.MIN_PART_SECONDS) lineage = self._lineage(record).values() @@ -669,7 +720,9 @@ def _finish_blocks(self, record, final_state='done'): return final_state if len(blocks) == 1 and record.get('original_audio') is None: output = blocks[0]['output'] - record.update(output=output, output_blocks=1 if output else 0, output_state='done', output_error=None) + parts, short = self._lineage(record)[blocks[0]['id']] + record.update(output=output, output_blocks=1 if output else 0, output_state='done', output_error=None, + output_parts=parts, output_short=short) return final_state if record['output_state'] == 'done' and record['output_blocks'] == len(blocks): return final_state @@ -966,8 +1019,13 @@ def lead_reader(following, previous): record.update(state=final_state, output_state='failed', output_error=failure) logging.warning('[FreeVideo] Chain joining failed: %s', identity) else: - record.update(state=final_state, output=output, output_blocks=len(store.leading_blocks(snapshot)), - output_state='done', output_error=None, output_seams=dict(expected=promised, kept=kept)) + # Only the blocks the joined video is made of count, as `_work_lineage` counts them for a record without these fields. + lineage, leading = self._lineage(snapshot), store.leading_blocks(snapshot) + output_parts = max((lineage[row['id']][0] for row in leading), default=1) + output_short = any(lineage[row['id']][1] for row in leading) + record.update(state=final_state, output=output, output_blocks=len(leading), + output_state='done', output_error=None, output_seams=dict(expected=promised, kept=kept), + output_parts=output_parts, output_short=output_short) if record['state'] not in store.ACTIVE: self._release_active(identity) self._save(record) diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py index 6177fc86..42d1028c 100644 --- a/freevideo_engine/chain_store.py +++ b/freevideo_engine/chain_store.py @@ -254,7 +254,7 @@ def save(directory, record): temporary.unlink(missing_ok=True) -def load_all(directory, limit=200, *, identity=None): +def load_all(directory, limit=200, *, identity=None, visitor=None): if identity is not None and (not isinstance(identity, str) or not re.fullmatch(r'c-[0-9a-f]{12}', identity)): return [] records = [] @@ -265,7 +265,9 @@ def load_all(directory, limit=200, *, identity=None): except OSError: logging.warning('[FreeVideo] Skipping inaccessible chain file: %s', path.name) paths.sort(key=lambda row: (row[0], row[1].name), reverse=True) - for _, path in paths[:limit]: + for position, (_, path) in enumerate(paths): + if limit is not None and position >= limit and visitor is None: + break try: record = json.loads(path.read_text(encoding='utf-8')) if not isinstance(record, dict) or record.get('version') != 1: @@ -283,6 +285,11 @@ def load_all(directory, limit=200, *, identity=None): if (type(record['output_blocks']) is not int or not 0 <= record['output_blocks'] <= len(record['blocks']) or record['output'] is None and record['output_blocks'] != 0): raise ValueError('Invalid chain output block count') + if ('output_parts' in record) != ('output_short' in record): + raise ValueError('Invalid output lineage') + if 'output_parts' in record and (type(record['output_parts']) is not int or record['output_parts'] < 1 + or type(record['output_short']) is not bool): + raise ValueError('Invalid output lineage') split = record.setdefault('split', None) if split is not None and (not isinstance(split, dict) or set(split) != {'state', 'reason'} or split['state'] not in SPLIT_STATES @@ -361,7 +368,10 @@ def load_all(directory, limit=200, *, identity=None): for key in ('created', 'updated'): if type(record[key]) not in (int, float) or not math.isfinite(record[key]): raise ValueError('Invalid chain timestamp') - records.append(record) + if visitor is not None: + visitor(record) + if limit is None or position < limit: + records.append(record) except (OSError, ValueError, TypeError, KeyError, OverflowError): # Keep the file for inspection; never log its workflow or sentences. logging.warning('[FreeVideo] Skipping invalid chain file: %s', path.name) diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 4628d13e..515b9341 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -85,8 +85,9 @@ def resolve_assets(serialized, input_directory): or media_kind(row['file']) != 'video'): raise ValueError('depth is only allowed on video references') # The character an @ mention added the row for; a continued part keeps one picture per character. - if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) != {'id'} - or not isinstance(row['cast']['id'], str) or not 0 < len(row['cast']['id']) <= 128): + if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) not in ({'id'}, {'id', 'setting'}) + or not isinstance(row['cast']['id'], str) or not 0 < len(row['cast']['id']) <= 128 + or row['cast'].get('setting', True) is not True): raise ValueError('Invalid media item') if type(row.get('enabled', True)) is not bool: raise ValueError('Media enabled must be a boolean') @@ -119,6 +120,8 @@ def resolve_assets(serialized, input_directory): reference.update(depth=True, early_steps=3) if cast_id(row) is not None: reference['cast'] = cast_id(row) # only a continued part reads it (comfy_media.continued) + if is_setting(row): + reference['scene'] = True if 'early_steps' in row: from .media_request import validate_early_steps validate_early_steps(dict(row, kind=kind)) @@ -172,7 +175,7 @@ def continued_assets(serialized): image = False if image: pictures.append(row) - return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)] + original) + return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id, is_setting)] + original) def cast_id(row): @@ -182,10 +185,16 @@ def cast_id(row): return value if isinstance(value, (str, int)) and not isinstance(value, bool) else None +def is_setting(row): + """Whether a panel row carries the picture of a setting card (an @ mention of a scene).""" + cast = row.get('cast') if isinstance(row, dict) else None + return cast_id(row) is not None and cast.get('setting') is True + + def _kept_row(row): kept = dict(file=row['file'], role='reference') if cast_id(row) is not None: - kept['cast'] = dict(id=cast_id(row)) + kept['cast'] = dict(id=cast_id(row), **({'setting': True} if is_setting(row) else {})) return kept diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 210fcc53..5d15cfbc 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -413,6 +413,24 @@ async def plan(request): except ValueError as failure: return response({'error': str(failure), 'code': 'invalid'}, 400) + @server.routes.post('/freevideo/chain/lineage') + async def lineage(request): + try: + value = await body(request) + works = value.get('works') + if set(value) != {'works'} or not isinstance(works, list) or len(works) > 16: + raise ValueError('works must be an array of at most 16 work references') + result = [] + for work in works: + if not isinstance(work, dict) or set(work) != {'work'} or not isinstance(work['work'], str): + raise ValueError('Each work reference must contain a work path') + chain_store.reference(work) + parts, short, known = runner.work_lineage(work['work']) + result.append(dict(parts=parts, short=short, known=known)) + return response({'lineage': result}) + except ValueError as failure: + return response({'error': str(failure)}, 400) + @server.routes.post('/freevideo/chain') async def create(request): try: diff --git a/freevideo_engine/comfy_host_stats.py b/freevideo_engine/comfy_host_stats.py new file mode 100644 index 00000000..b821c92d --- /dev/null +++ b/freevideo_engine/comfy_host_stats.py @@ -0,0 +1,100 @@ +"""Two host facts for the diagnostic report: booleans and integers only, no names or paths.""" +import json +import os +from pathlib import Path +import time + +from .comfy_source import RECEIPT + +COUNTER = 'comfy-host-stats.json' +DAYS = 30 +OURS = ('FreeVideo', 'FreeVideoExecutor') +# Files ComfyUI itself ships in custom_nodes. +UPSTREAM = ('websocket_image_save.py',) + + +def counter_path(root=None): + if root is None: + from .comfy_bridge import installation_root + root = installation_root() + return Path(root) / COUNTER + + +def _day(now): + return time.strftime('%Y-%m-%d', time.gmtime(now)) + + +def record_node_view(root=None, now=None): + """Count one opening of the node canvas on today's date; days older than the window are dropped.""" + now = time.time() if now is None else now + path = counter_path(root) + try: + days = json.loads(path.read_text(encoding='utf-8')).get('days', {}) + except (OSError, ValueError, AttributeError): + days = {} + if not isinstance(days, dict): + days = {} + cutoff = _day(now - DAYS * 86400) + days = {k: v for k, v in days.items() if isinstance(v, int) and not isinstance(v, bool) and k > cutoff} + today = _day(now) + days[today] = days.get(today, 0) + 1 + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + '.%d.tmp' % os.getpid()) + temporary.write_text(json.dumps(dict(days=days)), encoding='utf-8') + os.replace(temporary, path) + + +def node_view_opens(root=None, now=None): + now = time.time() if now is None else now + try: + days = json.loads(counter_path(root).read_text(encoding='utf-8')).get('days', {}) + cutoff = _day(now - DAYS * 86400) + return sum(v for k, v in days.items() if k > cutoff and isinstance(v, int) and not isinstance(v, bool) and v > 0) + except (OSError, ValueError, AttributeError, TypeError): + return 0 + + +def third_party_packs(folders): + names = set() + for folder in folders: + try: + entries = list(Path(folder).iterdir()) + except OSError: + continue + for entry in entries: + name = entry.name + if name in OURS or name in UPSTREAM or name.lower().startswith('freevideo-') or name.startswith(('.', '__')) or name.endswith('.disabled'): + continue + if entry.is_dir() or name.endswith('.py'): + names.add(str(entry.resolve())) + return len(names) + + +def _managed(comfy_root, install_root): + comfy_root = Path(comfy_root).resolve() + if (comfy_root / RECEIPT).is_file(): + return True + try: + # The portable package and the launcher keep their ComfyUI inside the FreeVideo folder. + comfy_root.relative_to(Path(install_root).resolve()) + return True + except ValueError: + return False + + +def host_fields(comfy_root=None, node_folders=None, root=None, now=None): + """own_comfyui: ComfyUI is not the copy FreeVideo installed, or it has third-party node packs.""" + try: + if comfy_root is None: + import folder_paths + comfy_root = folder_paths.base_path + node_folders = folder_paths.get_folder_paths('custom_nodes') + except (ImportError, AttributeError, OSError): + return None + if root is None: + from .comfy_bridge import installation_root + root = installation_root() + packs = third_party_packs(node_folders if node_folders is not None else [Path(comfy_root) / 'custom_nodes']) + return dict(own_comfyui=bool(packs) or not _managed(comfy_root, root), + third_party_node_packs=packs, + node_view_opens_30d=node_view_opens(root, now)) diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.py index 5b396861..7dccb659 100644 --- a/freevideo_engine/comfy_media.py +++ b/freevideo_engine/comfy_media.py @@ -60,7 +60,8 @@ def continued(exported, notes=None): original = [row for row in media.get('references', []) if row.get('kind') == 'audio' and row.get('use') == 'original'] keyframes = [name for name in ('first', 'last') if media.get(name)] # Characters added by an @ mention keep one picture each when there are several (pick_pictures). - chosen = pick_pictures(pictures, lambda row: row.get('cast')) + chosen = pick_pictures(pictures, lambda row: None if row.get('scene') else row.get('cast'), + lambda row: row.get('scene') is True) candidates = [_plain(row) for row in chosen] + [{'kind': 'image', 'path': media[name]} for name in keyframes] if candidates or original: kept['references'] = candidates[:PICTURES_MAX] + original @@ -77,8 +78,8 @@ def continued(exported, notes=None): def _plain(row): - """A reference row without the panel's character mark, which the engine's request does not carry.""" - return {key: value for key, value in row.items() if key != 'cast'} + """A reference row without the panel's character or scene mark, which the engine's request does not carry.""" + return {key: value for key, value in row.items() if key not in ('cast', 'scene')} def continued_note(pictures, keyframes, used): diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index 05999400..fd6174d4 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -433,6 +433,15 @@ async def progress(request): return web.json_response({'progress': STATE.snapshot()}, headers={'Cache-Control': 'no-store'}) + @server.routes.post('/freevideo/node-view-opened') + async def node_view_opened(request): + from .comfy_host_stats import record_node_view + try: + await asyncio.to_thread(record_node_view) + except (OSError, ValueError): + pass # The count is only a diagnostic aid. + return web.Response(status=204) + @server.routes.post('/freevideo/cancel') async def cancel(request): value = await request.json() diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py index f803fac1..c268e918 100644 --- a/freevideo_engine/continuation.py +++ b/freevideo_engine/continuation.py @@ -42,22 +42,35 @@ PICTURES_MAX = 3 -def pick_pictures(rows, person): +def pick_pictures(rows, person, scene=None): """The pictures a continued part keeps from image rows in panel order: at most PICTURES_MAX. person(row) is the id of the character a row belongs to, or None. With two or more characters each keeps only the last of its rows (the face close-up, or the one picture its card has) so that everyone is seen; the - other pictures follow. With none or one character the first rows are kept.""" + other pictures follow. With none or one character the first rows are kept. + scene(row) is true for the picture of a setting card: the last such row keeps one place, after the others, so the + part still shows the same place.""" rows = list(rows) + place = None + if scene is not None: + places = [row for row in rows if scene(row)] + if places: + place = places[-1] + rows = [row for row in rows if not scene(row)] last = {} for index, row in enumerate(rows): if person(row) is not None: last[person(row)] = index + room = PICTURES_MAX - (place is not None) if len(last) < 2: - return rows[:PICTURES_MAX] - faces = [rows[index] for index in sorted(last.values())] - others = [row for row in rows if person(row) is None] - return (faces + others)[:PICTURES_MAX] + chosen = rows[:room] + else: + faces = [rows[index] for index in sorted(last.values())] + others = [row for row in rows if person(row) is None] + chosen = (faces + others)[:room] + return chosen + ([place] if place is not None else []) + + INVALID_FRAMES = 'Frame count must be 5 + 17n and at least 39 (39, 56, 73, 90, ...).' INVALID_FILE = 'Choose an existing MP4 video using its absolute path.' diff --git a/freevideo_engine/editor_audio.py b/freevideo_engine/editor_audio.py index d65a2af4..ab605d8f 100644 --- a/freevideo_engine/editor_audio.py +++ b/freevideo_engine/editor_audio.py @@ -18,6 +18,17 @@ def clip_fields(raw, kind): or value == raw['id'] or raw.get('link')): raise ValueError('Invalid detached audio') result['detached_from'] = value + # A music piece's preset (prefilled when it is made again) and whether it is lowered under the first sound track. + if 'music' in raw: + value = raw['music'] + if (kind != 'audio' or not isinstance(value, dict) or set(value) != {'style', 'mood'} + or not all(isinstance(value[k], str) and re.fullmatch(r'[a-z]{1,20}', value[k]) for k in ('style', 'mood'))): + raise ValueError('Invalid music preset') + result['music'] = {'style': value['style'], 'mood': value['mood']} + if 'duck' in raw: + if kind != 'audio' or type(raw['duck']) is not bool: + raise ValueError('Invalid audio ducking') + result['duck'] = raw['duck'] return result @@ -157,3 +168,82 @@ def gain(positions): added[:, a - block_start:b - block_start] += (data[:, offset + a - begin:offset + b - begin] * gain(np.arange(a, b))[None, :] * clip.get('volume', 1)) return output + added + + +# Lower during speech: a clip with duck=True sits DUCK_DB lower wherever another audible clip has speech, +# with DUCK_RAMP_SECONDS ramps just outside each stretch. web/editor_audio_model.js has the same rules. +DUCK_DB = -8. +DUCK_RAMP_SECONDS = .3 + + +def _speed(clip): + return clip.get('speed', 1) if changed(clip) else 1 + + +def duck_ranges(tracks, spans, rate=48000, fps=24): + """Timeline sample ranges [first, last) with speech in a clip that is not itself lowered. + + spans maps 'kind:id' source keys to [start, end] seconds of that source; a source without an entry + has no known speech. Ranges closer than two ramps are joined so the level does not bounce between them. + """ + found = [] + for track in tracks: + if track.get('kind') != 'audio' or not track_audible(track, tracks): + continue + for clip in track['clips']: + source = clip.get('source') + if not source or clip.get('duck') or clip.get('volume', 1) == 0 or not clip_audible(clip, track, tracks): + continue + stretches = spans.get(source['kind'] + ':' + source['id']) or () + first, last = _samples(clip['start'], rate, fps), _samples(clip['start'] + clip['frames'], rate, fps) + origin = (source_sample_start(clip) * rate / 48000 if audio_changed(clip) + else _source_start(clip, rate, fps)) + speed = _speed(clip) + for begin, end in stretches: + a, b = (begin * rate - origin) / speed, (end * rate - origin) / speed + if clip.get('reverse'): + a, b = (last - first) - b, (last - first) - a + a, b = max(first, math.floor(first + a + .5)), min(last, math.floor(first + b + .5)) + if b > a: + found.append([a, b]) + found.sort() + ramp = 2 * max(1, math.floor(DUCK_RAMP_SECONDS * rate + .5)) + joined = [] + for a, b in found: + if joined and a <= joined[-1][1] + ramp: + joined[-1][1] = max(joined[-1][1], b) + else: + joined.append([a, b]) + return joined + + +def duck_gain(ranges, block_start, count, rate=48000): + """Gain per sample for a lowered clip over [block_start, block_start + count).""" + import numpy as np + gain = np.ones(count, dtype=np.float64) + ramp = max(1, math.floor(DUCK_RAMP_SECONDS * rate + .5)) + low = 10. ** (DUCK_DB / 20.) + position = np.arange(block_start, block_start + count, dtype=np.float64) + for first, last in ranges: + if last + ramp <= block_start or first - ramp >= block_start + count: + continue + into = np.clip((position - (first - ramp)) / ramp, 0, 1) + out_of = np.clip(((last + ramp) - position) / ramp, 0, 1) + np.minimum(gain, 1 - (1 - low) * np.minimum(into, out_of), out=gain) + return gain + + +def duck_sources(tracks): + """Source keys whose speech spans a lowered clip needs (none when nothing is lowered).""" + if not any(clip.get('duck') and clip.get('source') for track in tracks for clip in track['clips']): + return [] + keys = [] + for track in tracks: + if track.get('kind') != 'audio': + continue + for clip in track['clips']: + source = clip.get('source') + key = source and source['kind'] + ':' + source['id'] + if key and not clip.get('duck') and key not in keys: + keys.append(key) + return keys diff --git a/freevideo_engine/editor_export.py b/freevideo_engine/editor_export.py index 671ddf29..01a0659c 100644 --- a/freevideo_engine/editor_export.py +++ b/freevideo_engine/editor_export.py @@ -28,7 +28,8 @@ from .editor_text import is_text_track, has_text_data, validate_text_project, paint_project, check_export_text, MissingGlyphs from . import editor_export_options from .editor_speed import validate_clip, changed as speed_changed, audio_changed, independent_join, VideoReader, AudioReader -from .editor_audio import clip_fields, track_fields, clip_audible, track_audible, crossfade_reads, apply_track_audio +from .editor_audio import (clip_fields, track_fields, clip_audible, track_audible, crossfade_reads, apply_track_audio, + duck_ranges, duck_gain, duck_sources) FPS = 24 SAMPLE_RATE = 48000 @@ -693,6 +694,10 @@ def _render_once(project, resolve, target, *, progress, cancelled, options, repa audio_tracks = {track['id']: track for track in project['tracks'] if track['kind'] == 'audio'} envelopes = {key: _CrossfadeReads(track, resolved.__getitem__, cancelled) for key, track in audio_tracks.items()} + # Lower during speech: the same cached spans the preview reads. + keys = duck_sources(project['tracks']) + ducking = duck_ranges(project['tracks'], _speech_spans( + keys, lambda key: (resolved[key] if key in resolved else resolve(key)).get('audio'), cancelled)) if keys else [] def joinable(track): # A continuation joins its sound only where nothing else shapes that seam: @@ -800,6 +805,8 @@ def save_lead(lead): or not clip_audible(following, audio_tracks[track], project['tracks']) or following['start'] != clip['start'] + clip['frames'])): gain *= np.clip((clip['frames'] * SAMPLES_PER_FRAME - 1 - positions) / (FADE_SAMPLES - 1), 0, 1).astype(np.float32) + if clip.get('duck') and ducking: + gain *= duck_gain(ducking, number * SAMPLES_PER_FRAME, SAMPLES_PER_FRAME).astype(np.float32) track_block = samples * gain[None, :] * clip.get('volume', 1) extras = (envelopes[track].extras(number * SAMPLES_PER_FRAME, SAMPLES_PER_FRAME) if track_audible(audio_tracks[track], project['tracks']) else None) @@ -853,6 +860,15 @@ def save_lead(lead): target.unlink(missing_ok=True) +def _speech_spans(keys, audio, cancelled): + from .editor_music import SpeechSpans + from .speech import SpeechError + try: + return SpeechSpans(_output_root(), resolve=audio).spans(keys, cancelled) + except SpeechError: + raise ExportCancelled() from None + + def _output_root(): import folder_paths return Path(folder_paths.get_output_directory()).resolve() diff --git a/freevideo_engine/editor_music.py b/freevideo_engine/editor_music.py new file mode 100644 index 00000000..d0870241 --- /dev/null +++ b/freevideo_engine/editor_music.py @@ -0,0 +1,600 @@ +"""Music for the editing page: a stretch of the sound track made into one piece of music. + +The page plans the piece where the user dragged (start and length on the sound track), sends the +length and what the music should be (its own words, or a style and a mood), and lays the finished +file into that stretch. This module only makes the file: first-use download of the music model, +one job at a time, progress, cancel. The music service yields to video: a video that starts while +the music is made stops it, and the job waits for the video and composes again. +""" +import logging +import re +import secrets +import threading +import time +from collections import deque +from pathlib import Path + +FPS = 24 +FOLDER = 'freevideo-music' +MAX_JOBS = 20 +MAX_QUIET = 500 +MAX_TEXT = 512 +# A video that starts while the music is made stops it; the job waits and composes again (as the Agent's job does). +PREEMPTIONS = 8 +POLL_SECONDS = .5 +_JOB_ID = re.compile(r'm-[0-9a-f]{32}') +_FIELDS = {'frames', 'text', 'style', 'mood', 'seed', 'quiet'} +REASONS = ('music_download_failed', 'out_of_memory', 'music_failed', 'invalid_request', 'busy', 'unsupported', 'failed') + +# Presets: the genre and instruments, then the feeling and pace. Instrumental only, as the Agent asks for. +STYLES = { + 'cinematic': 'Cinematic orchestral score with strings, soft brass and piano', + 'pop': 'Modern instrumental pop with clean electric guitar, bass, light drums and synth pads', + 'electronic': 'Electronic track with warm synthesizers, a steady drum machine and a deep bass line', + 'piano': 'Solo piano piece with a clear melody and gentle sustain', + 'folk': 'Acoustic folk tune with fingerpicked guitar, light percussion and a soft violin', + 'ambient': 'Ambient soundscape with slow evolving pads, soft textures and distant bells', +} +MOODS = { + 'calm': 'calm and peaceful, slow tempo around 70 BPM', + 'upbeat': 'bright and upbeat, lively tempo around 120 BPM', + 'tense': 'tense and suspenseful, a low pulse around 90 BPM that slowly builds', + 'warm': 'warm and tender, moderate tempo around 85 BPM', + 'sad': 'sad and reflective, slow tempo around 65 BPM in a minor key', +} +# The feeling alone, without the pace: used after the user's own words, which set the pace themselves +# (a fixed "around 70 BPM" after "a festival parade" pulled the piece back to the preset). +FEELINGS = { + 'calm': 'calm and peaceful', + 'upbeat': 'bright and upbeat', + 'tense': 'tense and suspenseful', + 'warm': 'warm and tender', + 'sad': 'sad and reflective, in a minor key', +} +DEFAULT_STYLE, DEFAULT_MOOD = 'cinematic', 'calm' + + +def caption(style, mood, words=None): + """The English description the music model reads for a preset pair, or for the user's words with the pair after them.""" + if style not in STYLES or mood not in MOODS: + raise ValueError('Unknown style or mood') + if words: + return '%s. %s, %s. Instrumental only, no singing, no vocals.' % (words.rstrip('.'), STYLES[style], FEELINGS[mood]) + return '%s, %s. Instrumental only, no singing, no vocals.' % (STYLES[style], MOODS[mood]) + + +def _whole(value, low, high): + return type(value) is int and low <= value <= high + + +def max_frames(): + from .music import service + return service.MAX_SECONDS * FPS + + +def validate_request(body): + """The page's request -> what the job runs. Unknown fields, unknown presets and bad ranges are refused.""" + if not isinstance(body, dict) or set(body) - _FIELDS: + raise ValueError('Invalid request') + frames = body.get('frames') + if not _whole(frames, 1, max_frames()): + raise ValueError('Invalid length') + text = body.get('text', '') + if not isinstance(text, str) or len(text) > MAX_TEXT or '\x00' in text: + raise ValueError('Invalid description') + text = ' '.join(text.split()) + style, mood = body.get('style'), body.get('mood') + if (style is None) != (mood is None): + raise ValueError('Style and mood go together') + if style is not None and (style not in STYLES or mood not in MOODS): + raise ValueError('Unknown style or mood') + if not text and style is None: + style, mood = DEFAULT_STYLE, DEFAULT_MOOD + seed = body.get('seed') + if seed is None: + seed = secrets.randbelow(2**32) + elif not _whole(seed, 0, 2**32 - 1): + raise ValueError('Invalid seed') + quiet = body.get('quiet', []) + if not isinstance(quiet, list) or len(quiet) > MAX_QUIET: + raise ValueError('Invalid quiet spans') + spans = [] + for span in quiet: + if (not isinstance(span, dict) or set(span) != {'start', 'frames'} or not _whole(span['start'], 0, frames - 1) + or not _whole(span['frames'], 1, frames - span['start'])): + raise ValueError('Invalid quiet span') + spans.append((span['start'], span['frames'])) + # The user's own words go first, with the chosen style and feeling after them (not the preset's pace); + # the full preset alone when there are none. + if text: + words, stem = text if style is None else caption(style, mood, text), 'text' + else: + words, stem = caption(style, mood), '%s-%s' % (style, mood) + return dict(frames=frames, caption=words, stem=stem, style=style, mood=mood, seed=seed, quiet=spans) + + +def _gpu_busy(server): + """True while a video (queued or running), a picture or the prompt model has the GPU.""" + try: + from . import resident_process + if resident_process.OWNER.active: + return True + except Exception: # noqa: BLE001 - an optional check; the music service refuses again if the GPU is taken + pass + if server is None: + return False + try: + front = getattr(server, '_freevideo_front', None) + running, pending = front.current_queue() if front is not None else server.prompt_queue.get_current_queue() + if running or pending: + return True + except Exception: # noqa: BLE001 + pass + for name in ('_freevideo_prompt_vlm',): + service = getattr(server, name, None) + try: + if service is not None and service.busy(): + return True + except Exception: # noqa: BLE001 + pass + try: + from .image_gen.service import get_service + service = get_service(server) + return service is not None and service.busy() + except Exception: # noqa: BLE001 + return False + + +def run_job(request, *, root, out, phase, cancelled, server=None, service=None, busy=None, poll=POLL_SECONDS): + """Compose one piece into `out`; returns the music service's numbers. Raises InterruptedError when cancelled.""" + if service is None: + from .music import service + busy = busy or (lambda: _gpu_busy(server)) + for attempt in range(PREEMPTIONS + 1): + try: + return service.compose(out, request['caption'], request['frames'], seed=request['seed'], quiet=request['quiet'], + fps=FPS, server=server, cancelled=cancelled, phase=phase) + except service.Preempted: + if attempt == PREEMPTIONS: + raise service.MusicError('music_failed', 'preempted') from None + phase('waiting') + # Wait out the video (and anything queued with it) before the GPU is taken again. + while True: + if cancelled(): + raise InterruptedError('cancelled') from None + time.sleep(poll) + if not busy(): + break + + +class MusicQueue: + """One worker, first in first out, cancellable; keeps the last MAX_JOBS jobs for the page to read back.""" + def __init__(self, root, *, runner=None, read=None, clock=time.monotonic, **kwargs): + self.root = Path(root).absolute() + self.runner = runner or run_job + self.read = read or self._read + self.clock = clock + self.kwargs = kwargs + self.lock = threading.RLock() + self.jobs = {} + self.pending = deque() + self.worker = None + # Seconds of composing per second of music on this machine, from the last finished job. + self.pace = None + + def _read(self, source): + from . import comfy_editor + return comfy_editor.sources(self.root, [source])[source] + + def _view(self, job): + value = {key: job[key] for key in ('id', 'state', 'phase', 'fraction', 'reason', 'result')} + # Composing reports no steps; the page shows time instead, from how long it took here last time. + if job['state'] == 'running' and job['phase'] == 'composing' and job['began'] is not None: + value['elapsed'] = round(self.clock() - job['began'], 1) + value['estimate'] = None if self.pace is None else round(self.pace * job['request']['frames'] / FPS, 1) + return value + + def submit(self, body): + request = validate_request(body) + hexid = secrets.token_hex(16) + job = dict(id='m-' + hexid, state='queued', phase=None, fraction=0., reason=None, result=None, began=None, + request=request, relative='%s/%s-%s.mp4' % (FOLDER, request['stem'], hexid), stop=threading.Event()) + with self.lock: + for old in list(self.jobs): + if len(self.jobs) < MAX_JOBS: + break + if self.jobs[old]['state'] not in ('queued', 'running'): + del self.jobs[old] + if len(self.jobs) >= MAX_JOBS: + raise MusicBusy() + self.jobs[job['id']] = job + self.pending.append(job['id']) + initial = self._view(job) + if self.worker is None: + self.worker = threading.Thread(target=self._run, name='freevideo-editor-music', daemon=True) + self.worker.start() + return initial + + def get(self, identity): + if not isinstance(identity, str) or not _JOB_ID.fullmatch(identity): + raise ValueError('Invalid job') + with self.lock: + if identity not in self.jobs: + raise KeyError(identity) + return self._view(self.jobs[identity]) + + def cancel(self, identity): + with self.lock: + self.get(identity) + job = self.jobs[identity] + if job['state'] in ('queued', 'running'): + job['stop'].set() + if identity in self.pending: + self.pending.remove(identity) + job.update(state='cancelled', phase=None) + return self._view(job) + + def busy(self): + with self.lock: + return any(job['state'] in ('queued', 'running') for job in self.jobs.values()) + + def _run(self): + while True: + with self.lock: + if not self.pending: + self.worker = None + return + job = self.jobs[self.pending.popleft()] + if job['stop'].is_set(): + job['state'] = 'cancelled' + continue + job['state'] = 'running' + + def phase(name, fraction=None): + with self.lock: + if name == 'composing' and job['phase'] != 'composing': + job['began'] = self.clock() + job['phase'] = name + job['fraction'] = 0. if fraction is None else max(0., min(.99, float(fraction))) + + out = self.root / job['relative'] + try: + out.parent.mkdir(parents=True, exist_ok=True) + began = self.clock() + numbers = self.runner(job['request'], root=self.root, out=out, phase=phase, cancelled=job['stop'].is_set, + **self.kwargs) or {} + if job['stop'].is_set(): + raise InterruptedError('cancelled') + source = 'generated:' + job['relative'] + found = self.read(source) + if found.get('missing') or not found.get('frames') or not found.get('has_audio'): + raise MusicFailed('music_failed') + spent = numbers.get('load_s', 0) + numbers.get('generate_s', 0) if numbers.get('generate_s') else self.clock() - began + with self.lock: + self.pace = max(.05, spent / (job['request']['frames'] / FPS)) + job.update(state='done', phase=None, fraction=1., result=dict( + source=source, frames=int(found['frames']), seed=job['request']['seed'], + style=job['request']['style'], mood=job['request']['mood'])) + except BaseException as error: # noqa: BLE001 - every outcome is the job's; the worker keeps serving + out.unlink(missing_ok=True) + with self.lock: + if isinstance(error, InterruptedError) or job['stop'].is_set(): + job.update(state='cancelled', phase=None, reason=None) + else: + code = getattr(error, 'code', None) + if code not in REASONS: + logging.warning('[FreeVideo] Editor music failed', exc_info=error) + code = 'music_failed' + job.update(state='failed', phase=None, reason=code) + if not isinstance(error, Exception): + raise + + +class MusicBusy(Exception): + code = 'busy' + + +class MusicFailed(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class MusicPack: + """First-use download of the music model, with the download sources chosen in Settings.""" + def __init__(self, *, installation=None, service=None, status=None, free_bytes=None): + self._installation = installation + self._service = service + self._status = status + self._free = free_bytes + self.lock = threading.Lock() + self.value = dict(running=False, fraction=0., reason=None) + self.stop = threading.Event() + self.worker = None + + def installation(self): + if self._installation is not None: + return self._installation() + from .comfy_bridge import installation + return installation() + + def service(self): + if self._service is not None: + return self._service + from .music import service + return service + + def status(self): + root, machine = self.installation() + if self._status is not None: + pack = self._status(machine['model_root']) + else: + from .image_gen import image_pack + pack = image_pack.status(machine['model_root'], group='music') + if self._free is not None: + free = self._free(machine['model_root']) + else: + from . import disk_space + from .image_gen import image_pack + folder = image_pack.base(machine['model_root']) + while not folder.exists() and folder.parent != folder: + folder = folder.parent + free = disk_space.free_bytes(folder) + remaining = int(pack.get('remaining_bytes', pack.get('bytes', 0))) + from .image_gen.image_pack import RESERVE + with self.lock: + value = dict(self.value) + value.update(ready=bool(pack['ready']), bytes=remaining, + disk_ok=bool(pack['ready']) or free >= remaining + RESERVE) + return value + + def start(self): + """Begin the download unless it runs already; returns False when the disk cannot take it.""" + value = self.status() + if value['ready'] or value['running']: + return True + if not value['disk_ok']: + with self.lock: + self.value.update(reason='disk_space') + return False + root, machine = self.installation() + with self.lock: + if self.value['running']: + return True + self.stop = threading.Event() + self.value = dict(running=True, fraction=0., reason=None) + self.worker = threading.Thread(target=self._run, args=(root, machine['model_root'], self.stop), + name='freevideo-editor-music-pack', daemon=True) + self.worker.start() + return True + + def cancel(self): + self.stop.set() + + def _run(self, root, model_root, stop): + service = self.service() + + def progress(fraction): + with self.lock: + self.value['fraction'] = max(self.value['fraction'], min(.99, float(fraction))) + + reason = None + try: + service.install(model_root, root, progress, stop.is_set) + except InterruptedError: + pass + except service.MusicError as error: + reason = error.reason or 'failed' + except Exception: # noqa: BLE001 - the page says it did not download; the log keeps why + logging.warning('[FreeVideo] Music model download failed', exc_info=True) + reason = 'failed' + with self.lock: + self.value = dict(running=False, fraction=0. if reason or stop.is_set() else 1., reason=reason) + + +def available(installation=None): + """The music model runs where video runs through CUDA (NVIDIA and AMD ROCm); not on Apple silicon yet.""" + try: + if installation is None: + from .comfy_bridge import installation + _root, machine = installation() + return machine.get('device_backend', 'cuda') != 'mps' + except Exception: # noqa: BLE001 - no engine, no music + return False + + +class SpeechSpans: + """Speech spans per editor source for 'Lower during speech'; one file at a time, off the request thread. + + The preview asks per source and gets pending until the spans are ready; the export asks for all of its + sources at once and waits. Both read the same cached spans for the same file. + """ + def __init__(self, root, *, resolve=None, model_path=None, compute=None): + self.root = Path(root) + self._resolve, self._model_path, self._compute = resolve, model_path, compute + self.lock = threading.Lock() + self.pending, self.failed, self.known = deque(), {}, {} + self.worker = None + + def cache(self): + return self.root / 'FreeVideo' / '.editor' / 'speech' + + def model_path(self): + if self._model_path is not None: + return Path(self._model_path) + from .comfy_bridge import installation + from .image_gen import image_pack + from .speech import MODEL_FILE + return image_pack.base(installation()[1]['model_root']) / MODEL_FILE + + def audio(self, key): + if self._resolve is not None: + return self._resolve(key) + from .editor_export import resolve_source + return resolve_source(key).get('audio') + + def compute(self, path, cancelled=None): + if self._compute is not None: + return self._compute(path) + from .speech import speech_spans + return speech_spans(path, self.model_path(), self.cache(), cancelled) + + def get(self, key): + """{'spans': [...]} when known, {'state': 'pending'} while working, {'spans': None, 'reason': …} otherwise.""" + if not isinstance(key, str) or not re.fullmatch(r'(work|generated|import):[^\x00]{1,512}', key): + raise ValueError('Invalid source') + try: + path = self.audio(key) + except (FileNotFoundError, KeyError): + return dict(spans=None, reason='source_missing') + if path is None: + return dict(spans=[]) + with self.lock: + if key in self.known: + return dict(spans=self.known[key]) + model = self.model_path() + if not model.is_file(): + return dict(spans=None, reason='model_missing') + from .speech import cached + found = cached(path, model, self.cache()) + if found is not None: + return dict(spans=found) + with self.lock: + if key in self.failed: + return dict(spans=None, reason=self.failed[key]) + if key not in self.pending: + if len(self.pending) >= MAX_QUIET: + return dict(spans=None, reason='busy') + self.pending.append(key) + if self.worker is None or not self.worker.is_alive(): + self.worker = threading.Thread(target=self._run, name='freevideo-editor-speech', daemon=True) + self.worker.start() + return dict(state='pending') + + def _run(self): + from .speech import SpeechError + while True: + with self.lock: + if not self.pending: + self.worker = None + return + key = self.pending[0] + reason = found = None + try: + found = self.compute(self.audio(key)) + except SpeechError as error: + reason = error.reason + except Exception: # noqa: BLE001 - one unreadable source must not stop the others + logging.getLogger(__name__).warning('Editor speech spans failed for %s.', key, exc_info=True) + reason = 'failed' + with self.lock: + self.pending.popleft() + if reason is not None: + self.failed[key] = reason + else: + self.known[key] = found + + def spans(self, keys, cancelled=None): + """Speech spans for each key the export needs; a source that cannot be read has none.""" + found = {} + for key in keys: + try: + path = self.audio(key) + found[key] = self.compute(path, cancelled) if path is not None else [] + except Exception as error: # noqa: BLE001 - no spans for that source, as in the preview + if getattr(error, 'reason', None) == 'cancelled': + raise + logging.getLogger(__name__).warning('Editor export: no speech spans for %s (%s).', key, + getattr(error, 'reason', type(error).__name__)) + return found + + +def register(): + """Register the editing page's music endpoints once on the existing ComfyUI server.""" + import asyncio + from aiohttp import web + from server import PromptServer + from .editor_export import _output_root + server = PromptServer.instance + if server is None or getattr(server, '_freevideo_editor_music', None) is not None: + return + queue = MusicQueue(_output_root(), server=server) + pack = MusicPack() + server._freevideo_editor_music = queue + server._freevideo_editor_speech = speech = SpeechSpans(_output_root()) + headers = {'Cache-Control': 'no-store'} + + def response(value): + return web.json_response({'job': value}, headers=headers) + + def refused(reason, status): + return web.json_response({'reason': reason}, status=status, headers=headers) + + @server.routes.post('/freevideo/editor/music') + async def start(request): + try: + body = await request.json() + if not await asyncio.to_thread(available): + return refused('unsupported', 409) + return response(await asyncio.to_thread(queue.submit, body)) + except MusicBusy: + return refused('busy', 503) + except (ValueError, TypeError, KeyError): + return refused('invalid_request', 400) + + @server.routes.get('/freevideo/editor/music') + async def status(request): + identity = request.query.get('id') + if identity is None: + supported = await asyncio.to_thread(available) + try: + ready = supported and (await asyncio.to_thread(pack.status))['ready'] + except Exception: # noqa: BLE001 - an unreadable pack reads as not downloaded + ready = False + # pace: seconds of composing per second of music here, from the last piece (null before the first). + return web.json_response(dict(available=supported, ready=bool(ready), pace=queue.pace), headers=headers) + try: + return response(queue.get(identity)) + except KeyError: + return refused('missing', 404) + except ValueError: + return refused('invalid_request', 400) + + @server.routes.post('/freevideo/editor/music/cancel') + async def cancel(request): + try: + body = await request.json() + if not isinstance(body, dict) or set(body) != {'id'}: + raise ValueError('Invalid request') + return response(queue.cancel(body['id'])) + except KeyError: + return refused('missing', 404) + except (ValueError, TypeError): + return refused('invalid_request', 400) + + @server.routes.get('/freevideo/editor/speech') + async def speech_spans(request): + try: + return web.json_response(await asyncio.to_thread(speech.get, request.query.get('source')), headers=headers) + except ValueError: + return refused('invalid_request', 400) + + @server.routes.get('/freevideo/editor/music-pack') + async def pack_status(request): + return web.json_response(await asyncio.to_thread(pack.status), headers=headers) + + @server.routes.post('/freevideo/editor/music-pack') + async def pack_action(request): + try: + body = await request.json() + if not isinstance(body, dict) or set(body) != {'action'} or body['action'] not in ('install', 'cancel'): + raise ValueError('Invalid request') + if body['action'] == 'cancel': + pack.cancel() + elif not await asyncio.to_thread(available): + return refused('unsupported', 409) + elif not await asyncio.to_thread(pack.start): + return web.json_response(await asyncio.to_thread(pack.status), status=507, headers=headers) + except (ValueError, TypeError): + return refused('invalid_request', 400) + return web.json_response(await asyncio.to_thread(pack.status), headers=headers) diff --git a/freevideo_engine/encode.py b/freevideo_engine/encode.py index 0a621bda..de9f8ad7 100644 --- a/freevideo_engine/encode.py +++ b/freevideo_engine/encode.py @@ -8,6 +8,7 @@ from .hardware import detect from .kernel_capabilities import available_backends from .locking import runtime_lock, LOCK_ENV +from .music_request import without_music from .paths import comfy_root from .policy import choose from . import processes @@ -31,7 +32,7 @@ def interrupted(signum, frame): try: with runtime_lock() as descriptor, tempfile.TemporaryDirectory(prefix='freevideo-encode-', dir=destination.parent) as directory: request = Path(directory) / 'request.json' - request.write_text(json.dumps({'prompt': args.prompt_file.read_text(encoding='utf-8'), 'encoder': args.encoder, + request.write_text(json.dumps({'prompt': without_music(args.prompt_file.read_text(encoding='utf-8')), 'encoder': args.encoder, 'comfy_root': str(args.comfy_root or comfy_root()), 'model_paths': str(args.model_paths.resolve()) if args.model_paths else None, 'output': str(destination), 'gpu_budget_gb': policy.gpu_budget_bytes / 1e9, diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index af9d23c8..bad53f56 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -205,6 +205,11 @@ def setup_command_failure(label, log, exit_code, report=None): return '\n'.join(lines) + STEP_OUTPUT + output +# uv's error for an interpreter that exists but cannot be started, as in +# "error: Failed to run `C:\...\python.exe`" then "Caused by: (os error 5)". +PYTHON_START_DENIED = r'Failed to run [^\n]*python[^\n]*\s+Caused by:[^\n]*(?:os error 5|WinError 5)\b' + + # Leads the error of a step that needed an environment's Python file after it # disappeared, e.g. while security software was holding it in quarantine. ENVIRONMENT_PYTHON_REMOVED = 'The Python file of the FreeVideo environment is missing' @@ -219,6 +224,15 @@ def environment_python_removed(args, pythons): return '' +# Windows sharing violation: another program (often security software scanning +# a file that was just written) holds a file open. It is usually released within seconds. +FILE_IN_USE = r'os error 32\b|WinError 32\b|being used by another process|另一个程序正在使用' + + +def file_in_use(text): + return re.search(FILE_IN_USE, str(text)) is not None + + # Checked before free space: freeing space cannot fix a disk that cannot hold the installation. PLACE_RULES = ( (r'Git is unavailable\. Check the network connection', 'git-unavailable', @@ -738,17 +752,31 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re _joined(CHANGED_WHY['files'], CHANGED_REPAIR)), (r'The installation plan changed\. Detect configuration again', 'plan-changed', CHANGED_TITLE, _joined(CHANGED_WHY['other'], CHANGED_REPAIR)), + (FILE_IN_USE, 'file-in-use', + ('A file is in use by another program', '文件被其他程序占用'), + ('Security software or another program is holding a file that setup needs. Wait a moment and click Retry installation; ' + 'if it happens again, allow the FreeVideo folder in the security software.', + '安全软件或其他程序正在占用安装所需的文件。请稍候后点“重试安装”；仍然失败时，请在安全软件中允许 FreeVideo 所在的文件夹。')), (ENVIRONMENT_PYTHON_REMOVED, 'environment-python-removed', ('A program file is missing', '缺少一个程序文件'), ('It may have been quarantined by security software. Restore the file or allow the FreeVideo folder, ' 'then click "Retry installation".', '可能被安全软件隔离了。恢复这个文件或允许 FreeVideo 所在的文件夹后，点“重试安装”。')), + # uv could not start an environment's Python and Windows answered "access denied". The error code + # is matched, not the text, which Windows writes in the system language. + (PYTHON_START_DENIED, 'python-start-denied', + ('Windows refused to run the Python program', 'Windows 拒绝运行 Python 程序'), + ('Security software or a Windows protection feature is blocking the Python file of the FreeVideo environment. ' + 'Allow the FreeVideo folder there (for Windows Security: Virus & threat protection → Manage ransomware protection → ' + 'Controlled folder access, and the exclusions list), then click Retry installation.', + '安全软件或 Windows 的保护功能拦截了 FreeVideo 运行环境中的 Python 文件。请在其中允许 FreeVideo 所在的文件夹' + '（Windows 安全中心：病毒和威胁防护 → 管理勒索软件防护 → 受控文件夹访问，以及排除项），然后点“重试安装”。')), # Windows refuses PyTorch's unsigned DLLs under Smart App Control; reinstalling changes nothing. (r'WinError 4551|Application Control policy has blocked', 'app-control', ('Windows blocked some runtime files', 'Windows 拦截了运行组件'), ('Smart App Control in Windows blocked these files. Search Windows for "Smart App Control", or open Windows Security → App & browser control → Smart App Control settings.', 'Windows 的“智能应用控制”拦截了这些文件。在 Windows 中搜索“智能应用控制”，或打开“Windows 安全中心 → 应用和浏览器控制 → 智能应用控制设置”。')), - (r'PermissionError|Access is denied|Permission denied|WinError 5\b', 'permission', + (r'PermissionError|Access is denied|Permission denied|WinError 5\b|os error 5\b', 'permission', ('This folder is not writable', '无法写入这个目录'), ('Choose an installation folder you can write to, then retry.', '选择有写入权限的安装目录后重试。')), # A worker's own check of the cuDNN files it loaded (runtime_libraries). diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 3a97b5a7..b91e6f4d 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -21,6 +21,7 @@ from .paths import vdn_root, comfy_root from .monitoring import save from .encoder_checkpoint import preflight_encoder +from .music_request import without_music from . import tuning from . import processes from .system import (inference_headroom, inference_memory_sample, inference_environment, memory_sample, @@ -841,6 +842,7 @@ def interrupted(signum, frame): # The saved prompt stays the user's own; the model is also told what the picture is. from .continuation import first_frame_prompt prompt_text = first_frame_prompt(prompt_text) + prompt_text = without_music(prompt_text) encoding = {'prompt': prompt_text, 'encoder': args.encoder, 'comfy_root': str(args.comfy_root or comfy_root()), 'model_paths': str(args.model_paths.resolve()) if args.model_paths else None, diff --git a/freevideo_engine/image_gen/cast_views.py b/freevideo_engine/image_gen/cast_views.py index 012b99c6..e86d05ae 100644 --- a/freevideo_engine/image_gen/cast_views.py +++ b/freevideo_engine/image_gen/cast_views.py @@ -69,8 +69,8 @@ def view_prompt(card, view): """The prompt for one view of a card (a dict with name, description, kind).""" who = _who(card) if card.get('kind') == 'setting': - return ('%s. A wide view of the empty place, no people, even daylight, natural colours, sharp focus, ' - 'no text.' % who) + # The time, weather, light and colour the description gives are the point of a scene: only the people and text are barred. + return '%s. A wide view of the place, no people, sharp focus, no text.' % who if view == 'front': return ('%s. The person stands alone, the whole body from head to feet, facing the camera, neutral pose ' 'with the arms relaxed, %s.' % (who, BACKDROP)) @@ -99,13 +99,16 @@ def photo_prompt(): 'add no new accessories.' % BACKDROP) -def page_views(description): +def page_views(description, kind='character'): """What the page needs to make the pictures of a new character itself: each view's prompt and size. The page runs the pictures one by one so each shows when it is done; the prompts stay here, so a card made from a description and these pictures are made from the same words.""" if not isinstance(description, str) or not description.strip() or len(description) > 1000: raise ValueError('invalid_prompt') - card = dict(name=description.strip()[:120], description=description.strip(), kind='character') + if kind not in ('character', 'setting'): + raise ValueError('invalid_request') + card = dict(name=description.strip()[:120], description=description.strip(), kind=kind) + views = SETTING_VIEWS if kind == 'setting' else VIEWS return dict(views=[dict(view=view, prompt=view_prompt(card, view), width=SIZES[view][0], - height=SIZES[view][1], from_front=uses_front(view)) for view in VIEWS]) + height=SIZES[view][1], from_front=uses_front(view)) for view in views]) diff --git a/freevideo_engine/image_gen/image_models.json b/freevideo_engine/image_gen/image_models.json index 797515d9..86f7f861 100644 --- a/freevideo_engine/image_gen/image_models.json +++ b/freevideo_engine/image_gen/image_models.json @@ -661,6 +661,29 @@ "url": "https://www.modelscope.cn/models/ACE-Step/Ace-Step1.5/resolve/dd20ee00556360ae895e308efe9d11daf08fe9d2/vae/diffusion_pytorch_model.safetensors" } ] + }, + { + "file": "music/silero-vad/model.onnx", + "component": "music", + "bytes": 2243022, + "sha256": "a4a068cd6cf1ea8355b84327595838ca748ec29a25bc91fc82e6c299ccdc5808", + "repo": "onnx-community/silero-vad", + "revision": "e71cae966052b992a7eca6b17738916ce0eca4ec", + "model_id": "silero-vad", + "sources": [ + { + "id": "official", + "url": "https://huggingface.co/onnx-community/silero-vad/resolve/e71cae966052b992a7eca6b17738916ce0eca4ec/onnx/model.onnx" + }, + { + "id": "hf-mirror", + "url": "https://hf-mirror.com/onnx-community/silero-vad/resolve/e71cae966052b992a7eca6b17738916ce0eca4ec/onnx/model.onnx" + }, + { + "id": "modelscope", + "url": "https://www.modelscope.cn/models/onnx-community/silero-vad/resolve/2f080fcc38651e65b738157ad84c3c980beb08cc/onnx/model.onnx" + } + ] } ] } diff --git a/freevideo_engine/image_gen/image_pack.py b/freevideo_engine/image_gen/image_pack.py index fe60e97e..4b252f07 100644 --- a/freevideo_engine/image_gen/image_pack.py +++ b/freevideo_engine/image_gen/image_pack.py @@ -229,7 +229,9 @@ def _owned_intact(path, stamp): b'https://github.com/facebookresearch/sam2\n') MUSIC_NOTICE_TEXT = (b'ACE-Step 1.5, Copyright (c) 2026 ACEStep. Licensed under the MIT License; see ' b'freevideo_engine/licenses/ACE-Step-MIT.txt.\nhttps://github.com/ace-step/ACE-Step-1.5\n' - b'Qwen3-Embedding-0.6B, Copyright (c) Alibaba Cloud. Licensed under Apache-2.0.\n') + b'Qwen3-Embedding-0.6B, Copyright (c) Alibaba Cloud. Licensed under Apache-2.0.\n' + b'Silero VAD, Copyright (c) 2020-present Silero Team. Licensed under the MIT License; see ' + b'freevideo_engine/licenses/Silero-VAD-MIT.txt.\nhttps://github.com/snakers4/silero-vad\n') NOTICES = {'image': ('NOTICE.txt', NOTICE_TEXT), 'segment': ('SAM2-NOTICE.txt', SEGMENT_NOTICE_TEXT), 'music': ('MUSIC-NOTICE.txt', MUSIC_NOTICE_TEXT)} diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index 8889c7e6..4c1b57bc 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -1256,7 +1256,7 @@ async def action(request): name = request.match_info['action'] if name == 'cast-views': from .cast_views import page_views - result = page_views(value.get('description')) + result = page_views(value.get('description'), value.get('kind', 'character')) elif name == 'status': result = service.status(value.get('job')) elif name == 'cancel': diff --git a/freevideo_engine/macos_generate.py b/freevideo_engine/macos_generate.py index 7f3400e6..260efa14 100644 --- a/freevideo_engine/macos_generate.py +++ b/freevideo_engine/macos_generate.py @@ -19,6 +19,7 @@ from .geometry import geometry from .locking import LOCK_ENV, runtime_lock from .monitoring import save +from .music_request import without_music from .paths import data_root GiB = 2**30 @@ -278,9 +279,13 @@ def execute(value, name, module='freevideo_engine.macos_generate'): if not prompt.strip() and not getattr(args, 'allow_empty_prompt', False): raise ValueError('A nonempty prompt is required') prompt_file = Path(args.prompt_file).resolve() + encoder_prompt = prompt if report.get('first_frame_reference', {}).get('used'): from .continuation import first_frame_prompt - prompt = first_frame_prompt(prompt) + encoder_prompt = first_frame_prompt(encoder_prompt) + encoder_prompt = without_music(encoder_prompt) + if encoder_prompt != prompt: + prompt = encoder_prompt prompt_file = artifacts / 'encoder-prompt.txt' prompt_file.write_text(prompt, encoding='utf-8') checkpoint = encoder_checkpoint(args) diff --git a/freevideo_engine/music_request.py b/freevideo_engine/music_request.py new file mode 100644 index 00000000..e2cf17df --- /dev/null +++ b/freevideo_engine/music_request.py @@ -0,0 +1,64 @@ +"""Whether a prompt asks for no music, and what that means for the score the model is given. + +H3 reads the score from its own `non_diegetic_music` section. The starter prompt carries a full +score there, so a user who adds "no background music" to the scene text but leaves that section +as it came still gets the score. Only that untouched starter text is replaced by `N/A`; a score +the user wrote stays as written. +""" +import re + +from .default_prompt import DEFAULT_PROMPT + +SECTION = re.compile(r'(?m)^[ \t]*non_diegetic_music[ \t]*:[ \t]*(.*?)(?=\n\s*\n|\Z)', re.S) +NOTE = 'non_diegetic_music: N/A' + + +def _words(text): + return ' '.join(text.split()) + + +STARTER_MUSIC = _words(SECTION.search(DEFAULT_PROMPT).group(1)) + +# A music word that is not another noun: "music video", "music stand", 音乐会. +_EN_TERM = (r'(?:(?:background|backing|ambient|incidental|film|movie|cinematic)\s+)*' + r'(?:music(?!al\b|ian|\s+(?:video|stand|box|hall|store|shop|school|class|player|room|note|sheet|book|' + r'festival|stage|teacher|lesson))|bgm\b|soundtrack|musical\s+(?:score|accompaniment)|' + r'(?:background|film|movie)\s+score|backing\s+track)') +_EN_FILLER = (r'(?:\s+(?:to|the|any|a|an|of|add|include|use|play|put|have|want|need|generate|create|make|produce|' + r'insert|bring|with|in|on|over|under|any\s+kind\s+of|kind\s+of|sort\s+of|type\s+of|form\s+of)){0,5}') +_EN = [ + re.compile(r'\b(?:no|without|zero)\s+(?:(?:any|the|extra|added|additional)\s+){0,2}' + _EN_TERM, re.I), + re.compile(r"\b(?:do\s*n[o']t|do\s+not|don[’']t|never|avoid|omit|exclude|skip|remove|mute|disable|" + r'turn\s+off|leave\s+out|cut)\b' + _EN_FILLER + r'\s+' + _EN_TERM, re.I), + re.compile(r'\b(?:music|bgm|soundtrack)[- ]free\b|\bfree\s+of\s+' + _EN_TERM + r'|\bno-music\b', re.I), +] + +_ZH_TERM = r'(?:背景音乐|背景乐|背景配乐|电影配乐|配乐|伴奏|[Bb][Gg][Mm]|音乐(?!会|剧|人|家|盒|厅|节|学|课|教室|剧院|视频|短片|[Mm][Vv]|老师))' +_ZH_NEG = (r'(?:不要|不用|无需|不需要|不想要|不能有|不可以有|不得有|不许有|严禁|禁止|别|勿|去掉|去除|删掉|删除|取消|关闭|关掉|' + r'没有|无|不带|不加|不配|不含|不放|不出现|不添加|不使用|不生成|不包含|不要有|不要加|不要带|不要配|不要放)') +_ZH_FILLER = r'(?:任何|任何的|任何形式的|一切|所有|的|有|加|带|配|放|要|出现|添加|使用|生成|含|包含|额外|多余|额外的|多余的|任何额外的){0,3}' +_ZH = [ + re.compile(_ZH_NEG + _ZH_FILLER + _ZH_TERM), + re.compile(_ZH_TERM + r'(?:请)?(?:不要|不用|不需要|无需|关闭|关掉|去掉|去除|取消|删掉)'), +] + + +def asks_for_no_music(text): + """True when the prompt text says, in English or Chinese, that it wants no music.""" + return isinstance(text, str) and any(each.search(text) for each in _EN + _ZH) + + +def writes_own_music(text): + """True when the prompt has a score section that is neither the starter's nor N/A.""" + found = SECTION.search(text) + return bool(found) and _words(found.group(1)) not in (STARTER_MUSIC, 'N/A', '') + + +def without_music(text): + """The prompt with the starter's score replaced by `N/A` when the scene text asks for no music.""" + if not asks_for_no_music(text): + return text + found = SECTION.search(text) + if not found or _words(found.group(1)) != STARTER_MUSIC: + return text + return text[:found.start()] + NOTE + text[found.end():] diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index 61254492..ce917f2e 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -1146,6 +1146,9 @@ def verification(done, total): measured_rates = {} left_slow = set() for cycle in range(cycles): + # Per round: a source refused the retained prefix / another was only slow, + # stalled or interrupted (those may still finish from the same bytes). + resume_refused = retryable = False # Once every route has been tried, keep the fastest observed route even # if the user's entire connection is slow. Do not oscillate forever. if cycle and updating and not left_slow: @@ -1360,6 +1363,7 @@ def remember_source(): break if offset and (process.returncode == 33 or process.returncode == 22 and status == '416') and restart == 0: if model or keep_partial: + resume_refused = True note_failure(name, route, 'source-does-not-support-range', address=url) event(network, category=category, source=name, file=path.name, action='resume-refused', resume_bytes=offset, reason='source-does-not-support-range') @@ -1373,6 +1377,7 @@ def remember_source(): interrupted = process.returncode not in (0, 22, 33, 63) and current_size > offset transient = (stalled or interrupted or process.returncode in (5, 6, 7, 18, 28, 35, 52, 55, 56, 92, 95) or status in ('429', '500', '502', '503', '504')) + retryable = retryable or transient or slow or faster is not None if model and transient and restart < 2 and candidate_index + 1 == len(candidates) and process.returncode not in (5, 7): retry_wait(network, restart, category=category, source=name, file=path.name, resume_bytes=current_size, reason='no-progress' if stalled else 'connection-interrupted') @@ -1391,12 +1396,18 @@ def remember_source(): event(network, category=category, source=name, route=route, file=path.name, action='fallback', reason=reason, http_status=status, resume_bytes=current_size, **(dict(detail=curl_message) if curl_message else {})) break - if model and partial.exists() and partial.stat().st_size: + # Pause or start over only when a source refused the retained prefix and + # no other source was merely slow, stalled or interrupted (it may still + # finish from the same bytes in the next round). The last round never + # starts over, since nothing would download afterwards. + if (model and resume_refused and (not retryable or cycle + 1 == cycles) + and partial.exists() and partial.stat().st_size): if not network.get('allow_model_restart'): pause_download(network, name, path, partial.stat().st_size, 'sources could not continue the retained prefix') - event(network, category=category, source=name, file=path.name, action='restart-approved', - retained_bytes=partial.stat().st_size, reason='resume-unavailable') - retain_partial(partial, 'restart-approved') + if cycle + 1 < cycles: + event(network, category=category, source=name, file=path.name, action='restart-approved', + retained_bytes=partial.stat().st_size, reason='resume-unavailable') + retain_partial(partial, 'restart-approved') message = 'Every download source failed for %s. Check your network or choose another connection mode in Downloads. Partial files retained' % path.name error = DownloadError(message + sources_text(failures.values())) error.attempts = [dict(source=source, route=route, reason=reason) for (source, route), reason in attempts.items()] diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py index 16590f79..53d0b687 100644 --- a/freevideo_engine/offline_packages.py +++ b/freevideo_engine/offline_packages.py @@ -9,6 +9,7 @@ import threading import time import zipfile +import zlib import traceback import logging @@ -150,8 +151,24 @@ def check_runtime_platform(): raise ValueError('The Windows runtime package cannot be used on macOS.') +# A damaged download shows up as a bad CRC or directory (BadZipFile), a broken +# compressed stream (zlib.error) or a cut-off one (EOFError). +DAMAGED = (zipfile.BadZipFile, zlib.error, EOFError) + + +def damaged(path): + return ValueError('离线包已损坏，请重新下载：' + Path(path).name) + + def inspect_archive(path): """Read only bounded metadata; file contents are checked during import.""" + try: + return _inspect_archive(path) + except DAMAGED as error: + raise damaged(path) from error + + +def _inspect_archive(path): path = io_path(path) with zipfile.ZipFile(path) as archive: names = archive.namelist() @@ -327,8 +344,8 @@ def _import_archive(path, destination, progress, cancelled): output.write(data); done += len(data) digest.update(data) progress(done=done, total=total, detail=path.name) - except zipfile.BadZipFile as error: - raise ValueError('离线包已损坏，请重新下载：' + path.name) from error + except DAMAGED as error: + raise damaged(path) from error if partial.stat().st_size != row['bytes'] or digest.hexdigest() != row['sha256']: raise ValueError('离线包校验失败，已保留文件：' + path.name) partial.replace(target) diff --git a/freevideo_engine/precompile.py b/freevideo_engine/precompile.py index 11da79bf..6f48ee59 100644 --- a/freevideo_engine/precompile.py +++ b/freevideo_engine/precompile.py @@ -21,6 +21,7 @@ """ import argparse import json +import math import os from pathlib import Path import re @@ -40,7 +41,16 @@ # Medium starts from a larger first pass, which is another canvas to compile. An # installation with no request on record has not shown which level it uses. MEDIUM_SHAPE = dict(DEFAULT_SHAPE, upscale_factor=1.5) -DEFAULT_SHAPES = [DEFAULT_SHAPE, MEDIUM_SHAPE] +# A portrait canvas and a picture-referenced request are other compiled shapes +# (the frame count too, so portrait is compiled at 5.2 s, the length it is made +# at), and a first request of either compiled its own on the spot. The picture +# is one 1008-row reference with a prompt of 460 rows beside it. +REFERENCE_ROWS = 1008 +PORTRAIT_SHAPE = dict(DEFAULT_SHAPE, width=768, height=1344, frames=124) +REFERENCE_SHAPE = dict(DEFAULT_SHAPE, task='ref2va', reference_video_tokens=REFERENCE_ROWS, + text_tokens=(460 + REFERENCE_ROWS,)) +DEFAULT_SHAPES = [DEFAULT_SHAPE, MEDIUM_SHAPE, PORTRAIT_SHAPE, REFERENCE_SHAPE, + dict(REFERENCE_SHAPE, width=768, height=1344, frames=124)] SHAPES = 2 LENGTHS = 2 TIMEOUT_SECONDS = 600 @@ -106,6 +116,21 @@ def recent_shapes(history, limit=SHAPES): for shape, lengths in shapes.values()] +def with_other_defaults(recent): + """The recent shapes, then the portrait and picture-referenced defaults they lack. + + A request of one of those shapes is as likely to come next as a repeat, and + its first compile waits at the first step. With no recent shape there is + nothing to add them to: the caller compiles every default. + """ + if not recent: + return recent + def kind(shape): + return {name: value for name, value in shape.items() if name != 'text_tokens'} + seen = [kind(shape) for shape in recent] + return recent + [shape for shape in DEFAULT_SHAPES[2:] if kind(shape) not in seen] + + def plan_request(machine, shape, history, identity): """The worker request a generation of this shape would start with.""" from .compatibility import Store, apply @@ -117,6 +142,9 @@ def plan_request(machine, shape, history, identity): canvas = geometry(shape['width'], shape['height'], frames=shape['frames']) if shape['base_steps'] != 8: canvas['steps'] = shape['base_steps'] + if shape.get('reference_video_tokens'): + # A request's canvas carries its pictures' packed rows, which the plan counts. + canvas.update(reference_video_tokens=shape['reference_video_tokens'], reference_audio_tokens=0) from .resource_settings import read as read_resources # The installation's capacity caps and the user's saved reserve, as a # request's managed command line carries them. @@ -193,7 +221,7 @@ def run(machine, *, environment=None, timeout=TIMEOUT_SECONDS, background=False, hardware = detect() history = ResourceHistory(history_path()) identity = local_identity(hardware, Path(machine['cache'])) - shapes = recent_shapes(history) or DEFAULT_SHAPES + shapes = with_other_defaults(recent_shapes(history)) or DEFAULT_SHAPES except Exception as error: hardware, history, identity, shapes = None, None, None, DEFAULT_SHAPES receipt['history_error'] = repr(error) @@ -391,7 +419,8 @@ def compile_plan(receipt, shape, request, profile, index, folder, environment, d receipt['shapes'].append(row) tick = time.perf_counter() try: - request['conditioning'] = [str(zero_conditioning(folder, tokens)) for tokens in shape['text_tokens']] + request['conditioning'] = [str(zero_conditioning(folder, tokens, shape.get('reference_video_tokens', 0))) + for tokens in shape['text_tokens']] request['receipt'] = str(folder / ('shape-%d.json' % index)) path = folder / ('request-%d.json' % index) save(path, request) @@ -499,13 +528,24 @@ def worker_options(options): return dict(options, pin_host_gb=0., pin_host_weights=False, preload_host=False) -def zero_conditioning(folder, tokens): - """A text-to-video prompt of `tokens` rows; only its length reaches a kernel.""" +def zero_conditioning(folder, tokens, reference_rows=0): + """A prompt of `tokens` rows; only its length reaches a kernel. + + With reference rows the last of them are the picture's vision tokens, and the + picture is a zero latent that packs into as many rows before the video. + """ import torch - path = folder / ('zero-%d.pt' % tokens) + path = folder / ('zero-%d-%d.pt' % (tokens, reference_rows) if reference_rows else 'zero-%d.pt' % tokens) if not path.exists(): - torch.save({'prompt_embeds': torch.zeros(tokens, 5120, dtype=torch.bfloat16), - 'text_token_tags': torch.ones(tokens, dtype=torch.long), 'task': 't2va'}, path) + tags = torch.ones(tokens, dtype=torch.long) + value = {'prompt_embeds': torch.zeros(tokens, 5120, dtype=torch.bfloat16), 'text_token_tags': tags, 'task': 't2va'} + if reference_rows: + tags[tokens - reference_rows:] = 0 + # Even sides, as the H3 patches need: rows = (height / 2) x (width / 2). + high = next(n for n in range(math.isqrt(reference_rows), 0, -1) if reference_rows % n == 0) + value.update(task='ref2va', references=[dict(kind='image', latent=torch.zeros( + 1, 24, 1, 2 * high, 2 * reference_rows // high))]) + torch.save(value, path) return path diff --git a/freevideo_engine/prompt_vlm/rules.py b/freevideo_engine/prompt_vlm/rules.py index 20b65791..2c6384c4 100644 --- a/freevideo_engine/prompt_vlm/rules.py +++ b/freevideo_engine/prompt_vlm/rules.py @@ -6,6 +6,8 @@ import math import re +from ..music_request import asks_for_no_music, writes_own_music + BASE_FIELDS = ('integrated_multimodal_description', 'overall_soundscape', 'non_diegetic_music') REF_FIELDS = ('subject_definitions', 'summary', 'retention_analysis', 'detailed_description', 'overall_soundscape', 'non_diegetic_music') @@ -180,6 +182,9 @@ def validate_output(text, value): for each in required: if each not in text: raise InvalidOutput('missing_picture_reference', text, value) + if asks_for_no_music(value['text']) and not writes_own_music(value['text']): + # Asked for no music, and no score of the user's own: the score section says so whatever the rewrite wrote. + text = re.sub(r'^non_diegetic_music:.*\Z', 'non_diegetic_music: N/A', text, flags=re.M | re.S) return dialogue(text, value['text']) diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index cb8bc022..280e6d3f 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,31 +1,80 @@ { "schema": 1, - "product_version": "0.4.28", + "product_version": "0.4.29", "en": { - "summary": "Image generation, editing and character generation accept LoRAs, the image model stays on the card longer for people who mostly make pictures, and Repair replaces Python packages whose files are missing.", + "summary": "Music can be generated on the second audio track from the generation bar (not on Mac), scenes can be added to the character library, long videos can be extended by typing a length in seconds, and Autopilot chat continues a failed or stopped video from where it left off.", "changes": [ - "Image generation: image generation, image editing and character generation each have a LoRA control. Up to 8 LoRAs can be added, each with a file, a strength and a trigger word that is added to the prompt. Video and image LoRAs are told apart and a wrong choice is explained in one sentence. Without a LoRA the output is unchanged.", - "Speed: for people who mostly make pictures, the image model stays loaded for up to 10 minutes when it fits on the card, so the next picture starts sooner. It gives the card back at once when a video or another program needs it.", - "Speed: the usage habit that decides which model parts stay in memory (only the kind of each operation and its count, nothing from prompts or images) is now kept across restarts. When prompts change often, the text encoder is loaded while the engine is idle, as far as free video and system memory allow, and stops when a new job starts.", - "Speed: after the engine starts, the model files for the first request are read ahead as far as free memory allows: the video model for people who mostly generate again with the same prompt and images, and the text encoder otherwise. A new request or a busy queue stops the reading, and a machine with little free memory reads less.", - "Install: Repair now reinstalls Python packages whose files are missing or cut short, for example a numpy library that has gone missing, which setup did not notice before when the package was already at the required version; the repair review counts them in the download. When the hardware check before a request fails to load such a package, the message names the missing file and offers Repair.", - "Windows: on machines with little memory, a long video whose second pass stops for lack of system memory now retries in a fresh process, keeping the finished first pass, and can finish instead of failing.", - "Guide: the tutorial book covers the additions of 0.4.25 and 0.4.26 and has a new chapter on replacing a person, with a matching guide card. In the Chinese book, quoted interface names of up to eight characters and numbers with their units are no longer split across lines." + "Editor: music can be made from the generation bar by dragging a range on the second audio track, which new edits now show by default. Choose a style and a mood and optionally describe the music; if the music model is not installed yet, it is downloaded first. A selected generated piece can be regenerated in place, and one undo restores the previous piece. Generated music is lowered while someone speaks; this can be turned off with Lower during speech in the clip properties. Not available on Mac.", + "Editor: the timeline scrolls vertically when the second audio track, text or subtitle rows do not fit, with the ruler kept in place.", + "Character library: scenes can be added from your own pictures or from several AI candidates, and mentioned with @ in the prompt; the time of day, weather, light and tone written in the prompt are kept in the generated video.", + "Long videos: when an ungenerated draft part or a range on the picture track is selected, the generation bar shows a Duration field in whole seconds. It follows the same rules as dragging the right edge (whole seconds, at least 10 s per part, up to 8 parts); a value out of range is clamped with a short note, and one undo restores the previous length.", + "Long videos: joining parts no longer fails at the last step when one part differs from the first in size or encoding; such a part is now matched to the first part before the parts are joined. When that is not possible, the message names the part and the property that differs.", + "Long videos: a video that already has the most parts a continuous shot can have now says so as soon as it is chosen to be extended, and the limit is checked before generation starts.", + "Long videos: the last continuation part, including a single added part, can be as short as 2 s; parts in the middle still last at least 5 s.", + "Autopilot: Continue generating on a failed or stopped video picks up the same request and keeps the parts already made, also when the Agent is offline. A failed picture is submitted again with the original request. The latest message can be edited and sent again; unfinished tasks after it are stopped first.", + "Agent: when the Connect an agent card cannot get an address, it explains the reason, for example when the page was opened from another device or by machine name; the feedback report includes the status code and error code.", + "Prompt enhancement: when the Agent is asked to improve the prompt before generating, the generation uses the improved text exactly, and is no longer treated as a different prompt because quotation marks were changed while the text was copied.", + "Prompts: asking for no background music in the scene text now works when the prompt still has the starter prompt's music section: the unchanged starter score is replaced with N/A, and a rewritten prompt ends with N/A. A music section written by the user is kept.", + "Speed: the background compile also prepares portrait videos and videos with a reference picture, so their first request no longer compiles GPU programs at the first step of each pass. The one-time background compile takes about 80 seconds longer; results are unchanged.", + "Install: a damaged offline package is now reported by name with a request to download it again, instead of a raw decompression error.", + "Install: downloads over an unstable connection now continue from the part already downloaded instead of stopping after the first round.", + "Install: on Windows, setup now waits and creates the Python environment again when another program, such as security software, holds its Python file open, instead of failing at the first attempt.", + "Install: when Windows refuses to run FreeVideo's Python file, the failure message now says so in any system language and points to the security software.", + "Updates: a downloaded source update is verified against the release signature again every time the launcher starts, not only when it is downloaded.", + "Diagnostics: the diagnostic log times the video worker's start-up steps." ] }, "zh": { - "summary": "图片生成、改图和角色生成可以使用 LoRA；以做图为主时，图片模型在显存中保留更久；“修复”会重新安装文件缺失的 Python 包。", + "summary": "可以在第二条声音轨上用生成栏生成配乐（暂不支持 Mac）；角色库可以新建场景；长视频可以直接填写秒数来接着生成；AI 全自动对话中失败或停止的视频可以从中断处继续生成。", "changes": [ - "图片生成：图片生成、改图和角色生成各增加了 LoRA 控件。最多可添加 8 个 LoRA，每个可选择文件、调整强度，触发词会加入提示词。视频和图片的 LoRA 分开识别，选错时用一句话说明。不使用 LoRA 时输出不变。", - "速度：以做图为主的用户，图片模型在能完整放进显存时最多保留 10 分钟，下一张图开始更快；视频或其他程序需要显存时立即让出。", - "速度：决定哪些模型部分驻留内存的操作习惯（只记录每次操作的类型和次数，不含提示词和图片的任何内容）现在会跨重启保存。经常更换提示词时，引擎空闲期间会预先载入文本编码器，只载入空闲显存和内存放得下的部分，新任务开始即停止。", - "速度：引擎启动后，按空闲内存的允许范围预先读入第一次请求要用的模型文件：以相同提示词和图片反复生成为主时读视频模型，其余情况读文本编码器。新请求到来或队列忙碌时停止读取，空闲内存较少的机器读得更少。", - "安装：“修复”现在会重新安装文件缺失或被截短的 Python 包（例如 numpy 的某个库文件丢失）；此前包已是所需版本时，这类问题安装流程检测不到。修复审阅页会把它们计入下载量。请求前的硬件检测因此类包无法加载而失败时，失败信息会点名缺失的文件并提供“修复”。", - "Windows：内存较小的机器生成长视频时，第二遍因系统内存不足中断后，会保留已完成的第一遍，改在新进程里重试，可以继续完成，而不是整次失败。", - "教程：教程书补充了 0.4.25 和 0.4.26 新增功能的说明，并新增“替换人物”一章和对应的导览卡。中文教程书中，八个字以内的带引号界面名称以及数字与单位，不再被拆在两行。" + "剪辑视图：在第二条声音轨上拖出一段，即可用生成栏生成配乐；新剪辑默认显示这条声音轨。可选择风格和情绪，也可另写描述；配乐模型未安装时会先下载。选中已生成的配乐可以原位重新生成，一次撤销即可恢复原来那段。生成的配乐在有人说话处自动降低音量，可在片段属性中取消“对白时降低音量”。暂不支持 Mac。", + "剪辑视图：第二条声音轨、文字或字幕行放不下时，时间线可以纵向滚动，标尺保持固定。", + "角色库：可以用自己的图片或从 AI 生成的多张候选中新建场景，并在提示词中用 @ 引用；提示词里写的时间、天气、光线和色调会保留到生成的视频中。", + "长视频：选中未生成的草稿段或画面轨上的一段范围时，生成栏显示以整秒为单位的“时长”输入框。规则与拖动右边缘相同（整秒、每段至少 10 秒、最多 8 段）；超出范围时自动取到边界并给出简短提示，一次撤销即可恢复原来的时长。", + "长视频：某一段的尺寸或编码与第 1 段不同时，不再在最后一步接合失败：接合前会先把这样的段转换为与第 1 段一致；无法转换时，提示会指明是哪一段、哪一项不同。", + "长视频：已达到连续镜头最多段数的视频，选中接着生成时就会说明原因，生成前也会检查。", + "长视频：接着生成时，最后一个接续段（包括只补一段的情况）最短可为 2 秒，中间的接续段仍至少 5 秒。", + "AI 全自动：视频失败或被停止后，点“继续生成”会按原请求接着做，保留已完成的段，Agent 不在线时同样可用。图片失败时按原请求重新提交。最后一条消息可以修改后重新发送，其后未完成的任务会先停止。", + "Agent：“连接 Agent”卡片取不到地址时，会说明原因（例如从其他设备或用机器名打开页面）；反馈报告附带状态码和错误码。", + "提示词增强：让 Agent 先润色提示词再生成时，生成使用的就是润色后的原文，不会因抄写时引号被改动而被当作另一段提示词。", + "提示词：在场景文字里要求不要背景音乐，而提示词仍带着默认示例的配乐段时，这一要求现在会生效：没改过的默认配乐会被换成 N/A，提示词重写的结果也以 N/A 结尾。用户自己写的配乐段保持不变。", + "速度：后台预编译也准备竖向视频和带参考图的视频，它们的第一次请求不再在每遍的第一步现场编译 GPU 程序。一次性的后台编译多约 80 秒，结果不变。", + "安装：离线包损坏时，现在会指出文件名并提示重新下载，不再显示解压缩的原始错误。", + "安装：网络不稳定时，下载会接着已下载的部分继续，不再在第一轮失败后直接停止。", + "安装：在 Windows 上，Python 文件被其他程序（例如安全软件）占用时，安装程序会等待并重新创建 Python 环境，不再第一次失败就停止。", + "安装：Windows 拒绝运行 FreeVideo 的 Python 文件时，失败提示会说明原因并指向安全软件，不再受系统语言影响。", + "更新：已下载的源码更新在启动器每次启动时都会再按发布签名校验，不只在下载时校验。", + "诊断：诊断日志为视频 worker 的启动各步计时。" ] }, "history": [ + { + "product_version": "0.4.28", + "en": { + "summary": "Image generation, editing and character generation accept LoRAs, the image model stays on the card longer for people who mostly make pictures, and Repair replaces Python packages whose files are missing.", + "changes": [ + "Image generation: image generation, image editing and character generation each have a LoRA control. Up to 8 LoRAs can be added, each with a file, a strength and a trigger word that is added to the prompt. Video and image LoRAs are told apart and a wrong choice is explained in one sentence. Without a LoRA the output is unchanged.", + "Speed: for people who mostly make pictures, the image model stays loaded for up to 10 minutes when it fits on the card, so the next picture starts sooner. It gives the card back at once when a video or another program needs it.", + "Speed: the usage habit that decides which model parts stay in memory (only the kind of each operation and its count, nothing from prompts or images) is now kept across restarts. When prompts change often, the text encoder is loaded while the engine is idle, as far as free video and system memory allow, and stops when a new job starts.", + "Speed: after the engine starts, the model files for the first request are read ahead as far as free memory allows: the video model for people who mostly generate again with the same prompt and images, and the text encoder otherwise. A new request or a busy queue stops the reading, and a machine with little free memory reads less.", + "Install: Repair now reinstalls Python packages whose files are missing or cut short, for example a numpy library that has gone missing, which setup did not notice before when the package was already at the required version; the repair review counts them in the download. When the hardware check before a request fails to load such a package, the message names the missing file and offers Repair.", + "Windows: on machines with little memory, a long video whose second pass stops for lack of system memory now retries in a fresh process, keeping the finished first pass, and can finish instead of failing.", + "Guide: the tutorial book covers the additions of 0.4.25 and 0.4.26 and has a new chapter on replacing a person, with a matching guide card. In the Chinese book, quoted interface names of up to eight characters and numbers with their units are no longer split across lines." + ] + }, + "zh": { + "summary": "图片生成、改图和角色生成可以使用 LoRA；以做图为主时，图片模型在显存中保留更久；“修复”会重新安装文件缺失的 Python 包。", + "changes": [ + "图片生成：图片生成、改图和角色生成各增加了 LoRA 控件。最多可添加 8 个 LoRA，每个可选择文件、调整强度，触发词会加入提示词。视频和图片的 LoRA 分开识别，选错时用一句话说明。不使用 LoRA 时输出不变。", + "速度：以做图为主的用户，图片模型在能完整放进显存时最多保留 10 分钟，下一张图开始更快；视频或其他程序需要显存时立即让出。", + "速度：决定哪些模型部分驻留内存的操作习惯（只记录每次操作的类型和次数，不含提示词和图片的任何内容）现在会跨重启保存。经常更换提示词时，引擎空闲期间会预先载入文本编码器，只载入空闲显存和内存放得下的部分，新任务开始即停止。", + "速度：引擎启动后，按空闲内存的允许范围预先读入第一次请求要用的模型文件：以相同提示词和图片反复生成为主时读视频模型，其余情况读文本编码器。新请求到来或队列忙碌时停止读取，空闲内存较少的机器读得更少。", + "安装：“修复”现在会重新安装文件缺失或被截短的 Python 包（例如 numpy 的某个库文件丢失）；此前包已是所需版本时，这类问题安装流程检测不到。修复审阅页会把它们计入下载量。请求前的硬件检测因此类包无法加载而失败时，失败信息会点名缺失的文件并提供“修复”。", + "Windows：内存较小的机器生成长视频时，第二遍因系统内存不足中断后，会保留已完成的第一遍，改在新进程里重试，可以继续完成，而不是整次失败。", + "教程：教程书补充了 0.4.25 和 0.4.26 新增功能的说明，并新增“替换人物”一章和对应的导览卡。中文教程书中，八个字以内的带引号界面名称以及数字与单位，不再被拆在两行。" + ] + } + }, { "product_version": "0.4.27", "en": { diff --git a/freevideo_engine/resident_worker.py b/freevideo_engine/resident_worker.py index bd1b01a0..55467acb 100644 --- a/freevideo_engine/resident_worker.py +++ b/freevideo_engine/resident_worker.py @@ -38,6 +38,13 @@ def run(message): # (see worker.py): the session has taken this request. print(json.dumps({'event': 'worker_start', 'pid': os.getpid(), 'epoch': time.time(), 'resident': True}), flush=True) + + def stage(name): + # The video worker's start-up has no other timing: a cold first + # request spent 49 s here before the model began to load. + if module == 'freevideo_engine.worker': + print(json.dumps({'event': 'worker_stage', 'stage': name, + 'seconds': time.perf_counter() - started, 'epoch': time.time()}), flush=True) metrics = None diagnostics = None try: @@ -56,16 +63,19 @@ def run(message): bank = ModelBank() import torch torch.set_grad_enabled(False) + stage('torch_import') if diagnostics is not None: diagnostics.stage('worker_cuda_setup') # An explicit benchmark cap belongs to its request. A cached # process must not silently carry that cap into a later job. torch.cuda.set_per_process_memory_fraction(1.) torch.cuda.reset_peak_memory_stats() + stage('cuda_setup') from .processes import worker_signals with worker_signals(): if module.endswith('.worker'): from .worker import generate + stage('engine_import') bank.configure(request['gpu_budget_bytes'], request.get('ram_budget_bytes')) generate(request, resident=bank) # Only a fully returned video is a success, including its diff --git a/freevideo_engine/source_update.py b/freevideo_engine/source_update.py index 292ac469..2ccab59f 100644 --- a/freevideo_engine/source_update.py +++ b/freevideo_engine/source_update.py @@ -17,14 +17,7 @@ def signed(value): - try: - signature = bytes.fromhex(value.get('signature', '')) - payload = DOMAIN + source.canonical({k: v for k, v in value.items() if k != 'signature'}) - return (keys.ED25519_RELEASE_KEY_ENABLED and - keys.ED25519_PUBLIC_KEY != keys.PLACEHOLDER_PUBLIC_KEY and - ed25519.verify(keys.ED25519_PUBLIC_KEY, payload, signature)) - except (TypeError, ValueError): - return False + return source.signed(value) def metadata_name(candidate): diff --git a/freevideo_engine/speech.py b/freevideo_engine/speech.py new file mode 100644 index 00000000..0a8fd041 --- /dev/null +++ b/freevideo_engine/speech.py @@ -0,0 +1,279 @@ +"""Where someone speaks in a sound file, so generated music can sit lower under dialogue. + +Silero VAD (snakers4/silero-vad, MIT; the onnx-community export) runs on the CPU in torch: the 16 kHz +weights are read straight from the ONNX file, so no ONNX runtime is needed. The file is downloaded with the +music model. One pass over a file gives speech spans in seconds of that file; they are cached next to the +editor's other derived data, and the preview and the export read the same spans. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import threading + +import numpy as np + +MODEL_FILE = 'music/silero-vad/model.onnx' +VERSION = 1 # change when the spans for the same file would change +RATE = 16000 +WINDOW, CONTEXT = 512, 64 +ENTER, LEAVE = .5, .35 # Silero's own hysteresis +MIN_SPEECH, MIN_SILENCE, PAD = .25, .1, .03 +# Music under dialogue stays down across short pauses between sentences instead of pumping. +JOIN_GAP = .6 +LIMIT_SECONDS = 4 * 3600 + + +class SpeechError(Exception): + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +# --- a small protobuf reader: only what an ONNX file's tensors need ----------------------------------------- +def _varint(data, index): + shift = value = 0 + while True: + byte = data[index] + index += 1 + value |= (byte & 0x7f) << shift + if byte < 0x80: + return value, index + shift += 7 + + +def _fields(data): + index = 0 + while index < len(data): + key, index = _varint(data, index) + number, wire = key >> 3, key & 7 + if wire == 0: + value, index = _varint(data, index) + elif wire == 1: + value, index = data[index:index + 8], index + 8 + elif wire == 2: + size, index = _varint(data, index) + value, index = data[index:index + size], index + size + elif wire == 5: + value, index = data[index:index + 4], index + 4 + else: + raise SpeechError('model') + yield number, wire, value + + +def _tensor(data): + dims, kind, name, raw = [], None, '', None + for number, wire, value in _fields(data): + if number == 1: + if wire == 2: + index = 0 + while index < len(value): + dim, index = _varint(value, index) + dims.append(dim) + else: + dims.append(value) + elif number == 2: + kind = value + elif number == 8: + name = bytes(value).decode() + elif number == 9: + raw = value + if kind != 1 or raw is None or len(raw) != 4 * int(np.prod(dims or [1])): + return name, None + return name, np.frombuffer(bytes(raw), '= 2] + try: + weights = {'basis': found[prefix + 'stft.forward_basis_buffer']} + for layer in range(4): + weights[f'conv{layer}.weight'] = found[f'{prefix}encoder.{layer}.reparam_conv.weight'] + weights[f'conv{layer}.bias'] = found[f'{prefix}encoder.{layer}.reparam_conv.bias'] + weights['out.weight'] = found[prefix + 'decoder.decoder.2.weight'] + weights['out.bias'] = found[prefix + 'decoder.decoder.2.bias'] + w, r = [array[0] for array in lstm if array.shape == (1, 512, 128)] + (b,) = [array[0] for array in lstm if array.shape == (1, 1024)] + except (KeyError, ValueError): + raise SpeechError('model') from None + # ONNX gates are input, output, forget, cell; torch's are input, forget, cell, output. + order = lambda x: np.concatenate([x[0:128], x[256:384], x[384:512], x[128:256]]) + weights.update({'lstm.weight_ih_l0': order(w), 'lstm.weight_hh_l0': order(r), + 'lstm.bias_ih_l0': order(b[:512]), 'lstm.bias_hh_l0': order(b[512:])}) + if weights['basis'].shape != (258, 1, 256) or weights['conv0.weight'].shape != (128, 129, 3): + raise SpeechError('model') + return weights + + +def _model(weights): + import torch + from torch import nn + + class Model(nn.Module): + def __init__(self): + super().__init__() + self.register_buffer('basis', torch.zeros(258, 1, 256)) + self.conv0 = nn.Conv1d(129, 128, 3, padding=1) + self.conv1 = nn.Conv1d(128, 64, 3, stride=2, padding=1) + self.conv2 = nn.Conv1d(64, 64, 3, stride=2, padding=1) + self.conv3 = nn.Conv1d(64, 128, 3, padding=1) + self.lstm = nn.LSTM(128, 128, batch_first=True) + self.out = nn.Conv1d(128, 1, 1) + + def forward(self, windows): + """[n, 576] windows (64 context + 512 new samples) in order → [n] speech probabilities.""" + x = nn.functional.pad(windows, (0, 64), mode='reflect').unsqueeze(1) + x = nn.functional.conv1d(x, self.basis, stride=128) + x = torch.sqrt(x[:, :129] ** 2 + x[:, 129:] ** 2) + for conv in (self.conv0, self.conv1, self.conv2, self.conv3): + x = torch.relu(conv(x)) + x, _ = self.lstm(x.squeeze(-1).unsqueeze(0)) + x = torch.sigmoid(self.out(torch.relu(x[0]).unsqueeze(-1))) + return x[:, 0, 0] + + model = Model() + model.load_state_dict({name: torch.from_numpy(array) for name, array in weights.items()}) + return model.eval() + + +_LOADED = {} +_LOCK = threading.Lock() + + +def probabilities(samples, model_path): + """float32 mono samples at 16 kHz → one speech probability per 512 samples (32 ms).""" + import torch + with _LOCK: + key = str(model_path) + if key not in _LOADED: + _LOADED.clear() + _LOADED[key] = _model(onnx_weights(model_path)) + model = _LOADED[key] + samples = np.asarray(samples, np.float32) + count = -(-len(samples) // WINDOW) + if count == 0: + return np.zeros(0, np.float32) + padded = np.zeros(CONTEXT + count * WINDOW, np.float32) + padded[CONTEXT:CONTEXT + len(samples)] = samples + windows = np.lib.stride_tricks.sliding_window_view(padded, CONTEXT + WINDOW)[::WINDOW][:count] + threads = torch.get_num_threads() + try: + torch.set_num_threads(min(threads, 4)) + with torch.inference_mode(): + return model(torch.from_numpy(np.ascontiguousarray(windows))).numpy() + finally: + torch.set_num_threads(threads) + + +def spans(probs, duration): + """Speech probabilities → merged [start, end] seconds, with Silero's hysteresis and padding.""" + step = WINDOW / RATE + found, start, quiet = [], None, None + for index, value in enumerate(probs): + at = index * step + if start is None: + if value >= ENTER: + start, quiet = at, None + elif value < LEAVE: + quiet = at if quiet is None else quiet + if at + step - quiet >= MIN_SILENCE: + found.append([start, quiet]) + start, quiet = None, None + else: + quiet = None + if start is not None: + found.append([start, len(probs) * step]) + joined = [] + for begin, end in found: + if end - begin < MIN_SPEECH: + continue + begin, end = max(0., begin - PAD), min(duration, end + PAD) + if joined and begin - joined[-1][1] <= JOIN_GAP: + joined[-1][1] = max(joined[-1][1], end) + else: + joined.append([begin, end]) + return [[round(begin, 3), round(end, 3)] for begin, end in joined] + + +def decode(path, cancelled=None): + """The first audio stream of a media file as float32 mono at 16 kHz.""" + import av + parts, total = [], 0 + try: + with av.open(str(path)) as container: + if not container.streams.audio: + return np.zeros(0, np.float32) + stream = container.streams.audio[0] + resampler = av.AudioResampler(format='flt', layout='mono', rate=RATE) + for frame in container.decode(stream): + for out in resampler.resample(frame): + part = out.to_ndarray().reshape(-1) + parts.append(part) + total += len(part) + if total > LIMIT_SECONDS * RATE: + raise SpeechError('too_long') + if cancelled is not None and cancelled(): + raise SpeechError('cancelled') + for out in resampler.resample(None): + parts.append(out.to_ndarray().reshape(-1)) + except av.error.FFmpegError: + raise SpeechError('unreadable') from None + return np.concatenate(parts).astype(np.float32) if parts else np.zeros(0, np.float32) + + +def _identity(path, model_path): + stat, model = Path(path).stat(), Path(model_path).stat() + raw = json.dumps([str(Path(path).resolve()), stat.st_size, stat.st_mtime_ns, model.st_size, VERSION]) + return hashlib.sha256(raw.encode()).hexdigest()[:32] + + +def cached(path, model_path, cache): + """The cached spans for this exact file and model, or None.""" + try: + value = json.loads((Path(cache) / (_identity(path, model_path) + '.json')).read_text(encoding='utf-8')) + return value['spans'] if isinstance(value, dict) and isinstance(value.get('spans'), list) else None + except (OSError, ValueError, KeyError): + return None + + +def speech_spans(path, model_path, cache, cancelled=None): + """Speech spans in seconds of `path`; computed once per file and model, then read from `cache`.""" + found = cached(path, model_path, cache) + if found is not None: + return found + if not Path(model_path).is_file(): + raise SpeechError('model_missing') + samples = decode(path, cancelled) + found = spans(probabilities(samples, model_path), len(samples) / RATE) + folder = Path(cache) + folder.mkdir(parents=True, exist_ok=True) + target = folder / (_identity(path, model_path) + '.json') + temporary = target.with_name(f'{target.name}.{os.getpid()}.{threading.get_ident()}.tmp') + temporary.write_text(json.dumps({'spans': found}), encoding='utf-8') + os.replace(temporary, target) + return found diff --git a/freevideo_engine/support_report.py b/freevideo_engine/support_report.py index 5211703a..ddd114f8 100644 --- a/freevideo_engine/support_report.py +++ b/freevideo_engine/support_report.py @@ -507,6 +507,13 @@ def count(value): agent_poll.get('agent_generation_jobs')) if stalled: payload['agent_generation_stalls'] = stalled + try: + from .comfy_host_stats import host_fields + host = host_fields() + except Exception: + host = None + if host: + payload['comfy_host'] = host try: if preview: previous, previous_engine, previous_encoding = preview diff --git a/freevideo_source_bootstrap.py b/freevideo_source_bootstrap.py index b05644f3..fcc4c466 100644 --- a/freevideo_source_bootstrap.py +++ b/freevideo_source_bootstrap.py @@ -20,6 +20,7 @@ import freevideo_ed25519 import freevideo_update_keys +DOMAIN = b'FreeVideo source manifest v1\x00' MAX_MANIFEST = 2 * 2**20 MAX_FILES = 4096 MAX_FILE = 32 * 2**20 @@ -260,6 +261,18 @@ def compatible(value, base): (('target', 'windows-x86_64'), ('channel', None), ('packaging', None), ('track', 'stable')))) +def signed(value): + """True only for a manifest carrying a valid release signature from the frozen key.""" + try: + signature = bytes.fromhex(value.get('signature', '')) + payload = DOMAIN + canonical({k: v for k, v in value.items() if k != 'signature'}) + return (freevideo_update_keys.ED25519_RELEASE_KEY_ENABLED and + freevideo_update_keys.ED25519_PUBLIC_KEY != freevideo_update_keys.PLACEHOLDER_PUBLIC_KEY and + freevideo_ed25519.verify(freevideo_update_keys.ED25519_PUBLIC_KEY, payload, signature)) + except (AttributeError, TypeError, ValueError): + return False + + def overlay_name(value): return value['build']['product_version'] + '-' + value['identity'][:16] @@ -272,6 +285,9 @@ def validate_overlay(folder, base): if (receipt.get('sha256') != sha(canonical(value)) or not compatible(value, base) or folder.name != overlay_name(value)): raise ValueError('Overlay receipt changed') + # The receipt hash is self-made; only the release signature ties the manifest to the publisher. + if not signed(value): + raise ValueError('Overlay manifest is not signed') changed = {n: r for n, r in value['files'].items() if base['files'].get(n) != r} verify_tree(folder / 'files', changed) return value @@ -318,6 +334,8 @@ def prepare(value, previous, bundle, current_source, archive, home, base): """Validate a single hop, then save independent cumulative sparse overlay.""" value, previous = manifest(value), manifest(previous) delta = value.get('delta') or {} + if not signed(value): + raise ValueError('Source manifest is not signed') if (not compatible(value, base) or delta.get('from') != previous['identity'] or (not value['gui'] and value.get('launcher_files_changed', True))): raise ValueError('Incompatible source update') diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index f7c357c6..891d3781 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -520,8 +520,10 @@ test('main browser entry registers both views and preserves node hooks', async t await extension.beforeRegisterNodeDef(MediaNode, {name: 'FreeVideoMedia'}); const media = new MediaNode(); media.onNodeCreated(); const panel = media.dom.find(row => row.name === 'freevideo_media').element; - const [castRow, toolbar, mode, note, audioHelp, connections] = panel.children; + const [castRow, sceneRow, toolbar, mode, note, audioHelp, connections] = panel.children; assert.equal(castRow, casts[0].element); + assert.equal(sceneRow, casts[1].element); + assert.equal(casts[1].options.kind, 'setting'); assert.equal(casts[0].options.api, api); assert.equal(typeof casts[0].options.uploadOne, 'function'); assert.equal(typeof casts[0].options.viewURL, 'function'); @@ -558,10 +560,11 @@ test('main browser entry registers both views and preserves node hooks', async t await imagePicker.onchange(); assert.deepEqual(JSON.parse(media.widgets[0].value).map(row => row.role), ['reference', 'first']); media.onRemoved(); - assert.equal(casts[0].destroyed, true); + assert.equal(casts[0].destroyed, true); assert.equal(casts[1].destroyed, true); const mounted = new Element('mount'), disposeMedia = media.freevideoCreateMediaEditor(mounted); - assert.equal(mounted.children[0].children[0], casts[1].element); - disposeMedia(); assert.equal(casts[1].destroyed, true); + assert.equal(mounted.children[0].children[0], casts[2].element); + assert.equal(mounted.children[0].children[1], casts[3].element); + disposeMedia(); assert.equal(casts[2].destroyed, true); assert.equal(casts[3].destroyed, true); await t.test('Generate node removal preserves the original hook when setSize throws', async () => { const previousInterval = globalThis.setInterval, previousClear = globalThis.clearInterval; diff --git a/web/agent_connect.js b/web/agent_connect.js index 605438f1..e1fe3ecd 100644 --- a/web/agent_connect.js +++ b/web/agent_connect.js @@ -23,6 +23,15 @@ export async function agentInterfaceState(api) { } catch { return 'unknown'; } } +// What to tell the user when the address lookup is refused: only refusals that say what to do are named; any other +// failure keeps the plain "could not get address". +export function refusalHint(code, t) { + if (code === 'origin_forbidden') return t('Open FreeVideo on this computer with a localhost address, then retry.', '请在本机使用 localhost 地址打开 FreeVideo，然后重试。'); + if (code === 'local_only') return t('Agents can only be connected on the computer that runs FreeVideo.', '只能在运行 FreeVideo 的电脑上连接 Agent。'); + if (code === 'not_ready') return t('The agent connection service is not ready. Restart FreeVideo, then retry.', 'Agent 连接服务尚未就绪，请重启 FreeVideo 后重试。'); + return ''; +} + export async function agentInterfaceEnabled(api) { return await agentInterfaceState(api) === 'on'; } const ICONS = {plug: ['M9 7V3M15 7V3M7 7h10v4a5 5 0 0 1-10 0V7zM12 16v5', 1.8], copy: ['M11 9h7a2 2 0 0 1 2 2v7a2 2 0 0 1-2 2h-7a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2zM5 15V5a2 2 0 0 1 2-2h8', 2]}; @@ -56,17 +65,17 @@ export function agentConnectCard(t, api, shown = () => true, changed = () => {}) const status = el('p', 'fv-agent-connect-status', t('Getting the connection address…', '正在获取连接地址…')); status.setAttribute('role', 'status'); body.append(mark, title, intro, subtitle, actions, sentence, status); - let feedback = false, text = '', failures = 0; + let feedback = false, text = '', failures = 0, refusal = null; let timer, feedbackTimer, controller, generation = 0; const active = () => body.isConnected && shown(); const render = () => { label.textContent = !text && failures >= 3 ? t('Could not get address. Retry', '获取失败，重试') : feedback ? t('Copied', '已复制') : t('Copy', '复制'); if (!text && failures >= 3) { - if (!issue) { issue = issueButton(t, () => ({where: ['Agent connection', 'Agent 连接'], message: label.textContent})); button.after(issue); } + if (!issue) { issue = issueButton(t, () => ({where: ['Agent connection', 'Agent 连接'], message: refusal ? `${label.textContent} (HTTP ${refusal.status}${refusal.code ? ' ' + refusal.code : ''})` : label.textContent, kind: refusal?.code || null})); button.after(issue); } } else if (issue) { issue.remove(); issue = null; } button.disabled = !text && failures < 3; button.title = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : ''; - status.hidden = !!text; status.textContent = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : ''; + status.hidden = !!text; status.textContent = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : !text ? refusalHint(refusal?.code, t) : ''; }; render(); @@ -92,15 +101,18 @@ export function agentConnectCard(t, api, shown = () => true, changed = () => {}) // Only the sentence is read here, until it is known; who is connected comes from the surface that holds the card. async function load() { const current = generation; - const request = new AbortController(); controller = request; + const request = new AbortController(); controller = request; refusal = null; try { // The sentence carries no parameter: the server gives the agent the language this card showed. const response = await api.fetchApi('/freevideo/agent/connect?lang=' + t('en', 'zh'), {cache: 'no-store', signal: request.signal}); - const connect = response.ok ? await response.json() : null; + const connect = await response.json().catch(() => null); if (current !== generation) return; - if (!connect?.address) throw new Error('Missing address'); + if (!response.ok || !connect?.address) { + refusal = !response.ok ? {status: response.status, code: String(connect?.error?.code || '').slice(0, 40) || null} : null; + throw new Error('Missing address'); + } text = connectionSentence(connect.address, t); - failures = 0; render(); + failures = 0; refusal = null; render(); if (await state === 'unknown') { state = agentInterfaceState(api); changed(await state); } } catch { if (current === generation) { failures++; render(); } } finally { diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index 8170657a..4b27c035 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -202,3 +202,18 @@ .fv-chat-refs{width:max-content;max-width:100%;display:flex;flex-wrap:wrap;gap:6px;justify-content:flex-end;margin:0 0 6px auto;padding:0;list-style:none} .fv-chat-refs>li{width:96px;height:96px;border-radius:10px;overflow:hidden} @media(max-width:560px){.fv-chat-refs>li{width:64px;height:64px}} + +/* The newest sentence can be rewritten in place. Its Edit row keeps its height, so showing it moves nothing. */ +.fv-chat-edit-row{display:flex;justify-content:flex-end;min-height:28px;margin-top:4px} +.fv-chat-edit-row[hidden]{display:none} +.fv-studio .fv-chat-log .fv-chat-edit{min-height:0;padding:4px 8px;border:0;border-radius:7px;background:none;color:var(--fv-muted);font-size:12.5px;font-weight:var(--fv-regular);opacity:0;transition:opacity var(--fv-quick)} +.fv-studio .fv-chat-log .fv-chat-edit:hover:not(:disabled){background:var(--fv-raised);color:var(--fv-ink)} +.fv-chat-me:is(:hover,:focus-within) .fv-chat-edit,.fv-studio .fv-chat-log .fv-chat-edit:focus-visible{opacity:1} +@media(hover:none),(max-width:480px){.fv-studio .fv-chat-log .fv-chat-edit{opacity:1}} +.fv-chat-me[data-editing=true]{width:min(500px,88%)} +.fv-studio .fv-chat-log .fv-chat-edit-box{box-sizing:border-box;display:block;width:100%;min-height:44px;max-height:40vh;padding:10px 14px;border:1px solid var(--fv-accent-dim);border-radius:14px;background:#1d3550;color:var(--fv-ink);font:inherit;font-size:14.5px;line-height:22px;resize:none;outline:none;overflow-y:auto} +.fv-chat-me>.fv-chat-edit-actions{margin-top:8px;justify-content:flex-end} +.fv-chat-edit-actions>.fv-chat-error{text-align:right} +.fv-studio .fv-chat-log .fv-chat-edit-send{border-color:transparent;background:var(--fv-accent);color:var(--fv-bg)} +.fv-studio .fv-chat-log .fv-chat-edit-send:hover:not(:disabled){background:var(--fv-accent-hover)} +@media(max-width:480px){.fv-studio .fv-chat-log .fv-chat-edit-box{font-size:16px}} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 373d6d85..fc6e7fe1 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -45,6 +45,8 @@ const WORKING = ['running', 'queued', 'waiting', 'cancelling'], VIDEOS = ['gener const QUIET = {vlm: ['Checking', '检查中'], enhance: ['Enhancing prompt', '增强提示词']}; // Other jobs without a card keep their line while they run, and a finished one still marks the tab. const BUSY = {music: ['Making music', '制作配乐']}; +// A failed card's one action: go on keeping the finished parts, run the same request again, or say the words again. +const AGAIN = {continue: ['Continue generating', '继续生成'], retry: ['Retry', '重试'], resend: ['Send again', '再发一次']}; const WHITEBOX = { generate: ['Generate video', '生成视频'], edit: ['Edit prompt', '改描述'], decline: ['Skip', '跳过'], approved: ['Approved', '已确认'], superseded: ['Replaced', '已被替换'], declined: ['Skipped', '已跳过'], @@ -825,13 +827,71 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () try { await send(text); } finally { retrying.delete(key); if (!stopped) render(); } } + // Why a card's job did not go on. A refusal turns the card to saying the words again; a lost connection keeps its button. + const fallbacks = new Map(); + const againWay = (job, key) => fallbacks.get(key)?.resend ? 'resend' : job.again || 'resend'; + async function resumeJob(key, job) { + if (retrying.has(key)) return; + retrying.add(key); fallbacks.delete(key); render(); + const lost = {reason: t('Connection lost. Try again.', '连接中断，请重试。'), resend: false}; + try { + const response = await dataLayer.resume(job.id); + if (!response?.ok) fallbacks.set(key, response ? {reason: failureReason({error: await sendError(response)}, t), resend: true} : lost); + } catch { fallbacks.set(key, lost); } + finally { retrying.delete(key); if (!stopped) { render(); report(); void poll(); } } + } + function againJob(key, job, text) { + if (againWay(job, key) === 'resend') { if (text) void retryJob(key, text); } + else void resumeJob(key, job); + } // ---- one entry of the conversation each ---- function message(role) { const item = el('li', role === 'user' ? 'fv-chat-me' : 'fv-chat-ai'), text = el('p'); item.append(text); let status = null, error = null, retry = null, report = null, shown = ''; - return {element: item, update(entry) { - text.textContent = entry.text; text.hidden = !entry.text && !!entry.attachments?.length; + let current = null, editRow = null, editButton = null, editor = null; + // The newest sentence opens in place; the box at the bottom keeps its draft. + function openEditor() { + if (editor || !current) return; + const box = el('textarea', 'fv-chat-edit-box'), note = el('span', 'fv-chat-error'); + const actions = el('div', 'fv-chat-actions fv-chat-edit-actions'); + const cancel = press('fv-chat-edit-cancel', t('Cancel', '取消'), () => closeEditor()); + const go = press('fv-chat-edit-send', t('Send', '发送'), () => void submit()); + note.setAttribute('role', 'alert'); note.hidden = true; + box.value = current.text; box.rows = 1; box.setAttribute('aria-label', t('Edit message', '编辑消息')); + actions.append(note, cancel, go); + editor = {box, actions, note, cancel, go, target: current, id: newId(), busy: false}; + const fit = () => { box.style.height = 'auto'; box.style.height = `${box.scrollHeight + 2}px`; go.disabled = editor.busy || !box.value.trim(); }; + box.addEventListener('input', fit); + listenDialogKey(box, 'keydown', event => { + if (event.key === 'Escape' && !editor.busy) { event.preventDefault(); event.stopPropagation(); closeEditor(); return; } + if (event.key !== 'Enter' || event.shiftKey || event.isComposing || event.keyCode === 229) return; + event.preventDefault(); void submit(); + }); + item.dataset.editing = 'true'; text.hidden = true; editRow.hidden = true; + text.after(box, actions); + fit(); box.focus(); box.setSelectionRange?.(box.value.length, box.value.length); + } + function closeEditor() { + if (!editor) return; + editor.box.remove(); editor.actions.remove(); editor = null; + delete item.dataset.editing; text.hidden = false; if (editRow) editRow.hidden = false; + if (current) text.textContent = current.text; + } + async function submit() { + const words = editor?.box.value.trim(); + if (!words || editor.busy) return; + const open = editor; + open.busy = open.cancel.disabled = open.go.disabled = true; open.note.hidden = true; + const problem = await rewrite(open.target, words, open.id); + if (editor !== open) return; + open.busy = open.cancel.disabled = false; open.go.disabled = false; + if (problem) { open.note.textContent = problem; open.note.hidden = false; open.box.focus(); } + else { current = {...current, text: words}; closeEditor(); } + } + return {element: item, update(entry, {editable = false} = {}) { + current = entry; + if (!editor) { text.textContent = entry.text; text.hidden = !entry.text && !!entry.attachments?.length; } const names = (entry.attachments || []).map(picture => picture.name).join('|'); if (names !== shown) { shown = names; item.querySelector('.fv-chat-refs')?.remove(); @@ -842,6 +902,12 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } } if (role !== 'user') return; + if ((editable || editor) && !editRow) { + editRow = el('div', 'fv-chat-edit-row'); editButton = press('fv-chat-edit', t('Edit', '编辑'), openEditor); + editRow.append(editButton); item.append(editRow); + } else if (!editable && !editor && editRow) { editRow.remove(); editRow = editButton = null; } + // A rewrite the server has not shown yet has no number to rewrite. + if (editButton) editButton.disabled = !composeAllowed() || !Number.isFinite(entry.seq); item.dataset.taken = String(entry.taken !== false); item.dataset.failed = String(!!entry.error); if (entry.error || entry.sending && entry.retried) { @@ -1003,13 +1069,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } function jobCard() { const item = el('li', 'fv-chat-job'); - let mode = null, apply = null; - return {element: item, update(job, {latest, retry, key}) { + let mode = null, apply = null, retry = null; + // The card's one button and, on its row, why the last press did not go on. + const again = () => [el('span', 'fv-chat-error fv-chat-again-note'), press('fv-chat-retry', null, () => retry?.())]; + return {element: item, update(job, {latest, retry: action, key, text}) { + retry = action; // Jobs without a picture (a check, a prompt) are one quiet line while they run, and nothing after. // A picture takes seconds: the same quiet line while it is made, then the picture. const card = VIDEOS.includes(job.kind || 'generate') || (['image', 'production', 'cast'].includes(job.kind) && !WORKING.includes(job.status)); + // What the user stopped says so; a video with finished parts can go on from there. + const byUser = jobMode(job) === 'stopped' && job.cancelled_by === 'user' && VIDEOS.includes(job.kind || 'generate'); const next = card ? jobMode(job) + (jobMode(job) === 'done' ? `:${job.play}:${job.thumbnail}:${job.work}` - : jobMode(job) === 'failed' && job.kind === 'image' ? `:${job.error?.code}:${job.error?.report_id}:${job.error?.memory || ''}` : '') + : jobMode(job) === 'failed' && job.kind === 'image' ? `:${job.error?.code}:${job.error?.report_id}:${job.error?.memory || ''}` + : byUser ? `:user:${retry && job.again === 'continue' ? 'continue' : ''}` : '') : WORKING.includes(job.status) ? 'quiet' : 'none'; if (next !== mode) { mode = next; item.dataset.job = job.id; item.dataset.mode = next.split(':')[0]; item.hidden = next === 'none'; @@ -1022,7 +1094,13 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () apply = next => { label.textContent = stopping.has(next.id) || next.status === 'cancelling' ? t('Stopping', '正在停止') : t(...(QUIET[next.kind] || BUSY[next.kind] || ['Working', '进行中'])); }; } - else if (next === 'stopped' && (!job.error || job.kind === 'image')) { item.replaceChildren(el('span', null, t('Stopped', '已停止'))); apply = null; } + else if (next === 'stopped:user:continue') { + // The failed card's place and button, without a report: nothing went wrong. + const actions = el('div', 'fv-chat-actions fv-actions'), links = el('div', 'fv-result-links'); + actions.append(...again()); links.append(actions); + item.replaceChildren(el('p', null, t('Stopped', '已停止')), links); apply = null; + } + else if (next === 'stopped:user:' || next === 'stopped' && (!job.error || job.kind === 'image')) { item.replaceChildren(el('span', null, t('Stopped', '已停止'))); apply = null; } else if (next.startsWith('failed') && job.kind === 'image') { // Picture failures name their step with the picture's own report card. const host = el('div'); @@ -1030,7 +1108,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () memory: job.error?.memory}, t, {productVersion, fetchApi: (path, options) => api.fetchApi(path, options)}); item.replaceChildren(host); - if (retry) item.append(press('fv-chat-retry', t('Retry', '重试'), retry)); + if (retry) { const row = el('div', 'fv-chat-actions'); row.append(...again()); item.append(row); } apply = null; } else if (next === 'failed' || next === 'stopped') { @@ -1041,14 +1119,23 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () fallback: () => new Blob([JSON.stringify(createErrorReport({error: current.error, exception_type: current.error?.code, details: current.error?.request_id}, {stage: 'execution'}), null, 2)], {type: 'application/json'})}); actions.append(diagnostic.report); - if (retry) actions.append(press('fv-chat-retry', t('Retry', '重试'), retry)); + if (retry) actions.append(...again()); links.append(actions); item.replaceChildren(reason, links); apply = next => { current = next; reason.textContent = failureReason(next, t); diagnostic.updateReport({report_id: next.report_id}); }; } else { item.replaceChildren(); apply = null; } } apply?.(job, latest); - const again = item.querySelector('.fv-chat-retry'); if (again) again.disabled = !sendingAllowed() || retrying.has(key); + if (WORKING.includes(job.status)) fallbacks.delete(key); + const button = item.querySelector('.fv-chat-retry'); + if (button) { + const way = againWay(job, key), note = item.querySelector('.fv-chat-again-note'); + button.textContent = t(...AGAIN[way]); + button.hidden = way === 'resend' && !text; + // Going on needs no agent; saying the words again does. + button.disabled = retrying.has(key) || !inSelectedChat() || way === 'resend' && !sendingAllowed(); + if (note) { note.textContent = fallbacks.get(key)?.reason || ''; note.hidden = !note.textContent; } + } }}; } function whiteboxState(entry) { @@ -1205,6 +1292,43 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } const MAKE = {hello, examples, dots, user: () => message('user'), agent: () => message('agent'), job: jobCard, confirm: confirmCard}; + // A rewritten sentence takes the place of the one it rewrites. What was said, asked and left unfinished for the old + // words folds away; finished videos and pictures stay where they were. The session itself keeps every entry. + let rewriting = null; + function rewritten(entries) { + const list = [...entries]; + if (rewriting && list.some(entry => entry.id === rewriting.id)) rewriting = null; + if (rewriting && rewriting.session === state.session) + list.push({type: 'message', role: 'user', id: rewriting.id, text: rewriting.text, revises: rewriting.revises, seq: Infinity, taken: false}); + const home = new Map(), latest = new Map(), folded = []; + for (const entry of list) if (entry.type === 'message' && entry.role === 'user' && Number.isFinite(entry.revises)) { + // A rewrite of a rewrite still shows at the first sentence's place. + const origin = home.get(entry.revises) ?? entry.revises; + home.set(entry.seq, origin); latest.set(origin, entry); folded.push([entry.revises, entry.seq]); + } + if (!folded.length) return list; + const kept = entry => entry.type === 'job' && state.jobs[entry.job]?.status === 'succeeded'; + return list.flatMap(entry => { + if (entry.type === 'message' && latest.has(entry.seq)) return [{...latest.get(entry.seq), slot: entry.id}]; + return folded.some(([from, to]) => entry.seq > from && entry.seq <= to) && !kept(entry) ? [] : [entry]; + }); + } + // Rewrites the newest sentence: the server stops what began after it and declines what it still asked, then says the new words. + async function rewrite(entry, text, id) { + const response = await dataLayer.send({id, text, session: state.session, revises: entry.seq}); + if (response?.ok) { + rewriting = {session: state.session, id, text, revises: entry.seq}; + if (!stopped) { render(true); report(); void poll(); } + return null; + } + const error = await sendError(response); + if (!stopped) void poll(); + // Another sentence came first: the page reads it, and this one stays in the box. + if (error?.code === 'invalid_request') return t('A newer message was sent first. This one was not sent.', '已有更新的消息，此⁠条⁠未⁠发⁠送⁠。'); + if (WAIT_FOR_AGENT.includes(error?.code) || error?.code === 'session_full') return t('Not sent: ', '没发出去：') + failureReason({error}, t); + return t('Could not stop. Click again.', '未能停止，请再点一次。'); + } + function render(follow = false) { whiteboxButton.title = t(...WHITEBOX_PREVIEW.name); whiteboxButton.setAttribute('aria-label', whiteboxButton.title); @@ -1217,15 +1341,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const wanted = []; if (!entries.length && !localPending.length) wanted.push(['hello', 'hello', null]); let lastUser = null; - for (const entry of entries) { + const shownEntries = rewritten(entries); + // Only the newest sentence the agent already has can be rewritten, and not while another waits to go. + const editable = localPending.length ? null : shownEntries.findLast(entry => entry.type === 'message' && entry.role === 'user'); + for (const entry of shownEntries) { if (entry.type === 'message') { if (entry.role === 'user') lastUser = entry; - wanted.push([entry.role === 'user' && entry.id ? `m:${entry.id}` : `s:${entry.seq}`, entry.role === 'user' ? 'user' : 'agent', entry]); + wanted.push([entry.role === 'user' && entry.id ? `m:${entry.slot || entry.id}` : `s:${entry.seq}`, entry.role === 'user' ? 'user' : 'agent', entry, + {editable: entry === editable}]); } else if (entry.type === 'job') { const job = state.jobs[entry.job]; if (!job) continue; const text = lastUser?.text; const key = cardKey(state.session, job.id); - wanted.push([`s:${entry.seq}`, 'job', job, {latest: entry === finished, key, retry: text ? () => retryJob(key, text) : null}]); + wanted.push([`s:${entry.seq}`, 'job', job, {latest: entry === finished, key, text, retry: () => againJob(key, job, text)}]); const failure = state.failures[entry.entity]; if (failure) wanted.push([`e:${entry.seq}`, 'job', failure]); } else if (entry.type === 'production' || entry.type === 'cast') { diff --git a/web/cast_generate.css b/web/cast_generate.css index 828c679a..2dac44de 100644 --- a/web/cast_generate.css +++ b/web/cast_generate.css @@ -42,3 +42,15 @@ .fv-cg-view{flex:none;width:72px} } @media(prefers-reduced-motion:reduce){.fv-cg *{transition:none}} + +.fv-cg[data-kind=scene] .fv-cg-body{grid-template-columns:minmax(0,1fr)} +.fv-cg[data-kind=scene] .fv-cg-text{min-height:96px} +.fv-cg[data-kind=scene] .fv-cg-view{aspect-ratio:16/9;cursor:pointer} +.fv-cg[data-kind=scene] .fv-cg-view:not([data-state=done]){cursor:default} +.fv-cg[data-kind=scene] .fv-cg-view img{object-fit:cover} +.fv-cg[data-kind=scene] .fv-cg-view[aria-pressed=true]{border:2px solid var(--fv-accent,#6db8fa)} +.fv-cg[data-kind=scene] .fv-cg-view:focus-visible{outline:2px solid var(--fv-accent,#6db8fa);outline-offset:2px} +.fv-cg-stage{display:none} +.fv-cg-stage img{display:block;width:100%;aspect-ratio:16/9;object-fit:cover;border-radius:6px;background:#ffffff08} +@container (max-width:520px){.fv-cg[data-kind=scene] .fv-cg-views{display:grid}.fv-cg[data-kind=scene] .fv-cg-view{width:auto} + .fv-cg[data-kind=scene] .fv-cg-stage:not([hidden]){display:block}} diff --git a/web/cast_generate.js b/web/cast_generate.js index 99347dc6..07238c7f 100644 --- a/web/cast_generate.js +++ b/web/cast_generate.js @@ -9,6 +9,8 @@ import {createImageLoras} from './image_lora.js'; export {imageGenerationSupported as castGenerateSupported} from './image_generation.js'; export const CAST_VIEWS = Object.freeze(['front', 'three_quarter', 'side', 'back', 'face']); +// A scene is not a sheet of views: one description is drawn three times with different seeds and the user picks one. +export const SCENE_CANDIDATES = Object.freeze(['a', 'b', 'c']); const VIEW_TITLES = Object.freeze({ front: ['Front', '正面'], three_quarter: ['Three-quarter', '四分之三'], side: ['Side', '侧面'], back: ['Back', '背面'], face: ['Face', '脸部'], @@ -20,8 +22,9 @@ const failureOf = error => ({error: error?.code || 'generation_failed', report_i const randomSeed = () => crypto.getRandomValues(new Uint32Array(1))[0] >>> 1; // The five pictures of one description. No DOM: the page below draws its state. -export function createCastGenerator({client, onChange = () => {}, schedule = setTimeout, unschedule = clearTimeout}) { - const fresh = () => CAST_VIEWS.map(view => ({view, file: '', fraction: 0})); +export function createCastGenerator({client, kind = 'character', onChange = () => {}, schedule = setTimeout, unschedule = clearTimeout}) { + const scene = kind === 'setting', names = scene ? SCENE_CANDIDATES : CAST_VIEWS; + const fresh = () => names.map(view => ({view, file: '', fraction: 0})); let state = {pictures: fresh(), running: false, current: -1, error: '', report_id: null, waiting: '', description: ''}; let epoch = 0, timer = null, job = null, plan = null, seed = 0, disposed = false, loras; const emit = patch => { state = {...state, ...patch}; if (!disposed) onChange(state); }; @@ -31,7 +34,7 @@ export function createCastGenerator({client, onChange = () => {}, schedule = set async function runOne(index, alive) { const view = plan[index], front = state.pictures[0].file; - const row = await client.generate({prompt: view.prompt, width: view.width, height: view.height, seed, + const row = await client.generate({prompt: view.prompt, width: view.width, height: view.height, seed: scene ? seed + index : seed, images: view.from_front ? [front] : [], ...(view.view === 'back' ? {cast_view: 'back'} : {}), ...(loras === undefined ? {} : {loras})}); if (!row.job) throw Object.assign(new Error(row.error || 'generation_failed'), {code: row.error, report_id: row.report_id, memory: row.memory}); @@ -69,7 +72,8 @@ export function createCastGenerator({client, onChange = () => {}, schedule = set ...(resume ? {} : {pictures: fresh(), current: 0})}); try { if (!plan) { - plan = await client.castViews(description); + plan = await client.castViews(description, kind); + if (scene) plan = names.map(() => plan[0]); if (!alive()) return; } for (let index = 0; index < plan.length; index++) { @@ -108,7 +112,8 @@ export function createCastGenerator({client, onChange = () => {}, schedule = set dispose() { disposed = true; return stop(); }}; } -export function mountCastGenerate({container, api, text, t = text, viewURL, onState = () => {}, fetchApi, client, loras}) { +export function mountCastGenerate({container, api, text, t = text, viewURL, onState = () => {}, onPick = () => {}, fetchApi, client, loras, kind = 'character'}) { + const scene = kind === 'setting', names = scene ? SCENE_CANDIDATES : CAST_VIEWS; const tr = t || ((en) => en); const fetcher = fetchApi || (api ? api.fetchApi.bind(api) : globalThis.fetch.bind(globalThis)); client = client || createImageGenClient(fetcher); @@ -123,11 +128,12 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt } const root = el('div', 'fv-cg'); + if (scene) root.dataset.kind = 'scene'; const body = el('div', 'fv-cg-body'); const side = el('div', 'fv-cg-side'); const foot = el('div', 'fv-cg-foot'); const field = el('textarea', 'fv-cg-text'); - field.placeholder = tr('Describe the character', '描述角色'); + field.placeholder = scene ? tr('Describe the scene', '描述场景') : tr('Describe the character', '描述角色'); field.setAttribute('aria-label', field.placeholder); field.maxLength = 1000; const note = el('div', 'fv-cg-note'); note.setAttribute('role', 'status'); @@ -138,10 +144,16 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt foot.append(note, action); side.append(field, ...(picker ? [picker.element] : []), foot); const grid = el('div', 'fv-cg-views'); - const cells = CAST_VIEWS.map(view => { + const cells = names.map((view, index) => { const cell = el('div', 'fv-cg-view'); - const title = tr(...VIEW_TITLES[view]); - cell.title = title; cell.setAttribute('role', 'img'); + const title = scene ? tr(`Option ${index + 1}`, `方案 ${index + 1}`) : tr(...VIEW_TITLES[view]); + cell.title = title; cell.setAttribute('role', scene ? 'button' : 'img'); + if (scene) { + cell.tabIndex = 0; + const pick = () => { if (cell.dataset.state !== 'done') return; chosen = index; render(); }; + cell.onclick = pick; + cell.onkeydown = event => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); pick(); } }; + } const image = el('img'); image.alt = ''; image.draggable = false; const percent = el('span', 'fv-cg-percent'); const meter = el('i', 'fv-cg-meter'); const fill = el('b'); meter.append(fill); @@ -149,12 +161,14 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt grid.append(cell); return {cell, image, percent, fill, title}; }); - body.append(side, grid); + // A scene shows the chosen option large on narrow widths, where three small ones cannot show its mood. + const stage = el('div', 'fv-cg-stage'), stageImage = el('img'); stageImage.alt = ''; stage.append(stageImage); stage.hidden = true; + body.append(side, ...(scene ? [stage] : []), grid); root.append(body); container.append(root); - let info = null, infoTimer = null, disposed = false, failureKey = null, attention = null; - const generator = createCastGenerator({client, onChange: render}); + let chosen = 0, picked = '', info = null, infoTimer = null, disposed = false, failureKey = null, attention = null; + const generator = createCastGenerator({client, kind, onChange: render}); let last = ''; function missing() { return info && !info.installed && !info.ready; } @@ -171,18 +185,26 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt action.dataset.primary = String(!running && !finishedAll); const finished = state.pictures.filter(row => row.file).length; const currentFraction = running && state.current >= 0 ? state.pictures[state.current]?.fraction || 0 : 0; - const progress = generator.done() ? 1 : (finished + currentFraction) / CAST_VIEWS.length; + const progress = generator.done() ? 1 : (finished + currentFraction) / names.length; state.pictures.forEach((row, i) => { const cell = cells[i]; const active = running && i === state.current && !row.file; const src = row.file ? url(row.file) : ''; if (cell.image.getAttribute('src') !== src) src ? cell.image.src = src : cell.image.removeAttribute('src'); cell.cell.dataset.state = row.file ? 'done' : active ? 'running' : 'empty'; + if (scene) cell.cell.setAttribute('aria-pressed', String(!!row.file && i === chosen)); cell.cell.setAttribute('aria-label', cell.title + (row.file ? '' : active ? ' ' + Math.floor(row.fraction * 100) + '%' : '')); cell.percent.textContent = active && !state.waiting ? Math.floor(row.fraction * 100) + '%' : ''; cell.fill.style.width = (active ? row.fraction * 100 : 0) + '%'; }); + if (scene) { + const file = generator.done() && !running ? state.pictures[chosen]?.file || '' : ''; + const shown = file ? url(file) : ''; + stage.hidden = !shown; if (stageImage.getAttribute('src') !== shown) shown ? stageImage.src = shown : stageImage.removeAttribute('src'); + if (file && file !== picked) onPick(picked = file); + if (!file && !running) picked = ''; + } if (state.error) { const key = `${state.error}:${state.report_id || ''}:${state.stage || ''}:${state.memory || ''}`; if (failureKey !== key) { renderImageFailure(note, {...state, info}, tr, {fetchApi: fetcher}); failureKey = key; } @@ -214,13 +236,13 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt function begin() { if (!field.value.trim()) { - attention = tr('Describe the character first.', '请先描述角色。'); + attention = scene ? tr('Describe the scene first.', '请先描述场景。') : tr('Describe the character first.', '请先描述角色。'); render(); field.focus(); return; } - attention = null; - const chosen = picker ? picker.value() : loras; - void generator.start(field.value, picker && !chosen.length ? {} : {loras: chosen}); + attention = null; chosen = 0; + const selected = picker ? picker.value() : loras; + void generator.start(field.value, picker && !selected.length ? {} : {loras: selected}); } action.onclick = () => generator.state().running ? void generator.stop() : begin(); field.addEventListener('input', () => { @@ -235,7 +257,9 @@ export function mountCastGenerate({container, api, text, t = text, viewURL, onSt render(); void checkInfo(); return { - getPictures: () => generator.done() && !generator.state().running ? generator.state().pictures.map(row => ({file: row.file, label: row.view})) : [], + getPictures: () => !generator.done() || generator.state().running ? [] + : scene ? [{file: generator.state().pictures[chosen].file, label: 'other'}] + : generator.state().pictures.map(row => ({file: row.file, label: row.view})), isBusy: () => generator.state().running, cancel: () => generator.stop(), destroy() { disposed = true; picker?.dispose(); clearTimeout(infoTimer); void generator.dispose(); root.remove(); }, diff --git a/web/cast_library.css b/web/cast_library.css index 7028dfee..83f1eec6 100644 --- a/web/cast_library.css +++ b/web/cast_library.css @@ -48,3 +48,9 @@ .fv-cast-dialog [hidden],.fv-cast-library [hidden]{display:none!important} @media(max-width:540px){.fv-cast-dialog{padding:14px}.fv-cast-upload{gap:10px}.fv-cast-views{gap:5px}.fv-cast-actions{gap:5px}.fv-cast-actions button{padding:5px 9px}} .fv-cast-library .fv-cast-avatar.fv-cast-mentioned{border-color:var(--fv-accent);box-shadow:0 0 0 1px var(--fv-accent)} +.fv-cast-library[data-kind=scene] .fv-cast-avatar,.fv-cast-library[data-kind=scene] .fv-cast-add{flex-basis:56px;width:56px;height:32px;border-radius:6px} +.fv-cast-library[data-kind=scene] .fv-cast-avatar img{object-position:50% 50%;object-view-box:none} +.fv-cast-dialog[data-kind=scene] .fv-cast-upload{display:block} +.fv-cast-dialog[data-kind=scene] .fv-cast-views{grid-template-columns:repeat(3,minmax(0,1fr));grid-template-rows:none} +.fv-cast-dialog[data-kind=scene] .fv-cast-slot{aspect-ratio:16/9} +.fv-cast-dialog[data-kind=scene] .fv-cast-choose img{object-fit:cover} diff --git a/web/cast_library.js b/web/cast_library.js index 479b4cac..0f21f51e 100644 --- a/web/cast_library.js +++ b/web/cast_library.js @@ -23,7 +23,10 @@ const button = (className, label, action) => { }; const path = id => '/freevideo/agent/cast/' + encodeURIComponent(id); -export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () => {}}) { +// kind 'setting' is the scene row: wide pictures, up to 3 per card, and no front picture. +export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () => {}, kind = 'character'}) { + const scene = kind === 'setting', SLOTS = scene ? 3 : 6; + const word = (character, place) => scene ? place : character; if (!document.querySelector('link[data-freevideo-cast]')) { const css = el('link'); css.rel = 'stylesheet'; css.dataset.freevideoCast = ''; css.href = new URL('./cast_library.css', import.meta.url).href; document.head.append(css); @@ -34,11 +37,12 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () // The row shows, and the page session is asked for, only once /freevideo/agent says the interface is on. let available = false; element.hidden = true; const probe = agentInterfaceEnabled(api).then(on => { available = on && !disposed; element.hidden = !available; return available; }); - const heading = el('span', 'fv-cast-heading', text('Characters', '角色')); + element.dataset.kind = scene ? 'scene' : 'character'; + const heading = el('span', 'fv-cast-heading', word(text('Characters', '角色'), text('Scenes', '场景'))); const avatars = el('div', 'fv-cast-avatars'); const skipped = el('div', 'fv-cast-unreadable'); skipped.hidden = true; const add = button('fv-cast-add', '+', () => openDialog()); - add.setAttribute('aria-label', text('New character', '新建角色')); + add.setAttribute('aria-label', word(text('New character', '新建角色'), text('New scene', '新建场景'))); element.append(heading, avatars, add, skipped); async function value(response) { @@ -77,7 +81,7 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () async function refresh() { if (disposed || !await probe) return rows; const serial = ++request; - const data = await value(await agentGet(api, '/freevideo/agent/cast?kind=character&limit=200')); + const data = await value(await agentGet(api, `/freevideo/agent/cast?kind=${kind}&limit=200`)); if (!disposed && serial === request) { rows = data.cards; unreadable = Number.isSafeInteger(data.unreadable) && data.unreadable > 0 ? data.unreadable : 0; render(); onChange(rows); @@ -99,16 +103,16 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () if (disposed || !available) return; current?.close(); let closed = false, loading = !!id, saving = false, revision, active = 'upload'; - let generator = null, aiState = {busy: false, done: false}, deleteTimer = null, confirming = false, frontless = false; - const slots = Array(6).fill(null), slotUI = []; - const dialog = el('dialog', 'fv-cast-dialog'); + let generator = null, aiState = {busy: false, done: false}, deleteTimer = null, confirming = false, frontless = scene, beyond = []; + const slots = Array(SLOTS).fill(null), slotUI = []; + const dialog = el('dialog', 'fv-cast-dialog'); dialog.dataset.kind = scene ? 'scene' : 'character'; const top = el('div', 'fv-cast-top'); - const title = el('h2', '', id ? text('Edit character', '编辑角色') : text('New character', '新建角色')); + const title = el('h2', '', id ? word(text('Edit character', '编辑角色'), text('Edit scene', '编辑场景')) : word(text('New character', '新建角色'), text('New scene', '新建场景'))); dialog.setAttribute('aria-label', title.textContent); const closeButton = button('fv-cast-close', '×', () => close()); closeButton.setAttribute('aria-label', text('Close', '关闭')); top.append(title, closeButton); const name = el('input', 'fv-cast-name'); name.type = 'text'; name.maxLength = 120; - name.placeholder = text('Name', '名字'); name.setAttribute('aria-label', text('Name', '名字')); + name.placeholder = word(text('Name', '名字'), text('Name', '名称')); name.setAttribute('aria-label', name.placeholder); name.oninput = () => sync(); name.onkeydown = event => { if (event.key === 'Enter') event.preventDefault(); }; const tabs = el('div', 'fv-cast-tabs'); tabs.setAttribute('role', 'tablist'); @@ -121,7 +125,7 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () const aiPane = el('div', 'fv-cast-generate'); aiPane.hidden = true; const status = el('div', 'fv-cast-message'); status.setAttribute('role', 'status'); status.hidden = true; const actions = el('div', 'fv-cast-actions'); - const remove = button('fv-cast-delete', text('Delete character', '删除角色'), () => deleteCard()); remove.hidden = !id; + const remove = button('fv-cast-delete', word(text('Delete character', '删除角色'), text('Delete scene', '删除场景')), () => deleteCard()); remove.hidden = !id; const cancel = button('', text('Cancel', '取消'), () => close()); const save = button('fv-cast-save', text('Save', '保存'), () => saveCard()); actions.append(remove, cancel, save); @@ -138,9 +142,9 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () // What is still missing before Save works ('' when nothing is); the button says so on hover and when pressed. let missing = ''; const missingNow = () => !name.value.trim() ? text('Enter a name first', '先填写名字') - : aiBusy() ? text('Wait for the five pictures to finish', '等待五张图生成完成') - : active === 'upload' ? (frontless ? slots.some(Boolean) : slots[0]) ? '' : frontless ? text('Add an image first', '先添加图片') : text('Add a front image first', '先添加正面图片') - : aiState.done ? '' : text('Generate the images first', '先生成角色图片'); + : aiBusy() ? word(text('Wait for the five pictures to finish', '等待五张图生成完成'), text('Wait for the pictures to finish', '等待图片生成完成')) + : active === 'upload' || scene ? (frontless ? slots.some(Boolean) : slots[0]) ? '' : frontless ? text('Add an image first', '先添加图片') : text('Add a front image first', '先添加正面图片') + : aiState.done ? '' : word(text('Generate the images first', '先生成角色图片'), text('Generate the images first', '先生成场景图片')); function sync() { const busy = loading || saving; name.disabled = busy; remove.disabled = busy; @@ -202,23 +206,23 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () choose.append(image); } else { choose.append(el('span', 'fv-cast-slot-plus', '+')); - if (!index) choose.append(el('span', '', text('Front image', '正面图片'))); + if (!index && !scene) choose.append(el('span', '', text('Front image', '正面图片'))); } remove.hidden = true; if (image?.complete) remove.hidden = false; mark(index); sync(); } function chooseFiles(index, files) { if (loading || saving || closed) return; - for (const file of Array.from(files).slice(0, 6 - index)) { + for (const file of Array.from(files).slice(0, SLOTS - index)) { release(index); - slots[index] = {file, label: index ? 'other' : 'front', preview: URL.createObjectURL(file)}; + slots[index] = {file, label: index || scene ? 'other' : 'front', preview: URL.createObjectURL(file)}; paint(index++); } message(''); } - for (let index = 0; index < 6; index++) { - const slot = el('div', 'fv-cast-slot' + (index ? '' : ' fv-cast-front')); + for (let index = 0; index < SLOTS; index++) { + const slot = el('div', 'fv-cast-slot' + (index || scene ? '' : ' fv-cast-front')); const input = el('input'); input.type = 'file'; input.accept = 'image/*'; input.hidden = true; - const label = index ? text(`View ${index}`, `视角 ${index}`) : text('Front image', '正面图片'); + const label = scene ? text(`Image ${index + 1}`, `图片 ${index + 1}`) : index ? text(`View ${index}`, `视角 ${index}`) : text('Front image', '正面图片'); const choose = button('fv-cast-choose', '', () => input.click()); choose.setAttribute('aria-label', label); const remove = button('fv-cast-remove', '×', () => { release(index); paint(index); }); remove.setAttribute('aria-label', text('Remove image', '移除图片')); @@ -228,12 +232,12 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () const problem = el('div', 'fv-cast-slot-error'); problem.setAttribute('role', 'alert'); problem.hidden = true; // The remove button comes after the text so it stays above it on a narrow tile. slot.append(choose, problem, remove, input); slotUI.push({slot, choose, remove, problem}); - (index ? views : uploadPane).append(slot); paint(index); + (index || scene ? views : uploadPane).append(slot); paint(index); } uploadPane.append(views); function resetDelete() { clearTimeout(deleteTimer); deleteTimer = null; confirming = false; - remove.textContent = text('Delete character', '删除角色'); remove.classList.remove('fv-cast-confirm'); + remove.textContent = word(text('Delete character', '删除角色'), text('Delete scene', '删除场景')); remove.classList.remove('fv-cast-confirm'); } async function load() { loading = true; sync(); @@ -241,11 +245,13 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () const card = await get(id); if (closed) return; revision = card.revision; name.value = card.name; - for (let i = 0; i < 6; i++) release(i); + for (let i = 0; i < SLOTS; i++) release(i); // A card may have no front picture (the API does not require one): all its pictures stay, in order, and // saving does not ask for a front; the front slot is only reserved while there is a front to keep first. - frontless = card.images.length > 0 && card.images[0].label !== 'front'; - card.images.slice(0, 6).forEach((picture, index) => { slots[index] = {...picture}; }); + frontless = scene || card.images.length > 0 && card.images[0].label !== 'front'; + card.images.slice(0, SLOTS).forEach((picture, index) => { slots[index] = {...picture}; }); + // The API allows six pictures on a scene card but the dialog shows three: the rest stay as they are. + beyond = card.images.slice(SLOTS).map(picture => ({media: picture.media, label: picture.label})); slotUI.forEach((_, index) => paint(index)); generator?.cancel(); aiState = {busy: false, done: false}; select('upload'); resetDelete(); } finally { loading = false; if (!closed) sync(); } @@ -265,13 +271,14 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () saving = true; resetDelete(); message(''); slotUI.forEach((_, index) => mark(index)); sync(); let pictures = []; try { - pictures = active === 'upload' ? slots.filter(Boolean) : generator.getPictures().map((picture, index) => ({ + pictures = active === 'upload' || scene ? slots.filter(Boolean) : generator.getPictures().map((picture, index) => ({ ...(typeof picture.file === 'string' ? {media: picture.file} : {file: picture.file}), label: picture.label || (index ? 'other' : 'front'), })); - const frontNeeded = active === 'ai' || !frontless; - if (!pictures.length || pictures.length > 6 || (frontNeeded && !pictures.some(picture => picture.label === 'front'))) { - message(text('Choose one front image and up to 5 more angles', '选一张正面图片，再加最多 5 张其他角度的图片'), true); return; + const frontNeeded = !scene && (active === 'ai' || !frontless); + if (!pictures.length || pictures.length > SLOTS || (frontNeeded && !pictures.some(picture => picture.label === 'front'))) { + message(word(text('Choose one front image and up to 5 more angles', '选一张正面图片，再加最多 5 张其他角度的图片'), + text('Choose up to 3 images', '最多选 3 张图片')), true); return; } const images = []; for (const picture of pictures) { @@ -280,14 +287,15 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () if (picture.file && !picture.media) picture.media = (await uploadOne(picture.file)).file; images.push({media: picture.media, label: picture.label}); } + if (id && scene) images.push(...beyond); if (closed) return; const body = {name: name.value.trim(), images}; await value(await (id ? agentSend(api, 'PATCH', path(id), {revision, ...body}) - : agentPost(api, '/freevideo/agent/cast', {kind: 'character', ...body}))); + : agentPost(api, '/freevideo/agent/cast', {kind, ...body}))); close(); await saved(); } catch (error) { const at = /^\/images\/(\d+)\/media$/.exec(error.field || ''); - const index = at && active === 'upload' ? slots.indexOf(pictures[Number(at[1])]) : -1; + const index = at && (active === 'upload' || scene) ? slots.indexOf(pictures[Number(at[1])]) : -1; if (index >= 0 && !closed) mark(index, error.message); else await failure(error); } finally { saving = false; if (!closed) sync(); } @@ -309,7 +317,7 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () if (closed) return; closed = true; clearTimeout(deleteTimer); generator?.cancel(); generator?.destroy(); - for (let index = 0; index < 6; index++) release(index); + for (let index = 0; index < SLOTS; index++) release(index); if (dialog.open) dialog.close(); dialog.remove(); if (current?.element === dialog) current = null; add.focus(); @@ -321,7 +329,9 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () current = {element: dialog, message, close}; name.value = ''; select('upload'); document.body.append(dialog); dialog.showModal(); name.focus(); if (id) load().catch(error => message(error.message)); - mountGenerator({container: aiPane, api, text, viewURL, onState: state => { + // A scene has one set of pictures: the chosen option takes the first place, where the Upload tab shows it. + const pick = file => { if (closed || !scene) return; slots[0] = {media: file, label: 'other'}; paint(0); }; + mountGenerator({container: aiPane, api, text, viewURL, kind, onPick: pick, onState: state => { if (closed) return; aiState = state; sync(); }}).then(instance => { diff --git a/web/cast_mention.js b/web/cast_mention.js index e6364f8c..382f3f61 100644 --- a/web/cast_mention.js +++ b/web/cast_mention.js @@ -44,10 +44,15 @@ function pictureNumbers(rows, continued = false) { }); let kept = picked; if (continued) { + // The picture of a scene keeps its place after the others (continuation.pick_pictures). + const places = picked.filter(index => rows[index].cast?.setting === true); + const place = places.length ? places[places.length - 1] : null; + const rest = place === null ? picked : picked.filter(index => rows[index].cast?.setting !== true); const last = new Map(); - picked.forEach(index => { if (rows[index].cast?.id) last.set(rows[index].cast.id, index); }); - kept = last.size < 2 ? picked : [...last.values()].sort((a, b) => a - b).concat(picked.filter(index => !rows[index].cast?.id)); - kept = kept.slice(0, CONTINUE_PICTURES); + rest.forEach(index => { if (rows[index].cast?.id) last.set(rows[index].cast.id, index); }); + kept = last.size < 2 ? rest : [...last.values()].sort((a, b) => a - b).concat(rest.filter(index => !rows[index].cast?.id)); + kept = kept.slice(0, CONTINUE_PICTURES - (place === null ? 0 : 1)); + if (place !== null) kept.push(place); } return new Map(kept.map((index, at) => [index, at + 1])); } @@ -59,7 +64,7 @@ export function castInRows(rows) { if (!enabled(row) || !row.cast?.id || row.role !== 'reference' || !IMAGE.test(row.file || '')) return; const name = row.cast.name || ''; // `handle` is what the text says after the @: the name, or "name (2)" when another present character has the same name. - const person = people.get(row.cast.id) || {id: row.cast.id, name, handle: row.cast.handle || name, description: row.cast.description || '', rows: []}; + const person = people.get(row.cast.id) || {id: row.cast.id, name, handle: row.cast.handle || name, description: row.cast.description || '', setting: row.cast.setting === true, rows: []}; person.rows.push(index); people.set(row.cast.id, person); }); return [...people.values()]; @@ -101,6 +106,7 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) const people = castInRows(list); const ranges = mentionRanges(text, people); if (!ranges.length) return {text, subjects: []}; + const placed = []; if (!continued && list.some(row => enabled(row) && (row.role === 'first' || row.role === 'last'))) active = false; const numbers = pictureNumbers(list, continued); const order = []; @@ -108,6 +114,11 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) for (const range of ranges) { result += text.slice(at, range.start); at = range.end; const person = people.find(p => (p.handle ?? p.name) === range.name); + if (person?.setting) { + // A scene is not a subject: its name stays in the sentence and one line says which pictures show the place. + if (active && !placed.includes(person) && person.rows.some(index => numbers.has(index))) placed.push(person); + result += person.name; continue; + } if (!active || !person || !person.rows.some(index => numbers.has(index))) { result += person ? person.name : range.name; continue; } let number = order.indexOf(person) + 1; if (!number) { @@ -117,7 +128,12 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) result += ``; } result += text.slice(at); - if (!order.length) return {text: result, subjects: []}; + const places = placed.slice(0, 1).map(place => { + const number = place.rows.map(index => numbers.get(index)).filter(Boolean)[0]; + const description = String(place.description || '').trim().replace(/\.+$/, ''); + return `setting: ${place.name}${description ? ': ' + description : ''}. shows the place.`; + }); + if (!order.length && !places.length) return {text: result, subjects: []}; const subjects = order.map(person => { // Consecutive picture numbers make one range; a person whose pictures are apart gets several. const ranges = []; @@ -127,7 +143,8 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) } return {name: person.name, handle: person.handle ?? person.name, description: person.description, pictures: ranges, audio: null}; }); - return {text: subjectDefinitions(subjects) + '\n\n' + result, subjects}; + const definitions = [...(subjects.length ? [subjectDefinitions(subjects)] : []), ...places].join(' '); + return {text: definitions + '\n\n' + result, subjects}; } // The rewrite of an expanded prompt, written back with the names the box had: becomes "@handle" of the @@ -166,7 +183,9 @@ export function mentionState(rows, card, count = cardPictures(card).length, {con const people = castInRows(list); if (people.some(person => person.id === card.id)) return {ok: true, present: true}; if (!continued && list.some(row => enabled(row) && (row.role === 'first' || row.role === 'last'))) return {ok: false, reason: 'frames'}; - if (people.length >= MAX_PEOPLE) return {ok: false, reason: 'people'}; + if (card.kind === 'setting' ? people.some(person => person.setting) : people.filter(person => !person.setting).length >= MAX_PEOPLE) { + return {ok: false, reason: card.kind === 'setting' ? 'place' : 'people'}; + } if (continued && pictureNumbers(list).size >= CONTINUE_PICTURES) return {ok: false, reason: 'continued'}; if (list.filter(enabled).length + count > MAX_REFERENCES) return {ok: false, reason: 'full'}; return {ok: true, present: false}; @@ -190,6 +209,6 @@ export function withCharacter(rows, card, typed = '') { const list = Array.isArray(rows) ? rows.map(row => ({...row})) : []; if (castInRows(list).some(person => person.id === card.id)) return list; const handle = typed || mentionHandle(list, card); - const cast = {id: card.id, name: card.name, description: card.description || '', ...(handle === card.name ? {} : {handle})}; + const cast = {id: card.id, name: card.name, description: card.description || '', ...(card.kind === 'setting' ? {setting: true} : {}), ...(handle === card.name ? {} : {handle})}; return [...list, ...cardPictures(card).map(file => ({file, role: 'reference', enabled: true, cast}))]; } diff --git a/web/chain_rules.js b/web/chain_rules.js index b0575bfe..0204c0bc 100644 --- a/web/chain_rules.js +++ b/web/chain_rules.js @@ -1 +1 @@ -export const CHAIN_RULES = {"min_part_seconds": 5, "full_part_seconds": 10, "max_parts": 8, "short_max_parts": 4, "continue_seconds": 10, "short_continue_seconds": 5}; +export const CHAIN_RULES = {"min_part_seconds": 5, "full_part_seconds": 10, "max_parts": 8, "short_max_parts": 4, "continue_seconds": 10, "short_continue_seconds": 5, "min_tail_seconds": 2}; diff --git a/web/chat_data.js b/web/chat_data.js index 60665930..939f5195 100644 --- a/web/chat_data.js +++ b/web/chat_data.js @@ -96,6 +96,8 @@ export function chatData(api) { confirm: body => agentPost(api, '/freevideo/agent/chat/confirm', body), abandon: session => agentPost(api, '/freevideo/agent/chat/abandon', session ? {session} : {}), cancel: id => retryAgentPost(api, '/freevideo/agent/activity/cancel', {id}), + // Once: a second try after one that went through would only be refused as already running. + resume: id => agentPost(api, '/freevideo/agent/activity/resume', {id}), activity: (cursor, options) => read('/freevideo/agent/activity' + (cursor ? '?cursor=' + encodeURIComponent(cursor) : ''), options), sendMessage: (entry, options) => sendChatMessage(api, entry, options), send: (body, options) => retryAgentPost(api, '/freevideo/agent/chat', body, options), diff --git a/web/editor.css b/web/editor.css index 8375febc..3a00990c 100644 --- a/web/editor.css +++ b/web/editor.css @@ -84,6 +84,14 @@ .fv-editor .fv-ed-gen-retry:disabled{color:var(--fv-muted);cursor:default} .fv-ed-genbar .fv-request-assets{order:-2;flex:1 1 100%;min-width:0;font-size:12px;line-height:1.5} .fv-ed-genbar .fv-request-assets[hidden]{display:none} +/* A stretch of the music track: a style row and a mood row above the description. */ +.fv-ed-genbar.fv-ed-genbar-music{flex-wrap:wrap} +.fv-ed-music-presets{order:-1;flex:1 1 100%;display:flex;flex-direction:column;gap:6px;min-width:0} +.fv-ed-music-presets[hidden]{display:none} +.fv-ed-music-row{display:flex;align-items:center;gap:8px;min-width:0} +.fv-ed-music-label{flex:none;min-width:2.5em;font-size:13px;color:var(--fv-muted)} +.fv-editor .fv-ed-music-presets .fv-ed-adjust-segmented{width:auto;flex:0 1 auto;min-width:0;flex-wrap:wrap} +.fv-editor .fv-ed-music-presets .fv-ed-adjust-segmented button{flex:none} .fv-editor:has(.fv-ed-gen-failure:not([hidden])){--fv-ed-generation-row:max-content} .fv-ed-genbar:has(.fv-ed-gen-failure:not([hidden])){flex-wrap:wrap;padding-top:10px;padding-bottom:10px} .fv-editor .fv-ed-gen-failure{display:block;flex:1 0 100%;min-width:0} @@ -101,6 +109,15 @@ .fv-ed-first-frame input:checked{background:var(--fv-accent,#6db8fa)} .fv-ed-first-frame input:checked::before{transform:translateX(10px)} .fv-ed-first-frame:has(input:disabled){opacity:.4} +/* Duration of the parts not generated yet, typed in whole seconds; the seconds field of Studio, narrower. */ +.fv-ed-duration{position:relative;flex:none;font-size:13px;white-space:nowrap} +.fv-ed-duration[hidden]{display:none} +.fv-ed-duration label{display:flex;align-items:center;gap:6px} +.fv-studio .fv-ed-duration input[type=number]{width:56px;height:36px;min-height:36px;padding:0 6px;text-align:center;font-variant-numeric:tabular-nums} +.fv-ed-duration:has(input:disabled){opacity:.4} +.fv-ed-duration-hint{bottom:calc(100% + 6px);left:0} +.fv-ed-duration-hint[hidden]{display:none} +.fv-ed-genbar-break{display:none} .fv-ed-description:disabled{opacity:.4} .fv-ed-genbar{position:relative} .fv-ed-genbar:has(.fv-empty-prompt){flex-wrap:wrap} @@ -207,6 +224,7 @@ .fv-ed-ref-chips{max-width:calc(100vw - 96px)} .fv-ed-gen-hint{flex:1 0 auto;white-space:nowrap} .fv-ed-gen-note{order:-1;flex:1 1 100%;white-space:normal} + .fv-studio .fv-ed-duration input[type=number]{font-size:16px} .fv-editor .fv-ed-genbar>button{margin-left:auto;max-width:100%;height:36px;min-height:36px;font-size:12px} .fv-editor .fv-ed-genbar>button:not([hidden]) ~ button:not([hidden]){margin-left:0} .fv-ed-genbar .fv-empty-prompt{flex:0 0 100%;justify-content:flex-end;margin:0} @@ -235,6 +253,7 @@ .fv-studio.fv-editor:has(.fv-ed-gen-failure:not([hidden])){grid-template-rows:auto minmax(0,1fr) 40px var(--fv-ed-generation-row,max-content) auto minmax(160px,236px)} .fv-studio.fv-editor .fv-header:has(.fv-view-switch){grid-template-columns:minmax(0,1fr) auto} /* One row: undo, split, delete and zoom; editor_toolbar_more.js moves the rest behind ⋯. */ + .fv-ed-duration-word{display:none} .fv-ed-toolbar{gap:4px;padding:0 12px;overflow:visible} .fv-ed-toolbar .fv-ed-tool-group+.fv-ed-tool-group:before{display:none} .fv-ed-toolbar>.fv-ed-zoom{flex:1 1 auto;min-width:0;margin-left:4px} @@ -256,6 +275,13 @@ .fv-editor .fv-ed-library{grid-area:auto;top:116px;bottom:0;z-index:8} .fv-editor:has(.fv-ed-library[data-open=true]) .fv-ed-drawer-toggle{position:fixed;top:128px;right:20px;z-index:9} } +/* "Generate · about 4 minutes" in English does not fit beside the seconds field below 550 px: it keeps a line of its own, + and the line above it stays whether the field shows or not, so choosing a clip or a draft does not move the timeline. */ +@media(max-width:549px){ + .fv-ed-genbar:lang(en)[data-mode=selected]:not(:has(>.fv-ed-gen-failure:not([hidden]))){row-gap:0;padding-top:2px;align-content:space-between;min-height:calc(146px + var(--fv-ed-description-growth,0px))} + .fv-ed-genbar:lang(en)[data-mode=selected]:not(:has(>.fv-ed-gen-failure:not([hidden])))>*{margin-top:8px} + .fv-ed-genbar:lang(en)[data-mode=selected]:not(:has(>.fv-ed-gen-failure:not([hidden])))>.fv-ed-genbar-break{display:block;flex:0 0 100%;height:0;margin:0} +} /* Narrow window: the picture sits right under the header, the row takes the project's own shape (a 16:9 project is about 200 px tall at 400 px wide), and whatever height is left goes to the timeline. */ @media(max-width:680px){ diff --git a/web/editor.js b/web/editor.js index d1d3103d..61df5a51 100644 --- a/web/editor.js +++ b/web/editor.js @@ -21,9 +21,11 @@ import {createAdjustPicture} from './editor_adjust_picture.js'; import {pictureSession} from './editor_picture_session.js'; import {createPictureTools} from './editor_frame.js'; import {openExportDialog} from './editor_export_dialog.js'; -import {createAudioIntegration, handleAudioShortcut, AUDIO_SHORTCUTS} from './editor_audio.js'; +import {createAudioIntegration, createSpeechSpans, handleAudioShortcut, AUDIO_SHORTCUTS} from './editor_audio.js'; import {createSpeedComponent, readSpeedAudioSamples, handleSpeedKey, speedShortcuts} from './editor_speed.js'; import {createEditorGenbar, loadEditorComponents, editorBlockStates, blockMessages} from './editor_genbar.js'; +import {createEditorMusic} from './editor_music.js'; +import {MAX_MUSIC_TEXT as MUSIC_TEXT_LIMIT} from './editor_music_model.js'; import {createProgressConnection} from './progress_connection.js'; import {createSelectionActions, deletionStatus, isMultiple, isTyping, playheadIds, removableGaps, removeGaps} from './editor_select.js'; import {isolateDialogKeys, topDialog} from './dialog_keys.js'; @@ -48,10 +50,12 @@ export const wellFormed = text => typeof text === 'string' ? text.replace(/[\ud8 export const characters = text => Array.from(text).length; export const textLongText = (t, limit) => t('The description is too long. Shorten it to %d characters or fewer.', '描述过长，请缩短到 %d 字以内。').replace('%d', limit); // The engine's limits on continuing parts, refused before a generation starts; the engine says the same. -export const PARTS_LIMIT_CODES = ['chain_too_long', 'chain_part_too_short', 'chain_short_parts']; +export const PARTS_LIMIT_CODES = ['chain_too_long', 'chain_part_too_short', 'chain_short_parts', 'chain_source_full']; export function partsLimitText(code, t = en => en, limit = null) { const {followSeconds: least, fullSeconds: full, maxParts, shortMaxParts} = model.RULES; - if (code === 'chain_part_too_short') return t(`Each continued part must be at least ${least} seconds. Make it longer.`, `接着生成的每一段至少 ${least} 秒，请加长这一段。`); + if (code === 'chain_source_full') return t('This video already has the maximum of ' + maxParts + '\u00a0parts and cannot be extended.', + '这个视频已达 ' + maxParts + '\u00a0段上限，无法再接着生成。'); + if (code === 'chain_part_too_short') return t(`A part followed by another part must be at least ${least} seconds. Make it longer.`, `后面还接着其他段时，这一段至少 ${least} 秒，请加长这一段。`); if (code === 'chain_short_parts') { const most = limit ?? shortMaxParts; return t(`With a part shorter than ${full} seconds, one continuous shot can have up to ${most} parts. Make every part at least ${full} seconds long, or use fewer parts.`, `有短于 ${full} 秒的段时，一个连续镜头最多 ${most} 段。请把每段加长到至少 ${full} 秒，或减少段数。`); } return t(`One continuous shot can have up to ${limit ?? maxParts} parts. Make it shorter.`, `一个连续镜头最多 ${limit ?? maxParts} 段，请缩短。`); @@ -74,7 +78,7 @@ export function editorErrorText(value, t = en => en) { if (code === 'gpu_memory' || /out of memory|cuda.*memory|显存不足|显存不够/i.test(raw)) return t(`${stage}: not enough GPU memory.`, `${stage}时显存不足。`); const known = { - chain_part_too_short: t(`Each continued part must be at least ${model.RULES.followSeconds} seconds.`, `接着生成的每一段至少 ${model.RULES.followSeconds} 秒。`), + chain_part_too_short: t(`A part followed by another part must be at least ${model.RULES.followSeconds} seconds.`, `后面还接着其他段时，这一段至少 ${model.RULES.followSeconds} 秒。`), chain_short_parts: t(`With a part shorter than ${model.RULES.fullSeconds} seconds, one continuous shot can have up to ${model.RULES.shortMaxParts} parts.`, `有短于 ${model.RULES.fullSeconds} 秒的段时，一个连续镜头最多 ${model.RULES.shortMaxParts} 段。`), chain_too_long: t(`One continuous shot can have up to ${model.RULES.maxParts} parts.`, `一个连续镜头最多 ${model.RULES.maxParts} 段。`), @@ -188,8 +192,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given const t = (en, zh) => String(language()).toLowerCase().startsWith('zh') ? zh : en; const returnFocus = document.activeElement; const even = value => Math.max(2, Math.floor(value / 2) * 2); - const initial = () => model.emptyProject({width: even(panel.width || 1344), height: even(panel.height || 768), settings: panel.settings || null, - name: t('Untitled edit', '未命名剪辑')}); + // A new edit opens with an empty second sound track, where music is made. + const initial = () => model.addAudioTrack(model.emptyProject({width: even(panel.width || 1344), height: even(panel.height || 768), settings: panel.settings || null, + name: t('Untitled edit', '未命名剪辑')})); const state = createEditorState(initial()); const resumeCanvasPointer = suspendCanvasPointer(app?.canvas); const element = el('dialog', 'fv-studio fv-editor'); element.setAttribute('aria-label', t('Edit', '剪辑视图')); element.tabIndex = -1; @@ -374,7 +379,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given }, {onStatus: () => { if (!queue.dirty) { discardOnSwitch = false; saveWarning.hidden = true; } }}); - const player = createEditorPlayer(preview, transport, {state, sources: store.sources, api, t, error, readAudioSamples: panel.readAudioSamples || readSpeedAudioSamples, preservePreview: () => initializing, prepareAudio: () => !initializing}); + const player = createEditorPlayer(preview, transport, {state, sources: store.sources, api, t, error, readAudioSamples: panel.readAudioSamples || readSpeedAudioSamples, preservePreview: () => initializing, prepareAudio: () => !initializing, speech: createSpeechSpans(store.speech)}); function selectionClips(selection = state.selection) { if (selection?.kind !== 'clip') return []; return state.record.tracks[0].clips.filter(clip => selection.ids.includes(clip.id) || selection.ids.includes(clip.link)); @@ -515,6 +520,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given function completeSources() { return store.getSources(model.references(state.record).map(sourceKey)).then(() => { player.refresh(); tracks?.refresh?.(); }); } + // Music on the second sound track goes through the same bar; the bar shows a chosen piece's own job. + const music = createEditorMusic({state, store, t, completeSources, onStatus: () => genbar?.update(barValue)}); + let barValue = null; const exportMatches = () => exportSnapshot && exportSnapshot.id === state.record.id && exportSnapshot.revision === state.record.revision && exportSnapshot.content === content(state.record); function explainExport() { if (initializing || switching) { notify(t('Opening the edit…', '正在打开…')); return true; } @@ -655,7 +663,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given initialSelection = offerInitial = extendEntry = null; projectEpoch++; held.length = 0; player.pause(); for (const stop of jobWatchers.values()) stop(); jobWatchers.clear(); jobStatuses.clear(); progress.clear(); runProgress.clear(); failures.clear(); progressConnection?.reset(); - mostRecentJob = null; submissionFailure = null; genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); + mostRecentJob = null; submissionFailure = null; music.reset(); genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); suppressSave = true; try { state.replace(record); savedContent = content(record); } finally { suppressSave = false; } panel.onProjectLoaded?.(record); const width = tracksHost.clientWidth - 148; if (width > 0) state.setZoom(model.openingZoom(width, record)); @@ -689,7 +697,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given forgetPointer(); })); if (disposed) return; // A restored edit remains usable offline with its existing placeholders. - await completeSources().catch(value => error(value, null, undefined, 'sources')); if (disposed) return; recoverJobs(); + await completeSources().catch(value => error(value, null, undefined, 'sources')); if (disposed) return; recoverJobs(); music.recover(); state.restoreView(remembered); // Also when generating is confirmed only after the edit has opened. offerInitial = () => { if (epoch === projectEpoch && !state.selection && state.record.tracks.every(track => !track.clips.length)) selectInitial(); }; @@ -828,6 +836,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given }); } const metadata = store.sources.get(sourceKey(value.source)); + if (metadata?.path && value.source?.kind !== 'import') void store.lineage([{key: sourceKey(value.source), reference: {work: metadata.path}}]).then(() => hintFullSource()).catch(() => {}); try { state.apply(t('Insert clip', '放入素材'), record => model.insertSource(record, { ...value, width: metadata?.width ?? value.width, height: metadata?.height ?? value.height, text: value.text ?? metadata?.text, at: value.at ?? model.duration(record)}), {clearRange: state.selection !== initialSelection}); @@ -844,6 +853,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given interrupted: clip => clip.pending === 'stopped' && !clip.source && !!clip.job && (jobStatuses.get(clip.job)?.blocks || []).some(block => block.state === 'interrupted' && model.attemptClip(state.record, block.id)?.id === clip.id), selectedFailure() { + const tune = music.failure(); if (tune) return tune; const clip = state.selection?.kind === 'clip' && state.record.tracks.flatMap(track => track.clips).find(clip => state.selection.ids.includes(clip.id) && clip.pending === 'failed'); if (clip) return {id: clip.id, text: failures.text(clip.id)}; return submissionFailure === JSON.stringify(state.selection) ? {id: 'submission', text: failures.text('submission')} : null; @@ -875,11 +885,13 @@ export function openEditor({api, app, from = null, panel = {}, components: given // The description the box shows for a choice of planned parts, and the parts it is written to; null for // any other choice. partText(selection = state.selection) { + if (music.target(selection)) return music.text(selection); const clips = selectionClips(selection).sort((a, b) => a.start - b.start); return clips.length && clips.every(editablePart) ? {ids: clips.map(clip => clip.id), text: clips[0].text} : null; }, describeParts(ids, text) { text = wellFormed(text); + if (ids.length === 1 && music.target({kind: 'clip', ids})) return music.describe(ids[0], text); if (!ids.every(id => editablePart(model.find(state.record, id)))) return false; try { state.apply(t('Edit description', '修改说明'), record => model.describePlanned(record, ids, text)); return true; } catch (value) { if (value instanceof model.EditError) return false; throw value; } @@ -915,6 +927,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given }}, previewRange: selection => genbar?.preview(selection), async estimate(selection) { + if (music.target(selection)) return music.estimate(selection); const run = partRun(selection), range = selectionRange(run ? clipChoice(run) : selection); if (!range) return null; let seconds = 0; for (const count of model.rangeParts(state.record, {...range, ...capability})) { const value = capability.estimate(count); if (value == null) return null; seconds += value; } @@ -953,9 +966,18 @@ export function openEditor({api, app, from = null, panel = {}, components: given const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path])); const request = model.generationRequest(prepared, ids, {text: texts ? undefined : text, paths, queue: true}); // What the engine would refuse for its continuing parts is said here, before anything is sent. - const problem = model.partsProblem(prepared, ids); - if (problem) throw Object.assign(new Error(problem.code), {code: problem.code, limit: problem.limit}); - const beforeKeys = model.generationBefore(prepared, ids), sizes = new Map(store.sources); + const beforeKeys = model.generationBefore(prepared, ids); + const lineageWorks = beforeKeys.filter(key => key && !key.startsWith('chain:')).map(key => { + const metadata = store.sources.get(key); + return metadata?.path ? {key, reference: {work: metadata.path}} : null; + }).filter(Boolean); + if (lineageWorks.length) await store.lineage(lineageWorks); + const problem = model.partsProblem(prepared, ids, {sourceLineage: store.knownLineage()}); + if (problem) { + const full = problem.code === 'chain_too_long' && beforeKeys.some(key => store.knownLineage()[key]?.parts >= model.RULES.maxParts); + throw Object.assign(new Error(full ? 'chain_source_full' : problem.code), {code: full ? 'chain_source_full' : problem.code, limit: problem.limit}); + } + const sizes = new Map(store.sources); for (const key of beforeKeys.filter(key => key?.startsWith('chain:'))) { const job = await store.chain(key.split(':')[1]); sizes.set(key, job?.generation_size); @@ -1012,6 +1034,11 @@ export function openEditor({api, app, from = null, panel = {}, components: given async press(text, {allowEmpty = false} = {}) { if (disposed || closing || initializing) return {say: t('Opening the edit…', '正在打开…')}; text = wellFormed(text); + // A stretch of the music track, or a piece of music there: music, with or without words. + if (music.target()) { + if (characters(text) > MUSIC_TEXT_LIMIT) return {say: textLongText(t, MUSIC_TEXT_LIMIT), focus: true}; + return music.press(text); + } let selection = state.selection, said = null; if (actions.selectionHeld()) return {say: t('Queued', '排队中')}; if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')}; @@ -1076,9 +1103,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given await actions.generate({text: texts ? texts[run[0].id] : text, texts, selection, shown, allowEmpty}); return {text, say: [said, used, busyBefore ? t('Queued. It starts after the part now generating.', '已排队，正在生成的那段完成后自动开始。') : null].filter(Boolean).join(' ') || null, queued: busyBefore}; }, - selectionRunning: () => busySelection(state.selection), + selectionRunning: () => music.target() ? music.running() : busySelection(state.selection), // A choice over existing clips that Generate would move to an empty spot. - selectionMovable: () => !actions.supportedSelection() && !!emptySpot(state.selection), + selectionMovable: () => !music.target() && !actions.supportedSelection() && !!emptySpot(state.selection), selectionSending() { const range = selectionRange(state.selection); return !!range && ([...sendingRanges].some(other => overlaps(other, range)) || [...sending].some(id => { const clip = model.find(state.record, id); return clip && overlaps(clip, range); })); }, selectionHeld() { const ids = selectionClips(state.selection).map(clip => clip.id); return ids.length > 0 && ids.every(isHeld); }, // Takes the selected parts out of the line; they stay as drafts. @@ -1088,6 +1115,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given rememberView(); }, async cancel() { + if (music.target()) { await music.cancel(); return; } const range = selectionRange(state.selection); const own = range && state.record.tracks[0].clips.find(clip => clip.job && generating(clip) && clip.start < range.start + range.frames && clip.start + clip.frames > range.start)?.job; @@ -1214,7 +1242,16 @@ export function openEditor({api, app, from = null, panel = {}, components: given return running; } + actions.music = {target: selection => music.target(selection), preset: selection => music.preset(selection), choose: value => music.choose(value), + styles: music.styles, moods: music.moods, downloadBytes: music.downloadBytes}; genbar = createEditorGenbar(genHost, {state, t, actions, text: ''}); settleService(); + { + // What the editor gives the bar is the video job's; a chosen piece of music being made shows its own instead. + const update = genbar.update; + genbar.update = value => { barValue = value; update(music.barStatus() || value); }; + state.subscribe(event => { if (event.type === 'selection' || event.type === 'record') update(music.barStatus() || barValue); }); + music.refreshInfo(); + } const adjust = adjustPanel = createAdjustPanel(adjustHost, {state, t, actions}); const pictureEdit = pictureSession(state, t); const notes = createBlockNotes({state, t, actions}); actions.blockNote = notes.decorate; actions.describedTrack = describedTrack; @@ -1302,7 +1339,22 @@ export function openEditor({api, app, from = null, panel = {}, components: given updateTransport(); const stopSources = store.subscribeSources(() => { player.refresh(); tracks?.refresh?.(); pictureTools.update(); updateTop(); }); let capabilitySize = `${state.record.width}x${state.record.height}`; + // The reason, said as soon as the selected continuation follows a video that already has the most parts, once per selection. + let hintedFor = ''; + function hintFullSource() { + const ids = state.selection?.kind === 'clip' ? state.selection.ids : []; + const signature = ids.join(','); + if (!ids.length) { hintedFor = ''; return; } + if (signature === hintedFor) return; + try { + const known = store.knownLineage(); + if (model.partsProblem(state.record, ids, {sourceLineage: known})?.code !== 'chain_too_long' + || !model.generationBefore(state.record, ids).some(key => known[key]?.parts >= model.RULES.maxParts)) return; + } catch { return; } + hintedFor = signature; genbar?.say(partsLimitText('chain_source_full', t)); + } const unsubscribe = state.subscribe(event => { + if (event.type === 'selection') hintFullSource(); if (event.type === 'record') { const size = `${state.record.width}x${state.record.height}`; if (!initializing && size !== capabilitySize) { @@ -1315,7 +1367,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const next = content(state.record); if (!suppressSave && !initializing && next !== savedContent && exportStatus?.state === 'done') { exportStatus = null; exportButton.textContent = t('Export', '导出'); } if (!suppressSave && !initializing && next !== savedContent) { savedContent = next; queue.enqueue(state.record); } - completeSources().catch(value => error(value, null, undefined, 'sources')); recoverJobs(); + completeSources().catch(value => error(value, null, undefined, 'sources')); recoverJobs(); music.recover(); } if (event.type === 'selection') { if (state.selection !== initialSelection) initialSelection = null; } if (event.type === 'record' || event.type === 'selection') { updateTop(); updateTransport(); pictureTools.update(); } @@ -1413,7 +1465,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given // A workflow without FreeVideo nodes came to the front: the editing view makes way (unsaved edits stay open). const collapse = () => { void handle.close(); }; window.addEventListener('freevideo-collapse', collapse); - const handle = {element, state, actions, store, adjust, audio, notify, + const handle = {element, state, actions, store, adjust, audio, music, notify, async switchView(view) { if (closing || disposed || view === 'editor') return; if (await handle.close() !== false) await onView(view); @@ -1440,7 +1492,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given } } await store.activateProject(null).catch(() => {}); - rememberView(); disposed = true; guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); + rememberView(); disposed = true; music.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); textTools.dispose(); adjust.dispose(); notes.dispose(); pictureEdit.dispose(); audio.dispose(); headerFit.dispose(); listeners.splice(0).forEach(remove => remove()); @@ -1469,11 +1521,15 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (from?.work) { const work = from.work; const id = work.id || work.video?.replace(/^FreeVideo\//, '').replace(/\/video\.mp4$/, ''); + void store.getSources([`work:${id}`]).then(() => { + const source = store.sources.get(`work:${id}`); + if (source?.path) void store.lineage([{key: `work:${id}`, reference: {work: source.path}}]); + }).catch(() => {}); record = await store.byWork(id); if (!record) { const key = `work:${id}`; await store.getSources([key]); const metadata = store.sources.get(key), g = work.geometry || {}; - record = model.emptyProject({width: even(metadata?.width || g.width || panel.width || 1344), height: even(metadata?.height || g.height || panel.height || 768), - settings: panel.settings || null, name: t('Untitled edit', '未命名剪辑')}); + record = model.addAudioTrack(model.emptyProject({width: even(metadata?.width || g.width || panel.width || 1344), height: even(metadata?.height || g.height || panel.height || 768), + settings: panel.settings || null, name: t('Untitled edit', '未命名剪辑')})); record = model.insertSource(record, {source: {kind: 'work', id}, frames: metadata?.frames || g.frames || Math.round((g.seconds || 15) * 24), hasVideo: true, hasAudio: metadata?.has_audio ?? work.has_audio !== false, text: metadata?.text, at: 0}); if (!metadata) store.cache({[key]: {video: mediaURL(api, work.video), audio: mediaURL(api, work.video), frames: g.frames, @@ -1554,6 +1610,25 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (from?.work && from.extend) { const id = from.work.id || from.work.video?.replace(/^FreeVideo\//, '').replace(/\/video\.mp4$/, ''); const epoch = projectEpoch; + const showFullSourceHint = () => { + const key = `work:${id}`; + void store.getSources([key]).then(() => { + const metadata = store.sources.get(key); + return metadata?.path ? store.lineage([{key, reference: {work: metadata.path}}]) : null; + }).then(values => { + if (closing || disposed || epoch !== projectEpoch) return; + if (!values) return; + const history = values.get(key); + if (!history || history.known !== true || history.parts < model.RULES.maxParts) return; + const clips = state.record.tracks[0].clips, source = clips.find(clip => sourceKey(clip.source) === key); + if (!source) return; + const after = clips.filter(clip => clip.pending === 'draft' && clip.start >= source.start + source.frames + && model.continuesBefore(state.record, clip.id)); + if (!after.length) return; + const problem = model.partsProblem(state.record, after.map(clip => clip.id), {sourceLineage: store.knownLineage()}); + if (problem?.code === 'chain_too_long') genbar?.say(partsLimitText('chain_source_full', t)); + }).catch(() => {}); + }; // Selects the work and, when nothing follows it yet, adds one minimum draft. Runs once; when the // generation probe failed this time it runs after a later probe succeeds. const enter = (select = true) => { @@ -1565,11 +1640,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given const after = state.record.tracks[0].clips.filter(other => other.id !== clip.id && other.start >= edge); const waiting = after.find(other => other.pending === 'draft'); if (select) state.select({kind: 'clip', ids: model.pair(state.record, (waiting || clip).id).map(member => member.id)}); - if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) return true; + if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) { showFullSourceHint(); return true; } if (!store.generationAvailable) return false; const spare = Math.min(...model.pair(state.record, clip.id).map(member => member.length - member.in - member.frames)); tracks.extendClip(clip.id, spare + model.defaultContinueSeconds(capability.maxFollowSeconds, {short: store.continueSeconds != null && store.continueSeconds < model.RULES.fullSeconds}) * model.FPS, {silent: true}); + showFullSourceHint(); return true; }; await store.generationReady(); diff --git a/web/editor_audio.js b/web/editor_audio.js index b5b0fb3d..40c79aca 100644 --- a/web/editor_audio.js +++ b/web/editor_audio.js @@ -1,5 +1,5 @@ import * as model from './editor_model.js'; -import {row, slider, number, disabledReason} from './editor_adjust.js'; +import {row, slider, number, toggle, disabledReason} from './editor_adjust.js'; import * as sound from './editor_audio_model.js'; import * as rules from './editor_audio_rules.js'; import {replaceMode, attachReplacePicker} from './editor_audio_replace.js'; @@ -127,6 +127,11 @@ export function createAudioPanel(host, {state, player, t = en => en, error = () onChange: gain => { const ids = selectedAudio(state).map(clip => clip.id); editingVolume = false; apply(t('Volume', '音量'), record => ids.reduce((next, id) => model.setVolume(next, id, gain / 100), record)); }, }); + // Generated music only: lowered while someone speaks in the other sound. On by default. + const duck = toggle(t('Lower during speech', '对白时降低音量'), { + onChange: on => apply(t('Lower during speech', '对白时降低音量'), record => sound.setDuck(record, selectedAudio(state).filter(sound.duckable).map(clip => clip.id), on)), + }); + duck.classList.add('fv-ed-audio-duck'); form.append(duck); controls.push(duck); const fadeIn = input(form, 'fade-in', t('Fade in', '淡入'), 0, 5, 1 / 24, 'number', { onChange: seconds => apply(t('Fade in', '淡入'), record => sound.setFades(record, selectedAudio(state).map(clip => clip.id), 'fade_in', Math.round(seconds * 24))), }); @@ -177,6 +182,9 @@ export function createAudioPanel(host, {state, player, t = en => en, error = () const locked = selected.some(clip => sound.trackOf(state.record, clip.id).locked); const reason = disabledReason(ctx, {needs: 'audio'}) || (locked ? t('This track is locked', '这条轨道已锁定') : ''); for (const field of [volume, fadeIn, fadeOut]) { field.disabled = !!reason; field.title = ''; field.input.title = ''; } + const music = clips.filter(sound.duckable); + duck.hidden = !music.length || music.length !== clips.length; duck.disabled = !!reason; + if (!duck.hidden) { duck.value = music.every(clip => !!clip.duck); duck.input.indeterminate = music.some(clip => !!clip.duck) && !duck.value; } if (!editingVolume) { const gains = clips.map(clip => Math.round((clip.volume ?? 1) * 100)), mixed = gains.length > 1 && gains.some(gain => gain !== gains[0]); volume.value = mixed ? 100 : gains[0] ?? 100; setMixed(volume, mixed); @@ -314,10 +322,27 @@ export function attachAudioTracks(element, {state, t = en => en, actions = {}}) // A decoded, bounded look-ahead path gives the sample envelope to Web Audio. // The unchanged media path remains active for records without audio edits. -export function createAudioPreview({state, sources, api, volumePreview = new Map(), readSamples = null, error = () => {}}) { +// Speech spans per source for "Lower during speech": asked once, polled while the server works them out. +// A source that cannot be read, or a server without the route, has no speech: nothing is lowered under it. +export function createSpeechSpans(fetchSpans, {retry = 1000} = {}) { + const map = new Map(), asking = new Set(); let version = 0, disposed = false; + const ask = key => { + if (disposed || map.has(key) || asking.has(key)) return; + asking.add(key); + Promise.resolve().then(() => fetchSpans(key)).then(value => { + asking.delete(key); if (disposed) return; + if (value?.state === 'pending' || value?.reason === 'model_missing') { setTimeout(() => ask(key), value?.state === 'pending' ? retry : retry * 10); return; } + map.set(key, Array.isArray(value?.spans) ? value.spans : []); version++; + }).catch(() => { asking.delete(key); if (!disposed) { map.set(key, []); version++; } }); + }; + return {map, want(keys) { keys.forEach(ask); }, get version() { return version; }, dispose() { disposed = true; }}; +} +export function createAudioPreview({state, sources, api, volumePreview = new Map(), readSamples = null, error = () => {}, speech = null}) { let context = null, generation = 0, timer = null, identity = null, pending = false, disposed = false; const cache = new Map(), nodes = new Set(); let origin = 0, cursor = 0, previousPosition = 0; - const edited = record => record.tracks.some(track => track.muted || track.solo || track.clips.some(clip => clip.fade_in || clip.fade_out || clip.crossfade)); + const ducking = (record, rate = 48000) => speech && sound.duckSources(record.tracks).length ? sound.duckRanges(record.tracks, speech.map, rate) : []; + const edited = record => record.tracks.some(track => track.muted || track.solo || track.clips.some(clip => clip.fade_in || clip.fade_out || clip.crossfade)) + || ducking(record).length > 0; function stop() { generation++; clearTimeout(timer); timer = null; pending = false; identity = null; nodes.forEach(node => { try { node.stop(); } catch {} }); nodes.clear(); } async function decoded(clip) { const metadata = sources.get(sourceKey(clip.source)), url = mediaURL(api, metadata?.audio); @@ -343,7 +368,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map return raw(await decoded(clip), clip, offset, count); } async function chunk(record, at, count) { - const rate = context.sampleRate, total = [new Float64Array(count), new Float64Array(count)]; + const rate = context.sampleRate, total = [new Float64Array(count), new Float64Array(count)], lowered = ducking(record, rate); for (const track of record.tracks.filter(track => track.kind === 'audio')) { if (!sound.trackAudible(track, record.tracks)) continue; const block = [new Float64Array(count), new Float64Array(count)], extras = {}; @@ -353,6 +378,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map const a = Math.max(start, at), b = Math.min(stop, at + count); if (b <= a) continue; const data = await prepared(clip, a - start, b - a); + const duck = clip.duck && lowered.length ? sound.duckGain(lowered, a, b - a, rate) : null; data.forEach((channel, c) => { for (let i = 0; i < channel.length; i++) { const local = a - start + i, length = stop - start; const prior = track.clips[track.clips.indexOf(clip) - 1], next = track.clips[track.clips.indexOf(clip) + 1]; @@ -360,6 +386,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map let gain = clip.volume ?? 1; if (!clip.fade_in && (!prior?.source || prior.start + prior.frames !== clip.start || !prior.volume)) gain *= Math.min(1, local / (edge - 1)); if (!clip.fade_out && (!next?.source || next.start !== clip.start + clip.frames || !next.volume)) gain *= Math.min(1, (length - 1 - local) / (edge - 1)); + if (duck) gain *= duck[i]; block[c][a - at + i] = channel[i] * gain; }}); } @@ -396,6 +423,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map } return { render({record, position, playing, playbackRate = 1}) { + speech?.want(sound.duckSources(record.tracks)); if (!edited(record)) { if (identity) stop(); return false; } if (volumePreview.size) record = {...record, tracks: record.tracks.map(track => ({...track, clips: track.clips.map(clip => volumePreview.has(clip.id) ? {...clip, volume: volumePreview.get(clip.id)} : clip)}))}; if (!playing) { stop(); previousPosition = position; return true; } @@ -404,7 +432,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map if (identity !== unavailable) { stop(); identity = unavailable; error(new model.EditError('audio_prepare')); } return true; } - const nextIdentity = JSON.stringify(record.tracks); + const nextIdentity = JSON.stringify(record.tracks) + `:${speech?.version ?? 0}`; if (identity !== nextIdentity || Math.abs(position - previousPosition) > 3) { stop(); identity = nextIdentity; try { context ||= new (window.AudioContext || window.webkitAudioContext)({sampleRate: 48000}); context.resume().catch(error); } @@ -414,6 +442,6 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map } previousPosition = position; return true; }, - dispose() { disposed = true; stop(); cache.clear(); context?.close().catch(() => {}); }, + dispose() { disposed = true; stop(); cache.clear(); speech?.dispose(); context?.close().catch(() => {}); }, }; } diff --git a/web/editor_audio_model.js b/web/editor_audio_model.js index 754cd1f8..5d8a52e2 100644 --- a/web/editor_audio_model.js +++ b/web/editor_audio_model.js @@ -16,6 +16,18 @@ export function audioClipFields(raw, kind) { if (typeof raw.detached_from !== 'string' || !/^c[1-9]\d{0,8}$/.test(raw.detached_from) || raw.detached_from === raw.id || raw.link) fail('link'); fields.detached_from = raw.detached_from; } + // A music piece's preset (what it was made with, prefilled when it is made again) and whether it is + // lowered while the first sound track is heard. + if (raw.music !== undefined) { + const value = raw.music; + if (kind !== 'audio' || !value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join() !== 'mood,style' + || ![value.style, value.mood].every(name => typeof name === 'string' && /^[a-z]{1,20}$/.test(name))) fail('audio'); + fields.music = {style: value.style, mood: value.mood}; + } + if (raw.duck !== undefined) { + if (kind !== 'audio' || typeof raw.duck !== 'boolean') fail('audio'); + fields.duck = raw.duck; + } return fields; } export function audioTrackFields(raw) { @@ -97,6 +109,19 @@ export function setFades(record, ids, name, frames) { } return model.parse(next); } +// Lower during speech is a property of generated music only. +export const duckable = clip => !!clip?.music && !!clip.source; +export function setDuck(record, ids, on) { + if (typeof on !== 'boolean' || !ids.length) fail('audio'); + editable(record, ids); + const next = model.serialize(record); + for (const id of ids) { + const clip = model.find(next, id); + if (trackOf(next, id)?.kind !== 'audio' || !duckable(clip)) fail('audio'); + clip.duck = on; + } + return model.parse(next); +} export function setCrossfade(record, id, frames) { if (!whole(frames)) fail('audio'); const next = model.serialize(record), track = trackOf(next, id), right = model.find(next, id); @@ -205,3 +230,49 @@ export function applyTrackAudio(block, blockStart, track, tracks, {rate = 48000, output.forEach((channel, index) => { for (let i = 0; i < count; i++) channel[i] += added[index][i]; }); return output; } +// Lower during speech: a clip with duck true sits DUCK_DB lower wherever another audible clip has speech, +// with DUCK_RAMP_SECONDS ramps just outside each stretch. freevideo_engine/editor_audio.py has the same rules. +export const DUCK_DB = -8, DUCK_RAMP_SECONDS = .3; +const changedSpeed = clip => (clip.speed ?? 1) !== 1 || !!clip.reverse; +// spans: Map or object of 'kind:id' → [[start, end] seconds]; ranges are timeline samples [first, last). +export function duckRanges(tracks, spans, rate = 48000, fps = 24) { + const found = [], lookup = key => (spans instanceof Map ? spans.get(key) : spans?.[key]) || []; + for (const track of tracks) { + if (track.kind !== 'audio' || !trackAudible(track, tracks)) continue; + for (const clip of track.clips) { + if (!clip.source || clip.duck || (clip.volume ?? 1) === 0 || !clipAudible(clip, track, tracks)) continue; + const first = samples(clip.start, rate, fps), last = samples(end(clip), rate, fps); + const origin = changedSpeed(clip) || clip.speed_phase ? (clip.in * 2000 + (clip.speed_phase || 0) * 20) * rate / 48000 : sourceStart(clip, rate, fps); + const speed = changedSpeed(clip) ? (clip.speed ?? 1) : 1; + for (const [begin, stop] of lookup(`${clip.source.kind}:${clip.source.id}`)) { + let a = (begin * rate - origin) / speed, b = (stop * rate - origin) / speed; + if (clip.reverse) [a, b] = [(last - first) - b, (last - first) - a]; + a = Math.max(first, Math.floor(first + a + .5)); b = Math.min(last, Math.floor(first + b + .5)); + if (b > a) found.push([a, b]); + } + } + } + found.sort((x, y) => x[0] - y[0] || x[1] - y[1]); + const gap = 2 * Math.max(1, Math.floor(DUCK_RAMP_SECONDS * rate + .5)), joined = []; + for (const [a, b] of found) { + if (joined.length && a <= joined.at(-1)[1] + gap) joined.at(-1)[1] = Math.max(joined.at(-1)[1], b); + else joined.push([a, b]); + } + return joined; +} +export function duckGain(ranges, blockStart, count, rate = 48000) { + const gain = new Float64Array(count).fill(1), ramp = Math.max(1, Math.floor(DUCK_RAMP_SECONDS * rate + .5)), low = 10 ** (DUCK_DB / 20); + for (const [first, last] of ranges) { + if (last + ramp <= blockStart || first - ramp >= blockStart + count) continue; + for (let i = 0; i < count; i++) { + const position = blockStart + i; + const into = Math.max(0, Math.min(1, (position - (first - ramp)) / ramp)), outOf = Math.max(0, Math.min(1, ((last + ramp) - position) / ramp)); + gain[i] = Math.min(gain[i], 1 - (1 - low) * Math.min(into, outOf)); + } + } + return gain; +} +// The lowered clips' sources need speech spans from the clips they sit under. +export const duckSources = tracks => tracks.some(track => track.clips.some(clip => clip.duck && clip.source)) + ? [...new Set(tracks.filter(track => track.kind === 'audio').flatMap(track => track.clips) + .filter(clip => clip.source && !clip.duck).map(clip => `${clip.source.kind}:${clip.source.id}`))] : []; diff --git a/web/editor_duration.js b/web/editor_duration.js new file mode 100644 index 00000000..93e49c48 --- /dev/null +++ b/web/editor_duration.js @@ -0,0 +1,147 @@ +// "Duration [n] s" in the generation bar: the length of the parts not generated yet, or of a range drawn on v1. +// A typed length is applied as dragging the right edge would apply it: whole seconds, the same part rules, one step to undo. +import * as model from './editor_model.js'; +const {FPS, RULES} = model; +const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; +const end = clip => clip.start + clip.frames; +const draft = clip => clip.pending != null && !clip.source; +const running = clip => ['queued', 'running'].includes(clip.pending); +const limitsOf = actions => actions.generationLimits?.() || {}; + +// What the field stands for: one run of parts not generated yet on v1, or a range on v1; null hides the field. +export function durationTarget(record, selection) { + if (!selection) return null; + if (selection.kind === 'range') return selection.tracks.includes('v1') ? {kind: 'range', frames: selection.frames, busy: false} : null; + if (selection.kind !== 'clip') return null; + const video = record.tracks.find(track => track.id === 'v1'); + const clips = (video?.clips || []).filter(clip => selection.ids.includes(clip.id)).sort((a, b) => a.start - b.start); + if (!clips.length || !clips.every(draft) || !selection.ids.every(id => clips.some(clip => clip.id === id || clip.link === id))) return null; + if (!clips.every((clip, i) => !i || end(clips[i - 1]) === clip.start && model.continuesBefore(record, clip.id))) return null; + return {kind: 'parts', clips, frames: clips.reduce((sum, clip) => sum + clip.frames, 0), busy: clips.some(running)}; +} + +// The length a typed number becomes, why it was changed (`hint`), and the edit or selection that applies it. +export function durationPlan(record, selection, seconds, limits = {}) { + const target = durationTarget(record, selection); + if (!target || target.busy || typeof seconds !== 'number' || !Number.isFinite(seconds)) return null; + const requested = Math.round(seconds), current = Math.round(target.frames / FPS); + const clamp = (least, most) => { + const value = Math.max(least, Math.min(most, requested)); + return {value, hint: requested < least ? {least} : requested > most ? {most} : null}; + }; + if (target.kind === 'range') { + // A range drawn on its own starts a new video: at least the general minimum, at most the parts one video may have. + const {value, hint} = clamp(RULES.minSeconds, Math.floor(model.maxRangeFrames(record, {...limits, continues: false}) / FPS)); + const {parts, ...rest} = selection; + return {value, hint, current, selection: value === current ? null : {...rest, frames: value * FPS}}; + } + const [first, ...later] = target.clips, laterIds = later.map(clip => clip.id); + const base = laterIds.length ? model.dropPlanned(record, laterIds) : record; + const oldIds = new Set(base.tracks[0].clips.map(clip => clip.id)); + const runOf = r => r.tracks[0].clips.filter(clip => clip.id === first.id || !oldIds.has(clip.id)); + // Ask the very same model operation as a right-edge drag, then report the length it actually accepted. + const stop = Math.max(0, Math.min(1e7, first.start + requested * FPS)); + const refused = problem => Object.assign(new model.EditError(problem.code), problem); + const planned = model.planPlannedEnd(base, first.id, stop, limits), resized = planned.record; + // A failed resize returns its input. For several selected parts that input has already lost the later parts: + // never submit it, even when Enter was pressed on the original value. + if (resized === base) return {value: current, hint: null, current, change: null, error: refused(planned.problem)}; + const laidOut = r => { + const base = laterIds.length ? model.dropPlanned(r, laterIds) : r; + const next = model.planPlannedEnd(base, first.id, stop, limits); + if (next.record === base) throw refused(next.problem); + return next.record; + }; + const value = runOf(resized).reduce((sum, clip) => sum + clip.frames, 0) / FPS; + const hint = requested < planned.least ? {least: planned.least} : requested > planned.most ? {most: planned.most} : null; + if (value === current) return {value, hint, current, change: null}; + // Each part laid out again keeps the description of the part that stood in its place; parts added at the end take the last one's. + const change = r => keepWords(r, r === record ? resized : laidOut(r), target.clips.map(clip => clip.id), runOf); + const result = change(record); + const ids = runOf(result).flatMap(clip => model.pair(result, clip.id).map(member => member.id)); + return {value, hint, current, change, selectionAfter: {kind: 'clip', ids}}; +} + +// What a part says: its description, who wrote it and its note, on each of its tracks. +const WORDS = ['text', 'text_by', 'note']; +function keepWords(before, after, oldIds, runOf) { + const trackId = (record, id) => record.tracks.find(track => track.clips.some(clip => clip.id === id)).id; + const wordsOf = id => new Map(model.pair(before, id).map(clip => [trackId(before, clip.id), clip])); + const olds = oldIds.map(wordsOf), run = runOf(after); + const differs = run.some((clip, i) => model.pair(after, clip.id).some(member => { + const old = olds[Math.min(i, olds.length - 1)].get(trackId(after, member.id)); + return old && WORDS.some(field => old[field] !== member[field]); + })); + if (!differs) return after; + const next = model.serialize(after), clips = next.tracks.flatMap(track => track.clips.map(clip => [clip, track.id])); + run.forEach((clip, i) => { + const old = olds[Math.min(i, olds.length - 1)]; + for (const member of model.pair(after, clip.id)) { + const [target, track] = clips.find(([item]) => item.id === member.id), from = old.get(track); + if (!from) continue; + for (const field of WORDS) { if (from[field] === undefined) delete target[field]; else target[field] = from[field]; } + } + }); + return model.parse(next); +} + +// actions.generationLimits(): what this machine generates in one part; actions.editError(error): an edit that could not be made. +export function createDurationField({state, t, actions}) { + const element = el('div', 'fv-ed-duration'), label = el('label'); + const input = el('input'); input.type = 'number'; input.step = '1'; input.min = '1'; input.inputMode = 'numeric'; + input.setAttribute('aria-label', t('Duration in seconds', '时长（秒）')); + label.append(el('span', 'fv-ed-duration-word', t('Duration', '时长')), input, el('span', 'fv-ed-duration-unit', t('s', '秒'))); + // Styled as the timeline's drag label, with its sentences, above the field. + const hint = el('div', 'fv-ed-duration-hint'); hint.hidden = true; hint.setAttribute('role', 'status'); + element.append(label, hint); element.hidden = true; + // `typing`: a number typed and not applied yet, which a redraw of the bar keeps. + let hintTimer = null, disposed = false, typing = false; + const shown = () => { const target = durationTarget(state.record, state.selection); return target ? String(Math.round(target.frames / FPS)) : ''; }; + function say(text) { + clearTimeout(hintTimer); hint.textContent = text; hint.hidden = false; + hintTimer = setTimeout(() => { hint.hidden = true; }, 2000); + } + function commit() { + if (disposed || element.hidden) return; + typing = false; + if (input.disabled) { input.value = shown(); return; } + const typed = String(input.value ?? '').trim(), seconds = Number(typed); + try { + const plan = typed && Number.isFinite(seconds) ? durationPlan(state.record, state.selection, seconds, limitsOf(actions)) : null; + if (!plan) { input.value = shown(); return; } + clearTimeout(hintTimer); hint.hidden = true; + if (plan.error) throw plan.error; + if (plan.hint?.least != null) say(t(`At least ${plan.hint.least} s`, `至少 ${plan.hint.least} 秒`)); + else if (plan.hint?.most != null) say(t(`Max ${plan.hint.most} s`, `最长 ${plan.hint.most} 秒`)); + if (plan.change) state.apply(t('Extend', '拉长'), plan.change, {selectionAfter: plan.selectionAfter}); + else if (plan.selection) state.select(plan.selection, {history: true}); + } catch (error) { + clearTimeout(hintTimer); hint.hidden = true; + actions.editError?.(error); + } + input.value = shown(); + } + // Typing alone leaves the timeline as it is: a "1" on the way to "14" is not applied. + input.onkeydown = event => { + if (event.key === 'Enter' && !event.isComposing && event.keyCode !== 229) { event.preventDefault(); commit(); return; } + if (event.key !== 'ArrowUp' && event.key !== 'ArrowDown') return; + event.preventDefault(); + const typed = Number(String(input.value ?? '').trim() || shown()); + input.value = String((Number.isFinite(typed) ? Math.round(typed) : Number(shown())) + (event.key === 'ArrowUp' ? 1 : -1)); + commit(); + }; + input.oninput = () => { typing = true; }; + input.onblur = () => commit(); + // The browser's own step buttons. + input.onchange = () => commit(); + // While these parts are made the field stays in place, greyed like the description. + function update(busy = false) { + const target = durationTarget(state.record, state.selection); + element.hidden = !target; + if (!target) { clearTimeout(hintTimer); hint.hidden = true; return; } + input.disabled = busy || target.busy; + if (input.disabled) typing = false; + if (!typing) input.value = String(Math.round(target.frames / FPS)); + } + return {element, input, update, commit, dispose() { disposed = true; clearTimeout(hintTimer); }}; +} diff --git a/web/editor_genbar.js b/web/editor_genbar.js index b6992f0c..7a77500e 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -1,11 +1,13 @@ // The editor owns its generation controls; progress can use the optional component. import {createFirstFrameSwitch} from './editor_first_frame.js'; +import {createDurationField} from './editor_duration.js'; import {createEditorReferences} from './editor_references.js'; import {createExpandButton} from './editor_expand.js'; import {createRequestAssets} from './request_assets.js'; import {issueButton} from './issue_button.js'; import {waitingText} from './chain_support.js'; import {showEmptyPrompt} from './empty_prompt_notice.js'; +import {segmented} from './editor_adjust.js'; const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) @@ -65,6 +67,42 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (assets) host.prepend(assets.element); const firstFrame = createFirstFrameSwitch(host, {state, t, actions}); description.after(firstFrame.element); + // On the music track: a style and a mood above the description, each a row; the music's own first-use download + // in the same line above the bar as the video's. + const music = actions.music || null; + const presets = music ? el('div', 'fv-ed-music-presets') : null; + const musicAssets = music ? el('div', 'fv-request-assets fv-muted') : null; + let musicBytes = null, musicBytesAt = 0; + const size = bytes => bytes >= 1e9 ? (bytes / 1e9).toFixed(1) + '\u00a0GB' : Math.max(1, Math.round(bytes / 1e6)) + '\u00a0MB'; + const pickers = {}; + if (music) { + for (const [key, label, options] of [['style', t('Style', '风格'), music.styles()], ['mood', t('Mood', '情绪'), music.moods()]]) { + const row = el('div', 'fv-ed-music-row'), name = el('span', 'fv-ed-music-label', label); + const picker = segmented(options.map(([value, text]) => ({value, label: text})), {onChange: value => { music.choose({[key]: value}); render(); }}); + picker.setAttribute('aria-label', label); pickers[key] = picker; + row.append(name, picker); presets.append(row); + } + presets.hidden = true; musicAssets.hidden = true; + description.before(presets); host.prepend(musicAssets); + } + async function showMusicAssets(shown) { + if (!music) return; + if (!shown) { musicAssets.hidden = true; return; } + // Asked again at most every 5 s while a stretch of the music track is chosen. + if (musicBytes === null || Date.now() - musicBytesAt > 5000) { + musicBytesAt = Date.now(); + musicBytes = await music.downloadBytes(); + if (disposed) return; + } + musicAssets.textContent = musicBytes > 0 ? t(`Downloads before generating: music model ${size(musicBytes)}`, `生成前会先下载：配乐模型 ${size(musicBytes)}`) : ''; + musicAssets.hidden = !(musicBytes > 0) || host.dataset.mode !== 'selected' || !music.target(); + } + // How long the parts not generated yet are, between the description and the first-frame switch. + const duration = createDurationField({state, t, actions}); + description.after(duration.element); + // Where a narrow English bar starts Generate on a line of its own (editor.css); hidden elsewhere. + const lineBreak = el('span', 'fv-ed-genbar-break'); lineBreak.setAttribute('aria-hidden', 'true'); + generate.before(lineBreak); host.lang = t('en', 'zh-CN'); const failure = el('div', 'fv-ed-gen-failure fv-result-links'), failureBar = el('div', 'fv-ed-failure-actions fv-actions'); const failureReason = el('span', 'fv-ed-failure-reason'); failureReason.setAttribute('role', 'status'); const report = el('button', 'fv-ed-failure-report fv-action fv-action-secondary', t('Download diagnostic report', '下载诊断报告')); report.type = 'button'; @@ -119,9 +157,11 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { } failure.hidden = !selectedFailure; failureReason.textContent = selectedFailure?.text || ''; - report.hidden = !selectedFailure; + // A piece of music has no generation receipt to report. + report.hidden = !selectedFailure || selectedFailure.report === false; const running = ['queued', 'running', 'cancelling'].includes(status?.state); - const selected = !!selection && (!!previewSelection || (actions.supportedSelection?.() ?? true)); + const musical = !!music?.target(selection); + const selected = !!selection && (!!previewSelection || musical || (actions.supportedSelection?.() ?? true)); // While a part generates, choosing another place offers Generate again; it is queued behind. // A press is being sent for the parts in `pressing`; only that same selection waits for it. const sending = pressing.has(JSON.stringify(state.selection ?? null)), starting = pressing.size > 0 && !running; @@ -134,13 +174,13 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { const stopped = ['failed', 'cancelled', 'interrupted'].includes(status?.state); host.dataset.mode = busy ? 'running' : selected ? 'selected' : 'idle'; hint.hidden = !busy && !other; - // The description and Generate stay where they are while a part is made, greyed, so nothing around them moves. - input.hidden = generate.hidden = false; + // The description and Generate stay where they are while a part is made, greyed, so nothing around them moves; + // music being made shows only its progress and Cancel. + input.hidden = false; generate.hidden = musical && busy; // A choice that will move to an empty spot can still be described first. const writable = selected || !previewSelection && !!selection && !!actions.selectionMovable?.(); input.disabled = !writable || busy; - input.placeholder = writable ? t('Describe this part', '描述这一段') - : ''; + input.placeholder = !writable ? '' : musical ? t('Describe the music', '描述配乐') : t('Describe this part', '描述这一段'); input.setAttribute('aria-label', t('Description', '描述')); generate.classList.toggle('fv-primary', selected && !busy); // A cancel already accepted is not offered again; the bar says it is stopping until the run ends. @@ -169,15 +209,28 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { hint.replaceChildren(...shown.map(([cls, text]) => el('span', cls, text))); hint.title = shown.map(([, text]) => text).join(''); } + duration.update(busy); firstFrame.update(busy); + if (presets) { + presets.hidden = !musical || !selected && !busy; + if (!presets.hidden) { + const value = music.preset(selection); + for (const key of ['style', 'mood']) { pickers[key].value = value[key]; pickers[key].disabled = busy || !!previewSelection; } + } + host.classList.toggle('fv-ed-genbar-music', !presets.hidden); + } references?.update(busy, busy ? undefined : actions.references.visible() || false); - generate.textContent = t('Generate', '生成'); + // A piece already made, failed or stopped is made again in place. + const made = musical && !!music.target(selection)?.clip && music.target(selection).clip.pending !== 'draft'; + const verb = made ? t('Regenerate', '重新生成') : t('Generate', '生成'); + generate.textContent = verb; // Never disabled for a reason the press itself can explain or fix; only until the server says it can generate. generate.disabled = busy || !!previewSelection || !!actions.generationWaiting?.(); resizeDescription(); input.dispatchEvent(new Event('change')); generate.title = ''; - if (assets) { if (selected && !busy) assets.refresh(); else assets.element.hidden = true; } + if (assets) { if (selected && !busy && !musical) assets.refresh(); else assets.element.hidden = true; } + showMusicAssets(selected && !busy && musical && !previewSelection); if (selected && !busy) { const key = JSON.stringify(selection); if (estimateCache?.key === key) generate.textContent += estimateCache.label; @@ -185,7 +238,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (disposed || current !== serial) return; const label = seconds == null ? '' : ' · ' + timeEstimate(seconds, t); estimateCache = {key, label}; - generate.textContent = t('Generate', '生成') + label; + generate.textContent = verb + label; } } function resizeDescription() { @@ -281,7 +334,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (!value && note.contains(retryService)) { retryService.remove(); note.hidden = true; } showService(); }, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); if (bound && input.value !== bound.text) { dirty = true; written = input.value; } render(); }, - dispose() { disposed = true; ++serial; clearTimeout(noteTimer); clearTimeout(refNoticeTimer); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); assets?.dispose(); }}; + dispose() { disposed = true; ++serial; clearTimeout(noteTimer); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; } // A missing optional module leaves the editor's existing progress in place. diff --git a/web/editor_lifecycle.js b/web/editor_lifecycle.js index eed7b512..4054e097 100644 --- a/web/editor_lifecycle.js +++ b/web/editor_lifecycle.js @@ -6,9 +6,11 @@ export function discardEmptyEdit(record, initial, {busy = false, hasMedia = fals if (busy || hasMedia || record.keep_empty || !untitled(record.name) || record.tracks.some(track => track.clips.length) || record.replacements?.length) return false; // Compare settings, track controls and sparse project fields with initial(). + // An untouched empty second sound track does not make an edit differ from a new one. + const bare = track => track.id === 'a2' && !track.clips.length && Object.keys(track).every(key => ['id', 'kind', 'clips'].includes(key)); const content = value => { const {id, revision, serial, name, ...rest} = value; - return rest; + return {...rest, tracks: rest.tracks.filter(track => !bare(track))}; }; return JSON.stringify(sorted(content(record))) === JSON.stringify(sorted(content(initial))); } diff --git a/web/editor_model.js b/web/editor_model.js index 8439c36f..d34926f0 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -11,7 +11,7 @@ export const FPS = 24; // followSeconds long; a video with a continuing part shorter than fullSeconds has at most shortMaxParts parts unless // every part sees the first frame, any other at most maxParts. A new continuation starts at continueSeconds, or at // shortContinueSeconds where a full part is too much for the device. -export const RULES = Object.freeze({maxSeconds: 15, singleSeconds: 30, minSeconds: 2, tailSeconds: 5, +export const RULES = Object.freeze({maxSeconds: 15, singleSeconds: 30, minSeconds: 2, tailSeconds: 5, lastSeconds: CHAIN_RULES.min_tail_seconds, followSeconds: CHAIN_RULES.min_part_seconds, fullSeconds: CHAIN_RULES.full_part_seconds, continueSeconds: CHAIN_RULES.continue_seconds, shortContinueSeconds: CHAIN_RULES.short_continue_seconds, step: 17, first: 5, overlap: 22, @@ -644,16 +644,20 @@ export const setPictureFit = (record, id, fit) => setPicture(record, id, {fit: p // MIN_PART_FRAMES a continuing part is too short, below FULL_PART_FRAMES it is a short part. const partFrames = seconds => RULES.overlap + RULES.step * Math.ceil((FPS * seconds - (RULES.step - 1)) / RULES.step); export const MIN_PART_FRAMES = partFrames(RULES.followSeconds), FULL_PART_FRAMES = partFrames(RULES.fullSeconds); +// The last part of a video, one no other part continues, may be as short as lastSeconds. +export const MIN_TAIL_FRAMES = partFrames(RULES.lastSeconds); // The most parts one video may have (chain_plan.parts_limit). export const partsLimit = (short, firstFrame = false) => short && !firstFrame ? Math.min(RULES.maxParts, RULES.shortMaxParts) : RULES.maxParts; // The shortest a part that continues another may be; a device that cannot make a part that long keeps the general minimum. const followLeast = maxFollowSeconds => maxFollowSeconds >= RULES.followSeconds ? RULES.followSeconds : RULES.minSeconds; -// A length to continue by, in whole seconds, raised to one that parts of followSeconds..maxFollowSeconds can add up to. -export function continuedSeconds(seconds, maxFollowSeconds = RULES.maxSeconds) { +// A length to continue by, in whole seconds, raised to one that parts of followSeconds..maxFollowSeconds can add up to; +// with `tail` (no part continues the last one) that part may be as short as lastSeconds. +export function continuedSeconds(seconds, maxFollowSeconds = RULES.maxSeconds, {tail = true} = {}) { check(typeof seconds === 'number' && Number.isFinite(seconds) && typeof maxFollowSeconds === 'number' && Number.isFinite(maxFollowSeconds), 'range'); const least = followLeast(maxFollowSeconds), follow = Math.max(least, Math.floor(Math.min(RULES.maxSeconds, maxFollowSeconds))); - const length = Math.max(least, Math.round(seconds)); - return Math.max(length, Math.ceil(length / follow) * least); + const last = tail ? Math.min(least, RULES.lastSeconds) : least; + const length = Math.max(last, Math.round(seconds)); + return Math.max(length, (Math.ceil(length / follow) - 1) * least + last); } // The length a new continuation starts with: a full part, or a short one where the device cannot make a full part // or `short` asks for it. @@ -661,7 +665,7 @@ export function defaultContinueSeconds(maxFollowSeconds = RULES.maxSeconds, {sho const full = !short && maxFollowSeconds >= RULES.fullSeconds; return continuedSeconds(full ? RULES.continueSeconds : RULES.shortContinueSeconds, maxFollowSeconds); } -function segmentLengths(frames, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false) { +function segmentLengths(frames, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false, tail = true) { check(typeof maxSeconds === 'number' && Number.isFinite(maxSeconds) && maxSeconds >= RULES.minSeconds, 'range'); check(typeof maxFollowSeconds === 'number' && Number.isFinite(maxFollowSeconds) && maxFollowSeconds >= RULES.minSeconds, 'range'); const max = Math.floor(Math.min(RULES.maxSeconds, maxSeconds)) * FPS, follow = Math.floor(Math.min(RULES.maxSeconds, maxFollowSeconds)) * FPS, lengths = []; @@ -673,7 +677,7 @@ function segmentLengths(frames, maxSeconds = RULES.maxSeconds, maxFollowSeconds // Every part after the first continues the one before it, as chain_plan lays them out: the parts after the first // share what is left evenly, and a rest too short for them comes off the first. They are fullSeconds or longer // whenever the length allows it, else at least followSeconds. - const seconds = continues && follow >= RULES.followSeconds * FPS ? continuedSeconds(left / FPS, follow / FPS) : left / FPS; + const seconds = continues && follow >= RULES.followSeconds * FPS ? continuedSeconds(left / FPS, follow / FPS, {tail}) : left / FPS; for (const least of [RULES.fullSeconds, RULES.followSeconds]) { if (follow < least * FPS) continue; const head = continues ? 0 : Math.min(max / FPS, seconds), rest = seconds - head, count = Math.ceil(rest / (follow / FPS)); @@ -687,8 +691,8 @@ function segmentLengths(frames, maxSeconds = RULES.maxSeconds, maxFollowSeconds } while (left) { const length = Math.min(left, lengths.length || continues ? follow : max); lengths.push(length); left -= length; } const last = lengths.length - 1; - if (last > 0 && lengths[last] < Math.min(follow, RULES.tailSeconds * FPS)) { - const both = lengths[last - 1] + lengths[last]; lengths[last - 1] = Math.min(last === 1 && !continues ? max : follow, Math.max(both - follow, Math.ceil(both / 2 / FPS) * FPS)); lengths[last] = both - lengths[last - 1]; + if (last > 0 && lengths[last] < Math.min(follow, RULES.tailSeconds * FPS) && !(continues && tail && follow >= RULES.followSeconds * FPS && lengths[last] >= RULES.lastSeconds * FPS)) { + const both = lengths[last - 1] + lengths[last]; lengths[last - 1] = Math.min(last === 1 && !continues ? max : follow, Math.max(both - follow, Math.ceil(both / 2 / FPS) * FPS, last > 1 || continues ? followLeast(follow / FPS) * FPS : 0)); lengths[last] = both - lengths[last - 1]; } return lengths; } @@ -737,10 +741,10 @@ export function continuesBefore(record, id) { return !!video && trackBy(record, 'v1').clips.includes(video) && !!partBefore(record, video); } // The lengths one range is generated in, as planRange lays them out on empty tracks. -export const partLengths = (frames, {maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false} = {}) => segmentLengths(frames, maxSeconds, maxFollowSeconds, continues); +export const partLengths = (frames, {maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false, tail = true} = {}) => segmentLengths(frames, maxSeconds, maxFollowSeconds, continues, tail); // One video is generated in at most partsLimit parts: the longest range that still fits. A device whose continuing // parts are all short has the short-part limit. -export function maxRangeFrames(record, {continues = false, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, firstFrame = false}) { +export function maxRangeFrames(record, {continues = false, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, firstFrame = record.settings?.first_frame_reference === true}) { const before = continues; const follow = Math.floor(Math.min(RULES.maxSeconds, maxFollowSeconds)) * FPS; const parts = partsLimit(follow < RULES.fullSeconds * FPS, firstFrame); @@ -748,34 +752,38 @@ export function maxRangeFrames(record, {continues = false, maxSeconds = RULES.ma return before ? (parts - 1) * follow : Math.floor(Math.min(RULES.maxSeconds, maxSeconds)) * FPS + (parts - 1) * follow; } // Manual boundaries use device limits, not the automatic fifteen-second preference. -export function rangeParts(record, {start, frames, parts, continues, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds}) { +export function rangeParts(record, {start, frames, parts, continues, tail, maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds}) { // A range that starts on a part pulled out of the one before it keeps that part's shorter limit. const head = trackBy(record, 'v1').clips.find(clip => clip.start === start); const following = continues ?? !!(head && partBefore(record, head)), first = following ? maxFollowSeconds : maxSeconds; - // A part that continues another is at least followSeconds long. - const least = index => (index || following ? followLeast(maxFollowSeconds) : RULES.minSeconds) * FPS; + // A part that continues another is at least followSeconds long; the last part of a video, one no part continues, lastSeconds. + const last = tail ?? !trackBy(record, 'v1').clips.some(clip => clip.start === start + frames && partBefore(record, clip)); + const least = index => (Array.isArray(parts) && index === parts.length - 1 && last ? Math.min(RULES.lastSeconds, followLeast(maxFollowSeconds)) + : index || following ? followLeast(maxFollowSeconds) : RULES.minSeconds) * FPS; const valid = Array.isArray(parts) && parts.length > 0 && parts.length <= RULES.maxClips && parts.every((count, i) => whole(count, least(i)) && count % FPS === 0 && count <= Math.floor(i ? maxFollowSeconds : first) * FPS) && parts.reduce((sum, count) => sum + count, 0) === frames; - return valid ? [...parts] : partLengths(frames, {maxSeconds: first, maxFollowSeconds, continues: following}); + return valid ? [...parts] : partLengths(frames, {maxSeconds: first, maxFollowSeconds, continues: following, tail: last}); } -export function adjustPartBoundary(parts, index, leftFrames, {maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false} = {}) { +export function adjustPartBoundary(parts, index, leftFrames, {maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds, continues = false, tail = false} = {}) { check(Array.isArray(parts) && whole(index) && index < parts.length - 1 && Number.isFinite(leftFrames), 'range'); check(parts.every(count => whole(count, RULES.minSeconds * FPS) && count % FPS === 0), 'range'); const minimum = RULES.minSeconds * FPS, total = parts[index] + parts[index + 1]; const leftMax = Math.floor(index ? maxFollowSeconds : maxSeconds) * FPS, rightMax = Math.floor(maxFollowSeconds) * FPS; // Every part that continues another is at least followSeconds long; a pair too short for that keeps the general minimum. + // The part after the boundary may be as short as lastSeconds when no part continues it (`tail`). const follows = followLeast(maxFollowSeconds) * FPS, leftLeast = index || continues ? follows : minimum; - const tight = total >= leftLeast + follows; - const low = Math.max(tight ? leftLeast : minimum, total - rightMax), high = Math.min(leftMax, total - (tight ? follows : minimum)); + const rightLeast = tail ? Math.min(follows, RULES.lastSeconds * FPS) : follows; + const tight = total >= leftLeast + rightLeast; + const low = Math.max(tight ? leftLeast : minimum, total - rightMax), high = Math.min(leftMax, total - (tight ? rightLeast : minimum)); check(Number.isFinite(low) && Number.isFinite(high) && low <= high, 'range'); const desired = Math.round(leftFrames / FPS) * FPS, left = Math.max(low, Math.min(high, desired)); const result = [...parts]; result[index] = left; result[index + 1] = total - left; // The left part also stops where the part after it would fall under followSeconds. return {parts: result, limited: desired >= leftMax && left === leftMax ? leftMax / FPS : total - desired >= rightMax && total - left === rightMax ? rightMax / FPS - : tight && desired > left && left === total - follows ? left / FPS : null}; + : tight && desired > left && left === total - rightLeast ? left / FPS : null}; } export function resizePlannedBoundary(record, leftId, rightId, frame, limits = {}) { const left = find(record, leftId), right = find(record, rightId); @@ -785,9 +793,11 @@ export function resizePlannedBoundary(record, leftId, rightId, frame, limits = { check(members.every(clip => clip.pending && !clip.source), 'unsupported'); const video = trackOf(record, leftId).id === 'v1' ? left : find(record, left.link); const preceding = !!video && !!partBefore(record, video); + const rightVideo = trackOf(record, rightId).id === 'v1' ? right : find(record, right.link); + const tail = !trackBy(record, 'v1').clips.some(clip => continuing(clip, rightVideo)); const maxSeconds = limits.maxSeconds ?? RULES.maxSeconds, maxFollowSeconds = limits.maxFollowSeconds ?? maxSeconds; const result = adjustPartBoundary([left.frames, right.frames], 0, frame - left.start, - {maxSeconds: preceding ? maxFollowSeconds : maxSeconds, maxFollowSeconds, continues: preceding}); + {maxSeconds: preceding ? maxFollowSeconds : maxSeconds, maxFollowSeconds, continues: preceding, tail}); const next = copy(record); detachAttempt(next, members.map(clip => clip.id)); for (const old of members) find(next, old.id).part = left.part; for (const clip of pair(next, leftId)) { clip.frames = clip.length = result.parts[0]; clearPlan(clip); } @@ -795,33 +805,116 @@ export function resizePlannedBoundary(record, leftId, rightId, frame, limits = { return finish(next); } export function resizePlanned(record, id, edge, frame, limits = {}) { - check(['start', 'end'].includes(edge) && whole(frame), 'range'); + if (edge !== 'end') return layoutPlanned(record, id, edge, frame, limits); + return planPlannedEnd(record, id, frame, limits).record; +} +// Immutable records let repeated pointer previews and release share their last complete layout. +const plannedEndCache = new WeakMap(); +export function planPlannedEnd(record, id, frame, limits = {}) { + check(whole(frame), 'range'); + const geometry = plannedGeometry(record, id, limits), {primary: first, video} = geometry; + const most = maxRangeFrames(record, {...limits, continues: continuesBefore(record, id)}); + const capped = Math.min(frame, first.start + most), position = Math.round(capped / FPS) * FPS; + const key = JSON.stringify(Object.keys(limits).sort().map(key => [key, limits[key]])); + const cached = plannedEndCache.get(record); + if (cached?.id === id && cached.position === position && cached.key === key) return cached.result; + const clips = trackBy(record, 'v1').clips, draft = clip => clip?.pending && !clip.source; + const earlier = []; + for (let before = video && partBefore(record, video); draft(before); before = partBefore(record, before)) earlier.unshift(before); + const before = partBefore(record, earlier[0] || video); + const lineage = before ? madeLineage(record, before) : {parts: 0, short: false}; + const firstFrame = limits.firstFrame ?? record.settings?.first_frame_reference === true; + const candidates = new Map(); + const candidate = frame => { + const {start, stop} = fitPlanned(geometry, 'end', frame); + if (candidates.has(stop)) return candidates.get(stop); + // Only numbers and the existing draft lineage participate in the search. No project copy, parse, + // finish or split occurs until a length has been chosen. Use the same generation grid as partsProblem. + const head = {id: (video || first).id, start, frames: stop - start, part: first.part || Symbol(), + follows: video?.follows, source: null, job: null}; + const context = {tracks: [{id: 'v1', clips: [...(before ? [before] : []), ...earlier, head]}]}; + const lengths = rangeParts(context, {start, frames: stop - start, ...limits, tail: geometry.tail}); + lengths[lengths.length - 1] = stop - start - lengths.slice(0, -1).reduce((sum, count) => sum + count, 0); + const run = [...earlier]; let at = start; + for (const [index, frames] of lengths.entries()) { + const clip = {...head, id: index ? Symbol() : head.id, start: at, frames}; + if (index) clip.follows = run.at(-1).id; + run.push(clip); at += frames; + } + // Resizing may detach a following draft, or reach it. Splitting transfers its follows link to the last part. + if (video) for (let next = clips.find(clip => clip.start === at && clip.id !== video.id); draft(next);) { + if (at === stop && next.follows === video.id) next = {...next, follows: run.at(-1).id}; + if (!continuing(next, run.at(-1))) break; + run.push(next); at = end(next); + next = clips.find(clip => clip.start === at); + } + let problem = null; + try { if (video) problem = plannedPartsProblem(generationPlan(context, run), () => lineage, firstFrame); } + catch (error) { + if (!(error instanceof EditError) || error.code !== 'range') throw error; + // A frame-split tail may not fit the generation grid. Skip this candidate, including in hint bounds. + problem = {code: error.code}; + } + const result = {frame, seconds: (stop - start) / FPS, problem, + allowed: !['range', 'chain_too_long', 'chain_short_parts'].includes(problem?.code)}; + candidates.set(stop, result); return result; + }; + const initial = candidate(capped), wanted = initial.seconds; + const least = continuesBefore(record, id) + ? continuedSeconds(0, limits.maxFollowSeconds ?? limits.maxSeconds) : RULES.minSeconds; + let chosen = initial.allowed ? initial : null; + for (let seconds = wanted - 1; !chosen && seconds >= least; seconds--) { + const next = candidate(first.start + seconds * FPS); if (next.allowed) chosen = next; + } + for (let seconds = wanted + 1; !chosen && seconds <= most / FPS; seconds++) { + const next = candidate(first.start + seconds * FPS); if (next.allowed) chosen = next; + } + // Bounds describe the entire feasible set, including any holes, rather than just the chosen adjustment. + for (let seconds = least; seconds <= most / FPS; seconds++) candidate(first.start + seconds * FPS); + const feasible = [...candidates.values()].filter(value => value.allowed).map(value => value.seconds); + const result = {record: chosen ? layoutPlanned(record, id, 'end', chosen.frame, limits) : record, + least: feasible.length ? Math.min(...feasible) : null, most: feasible.length ? Math.max(...feasible) : null, + problem: chosen ? null : initial.problem}; + plannedEndCache.set(record, {id, position, key, result}); + return result; +} +function plannedGeometry(record, id, limits) { const members = pair(record, id); check(members.length, 'missing'); busy(members); check(members.every(clip => clip.pending && !clip.source), 'unsupported'); const primary = members[0]; - const video = trackOf(record, id).id === 'v1' ? primary : find(record, primary.link); + const video = members.find(clip => trackOf(record, clip.id).id === 'v1'); const maxFollowSeconds = limits.maxFollowSeconds ?? limits.maxSeconds ?? RULES.maxSeconds; - // A planned part that continues another is at least followSeconds long. - const minimum = (video && partBefore(record, video) ? followLeast(maxFollowSeconds) : RULES.minSeconds) * FPS; + // A planned part that continues another is at least followSeconds long, unless no part continues it. + const follows = !!video && !!partBefore(record, video), tail = !video || !trackBy(record, 'v1').clips.some(clip => continuing(clip, video)); + const minimum = (follows ? Math.min(followLeast(maxFollowSeconds), tail ? RULES.lastSeconds : Infinity) : RULES.minSeconds) * FPS; const neighbors = members.flatMap(clip => trackOf(record, clip.id).clips.filter(other => other.id !== clip.id)); const low = Math.max(0, ...neighbors.filter(clip => end(clip) <= primary.start).map(end)); const high = Math.min(1e7, ...neighbors.filter(clip => clip.start >= end(primary)).map(clip => clip.start)); + return {members, primary, video, minimum, low, high, maxFollowSeconds, follows, tail}; +} +function fitPlanned({primary, minimum, low, high, maxFollowSeconds, follows, tail}, edge, frame) { const position = Math.round(frame / FPS) * FPS; let start = edge === 'start' ? Math.max(low, Math.min(end(primary) - minimum, position)) : primary.start; let stop = edge === 'end' ? Math.min(high, Math.max(start + minimum, position)) : end(primary); - if (minimum > RULES.minSeconds * FPS) { - // Lengths no run of continuing parts can fill (16–19 s at fifteen-second parts) are raised to the next one that fits. - const fitted = continuedSeconds((stop - start) / FPS, maxFollowSeconds) * FPS; + if (follows) { + // Lengths no run of continuing parts can fill are raised to the next one that fits. + const fitted = continuedSeconds((stop - start) / FPS, maxFollowSeconds, {tail}) * FPS; if (fitted > stop - start) { if (edge === 'end' && start + fitted <= high) stop = start + fitted; else if (edge === 'start' && stop - fitted >= low) start = stop - fitted; } } + return {start, stop}; +} +function layoutPlanned(record, id, edge, frame, limits = {}) { + check(['start', 'end'].includes(edge) && whole(frame), 'range'); + const geometry = plannedGeometry(record, id, limits), {members, primary, tail} = geometry; + const {start, stop} = fitPlanned(geometry, edge, frame); let next = copy(record); detachAttempt(next, members.map(clip => clip.id)); const part = primary.part || freshId(next); for (const clip of pair(next, id)) { clip.start = start; clip.frames = clip.length = stop - start; clip.part = part; clearPlan(clip); } next = finish(next); - const lengths = rangeParts(next, {start, frames: stop - start, ...limits}); + const lengths = rangeParts(next, {start, frames: stop - start, ...limits, tail}); let positionAt = start, currentId = id; for (const count of lengths.slice(0, -1)) { positionAt += count; next = split(next, currentId, positionAt); @@ -1078,12 +1171,21 @@ function madeLineage(record, clip) { // part shorter than followSeconds, a video continued through more than maxParts parts, or one with a short // continuing part through more than partsLimit allows. Parts made before the request count as the timeline shows // them. Returns {code, parts, limit} for the first problem, or null. -export function partsProblem(record, ids, {firstFrame = record.settings?.first_frame_reference === true} = {}) { +export function partsProblem(record, ids, {firstFrame = record.settings?.first_frame_reference === true, sourceLineage = {}} = {}) { const rows = generationPlan(record, generationClips(record, ids)); - if (rows.some(row => row.before && row.request_frames < MIN_PART_FRAMES)) return {code: 'chain_part_too_short', parts: null, limit: RULES.followSeconds}; + return plannedPartsProblem(rows, clip => { + // A video chosen from the library counts the parts it was made in, when they are known. + const inherited = clip.source ? sourceLineage[`${clip.source.kind}:${clip.source.id}`] : null; + const known = inherited && inherited.known !== false && Number.isInteger(inherited.parts) && inherited.parts > 0; + return known ? {parts: inherited.parts, short: inherited.short === true} : madeLineage(record, clip); + }, firstFrame); +} +function plannedPartsProblem(rows, previousLineage, firstFrame) { + const followed = new Set(rows.map(row => row.before?.id)); + if (rows.some(row => row.before && row.request_frames < (followed.has(row.clip.id) ? MIN_PART_FRAMES : MIN_TAIL_FRAMES))) return {code: 'chain_part_too_short', parts: null, limit: RULES.followSeconds}; const lineage = new Map(); for (const row of rows) { - const previous = !row.before ? {parts: 0, short: false} : lineage.get(row.before.id) ?? madeLineage(record, row.before); + const previous = !row.before ? {parts: 0, short: false} : lineage.get(row.before.id) ?? previousLineage(row.before); lineage.set(row.clip.id, {parts: previous.parts + 1, short: previous.short || !!row.before && row.request_frames < FULL_PART_FRAMES}); } const values = [...lineage.values()], parts = Math.max(...values.map(value => value.parts)); @@ -1100,7 +1202,8 @@ export function setContinuationSeconds(record, id, seconds, {maxSeconds = RULES. check(members.every(clip => clip.pending && !clip.source), 'unsupported'); const video = members.find(clip => trackOf(record, clip.id).id === 'v1'); check(!!video && !!partBefore(record, video), 'unsupported'); - check(Number.isInteger(seconds) && seconds >= followLeast(maxFollowSeconds) + const last = !trackBy(record, 'v1').clips.some(clip => continuing(clip, video)); + check(Number.isInteger(seconds) && seconds >= (last ? Math.min(RULES.lastSeconds, followLeast(maxFollowSeconds)) : followLeast(maxFollowSeconds)) && seconds <= Math.floor(Math.min(RULES.maxSeconds, maxFollowSeconds)), 'range'); const delta = seconds * FPS - video.frames; if (!delta) return record; diff --git a/web/editor_music.js b/web/editor_music.js new file mode 100644 index 00000000..ff39c665 --- /dev/null +++ b/web/editor_music.js @@ -0,0 +1,177 @@ +// Music on the second sound track through the generation bar: a stretch chosen there (or a piece of music +// there) is what Generate makes, with the bar's own description box, a style and a mood. The pieces follow the +// music job's progress and take its file in place; the bar shows the job of the piece chosen. +import * as music from './editor_music_model.js'; +import {EditError} from './editor_model.js'; +import {timeEstimate} from './editor_genbar.js'; + +export const STYLE_NAMES = {cinematic: ['Cinematic', '电影感'], pop: ['Pop', '流行'], electronic: ['Electronic', '电子'], + piano: ['Piano', '钢琴'], folk: ['Folk', '民谣'], ambient: ['Ambient', '氛围']}; +export const MOOD_NAMES = {calm: ['Calm', '舒缓'], upbeat: ['Upbeat', '明快'], tense: ['Tense', '紧张'], warm: ['Warm', '温暖'], sad: ['Sad', '悲伤']}; + +// What the bar says about a piece's job: queued, downloading on first use, waiting for a video, or composing. +export function musicBarStatus(job, t, clock = Date.now) { + if (!job || !['queued', 'running'].includes(job.state)) return null; + let stage = '', time = ''; + if (job.state === 'queued') stage = t('Queued', '排队中'); + else if (job.phase === 'downloading') stage = t('Downloading files for generation', '正在下载生成所需文件'); + else if (job.phase === 'waiting') stage = t('Queued · waiting for the previous one', '排队中 · 等前一个完成'); + else if (job.phase === 'composing') { + stage = t('Generating music', '正在生成配乐'); + // The server's elapsed time, carried forward between polls. + const elapsed = Number.isFinite(job.elapsed) ? job.elapsed + Math.max(0, (clock() - (job.seen_at ?? clock())) / 1000) : null; + if (Number.isFinite(job.estimate) && elapsed != null) time = elapsed > job.estimate * 1.2 + 5 + ? t('Taking longer than expected', '比预计久，仍在生成') : timeEstimate(Math.max(1, job.estimate - elapsed), t); + } else stage = t('Preparing', '正在准备'); + return {state: 'running', blocks: [], music: true, words: {stage, ...(time ? {time} : {})}}; +} + +// A failed piece's sentence on the bar's failure line. +export function musicFailureText(reason, t) { + return { + music_download_failed: t('Download paused. Retry to resume.', '下载已暂停，重试可继续。'), + disk_space: t('The disk is full. Free some space, then generate again.', '磁盘空间不足，请清理出空间后再生成。'), + out_of_memory: t('Not enough memory to generate the music. Close other programs that use the graphics card, then generate again.', '生成配乐时内存不足，请关闭其他占用显卡的程序后重新生成。'), + unsupported: t('Music cannot be generated on this computer yet.', '这台电脑暂不能生成配乐。'), + missing: t('Generation stopped. Generate again.', '生成已中断，请重新生成。'), + }[reason] || t('The music could not be generated. Generate again.', '无法生成配乐，请重新生成。'); +} + +export function createEditorMusic({state, store, t, completeSources = async () => {}, onStatus = () => {}, clock = Date.now, random = Math.random}) { + const statuses = new Map(), watching = new Map(); + let info = null, infoAsked = null, chosen = null, chosenFor = null, disposed = false; + const target = (selection = state.selection) => music.musicTarget(state.record, selection); + const refreshInfo = () => infoAsked ??= Promise.resolve().then(() => store.musicInfo()).then(value => { info = value || {available: false}; return info; }) + .catch(() => (info = null)).finally(() => { infoAsked = null; onStatus(); }); + // The style and mood the bar shows: the chosen piece's own, else the edit's last choice; a pick on the bar holds + // until the choice moves elsewhere. + function preset(selection = state.selection) { + const key = JSON.stringify(selection ?? null); + if (chosen && chosenFor === key) return chosen; + const found = target(selection); + return found?.clip?.music && music.MUSIC_STYLES.includes(found.clip.music.style) && music.MUSIC_MOODS.includes(found.clip.music.mood) + ? {...found.clip.music} : music.lastPreset(state.record); + } + function choose(value) { + chosen = {...preset(), ...value}; chosenFor = JSON.stringify(state.selection ?? null); + } + // A piece removed or changed meanwhile is left as it is. + const settle = change => { try { state.settle(change); } catch (value) { if (!(value instanceof EditError)) throw value; } }; + function deliver(id, jobId, job) { + if (disposed || !job) return; + statuses.set(jobId, {...job, seen_at: clock()}); + if (job.state === 'done' && job.result) { + stopWatching(jobId); + settle(record => music.finishMusic(record, id, jobId, {source: job.result.source, frames: job.result.frames})); + completeSources().catch(() => {}); + // The piece just made sets this machine's pace, so the next Generate can show its estimate. + refreshInfo(); + } else if (['failed', 'cancelled'].includes(job.state)) { + stopWatching(jobId); + settle(record => music.failMusic(record, id, jobId, job.state === 'cancelled' ? 'cancelled' : job.reason || 'music_failed')); + } else settle(record => music.musicProgress(record, id, jobId, job.state)); + onStatus(); + } + const starting = new Set(); let pressing = false; + function stopWatching(jobId) { watching.get(jobId)?.(); watching.delete(jobId); } + function watch(id, jobId) { + if (watching.has(jobId)) return; + watching.set(jobId, store.watchMusic(jobId, job => deliver(id, jobId, job), () => deliver(id, jobId, {id: jobId, state: 'failed', reason: 'missing'}))); + } + async function begin(id, words) { + state.select({kind: 'clip', ids: [id]}); + chosen = null; + let job; + try { job = await store.startMusic(music.musicRequest(state.record, id, {seed: Math.floor(random() * 2 ** 32)})); } + catch (value) { + settle(record => music.failMusic(record, id, undefined, value?.reason === 'unsupported' ? 'unsupported' : 'music_failed')); + onStatus(); + return {say: musicFailureText(value?.reason === 'unsupported' ? 'unsupported' : 'music_failed', t)}; + } + if (!job?.id) { settle(record => music.failMusic(record, id, undefined)); onStatus(); return {say: musicFailureText('music_failed', t)}; } + settle(record => music.assignMusicJob(record, id, job.id)); + statuses.set(job.id, {...job, seen_at: clock()}); + watch(id, job.id); onStatus(); + return {text: words}; + } + const api = { + target, preset, choose, refreshInfo, + get info() { return info; }, + styles: () => music.MUSIC_STYLES.map(id => [id, t(...STYLE_NAMES[id])]), + moods: () => music.MUSIC_MOODS.map(id => [id, t(...MOOD_NAMES[id])]), + // The chosen piece is being made. + running(selection = state.selection) { + const found = target(selection); + return !!found?.clip && (starting.has(found.clip.id) || ['queued', 'running'].includes(found.clip.pending)); + }, + // The bar's status for the chosen piece's job; null when it has none running. + barStatus(selection = state.selection) { + const found = target(selection), job = found?.clip?.job && statuses.get(found.clip.job); + return job && ['queued', 'running'].includes(found.clip.pending) ? musicBarStatus(job, t, clock) : null; + }, + // The words the box shows for a piece of music; null for a stretch not made yet. + text(selection = state.selection) { + const found = target(selection); + return found?.clip && !api.running(selection) ? {ids: [found.clip.id], text: found.clip.text} : null; + }, + describe(id, text) { + const found = target({kind: 'clip', ids: [id]}); + if (!found?.clip || api.running({kind: 'clip', ids: [id]}) || found.clip.text === text) return false; + state.apply(t('Edit description', '修改说明'), record => { + const next = JSON.parse(JSON.stringify(record)); + next.tracks.find(track => track.id === music.MUSIC_TRACK).clips.find(clip => clip.id === id).text = text; + return next; + }); + return true; + }, + failure(selection = state.selection) { + const found = target(selection); + return found?.clip?.pending === 'failed' ? {id: found.clip.id, text: musicFailureText(found.clip.reason, t), report: false} : null; + }, + // Seconds the piece will take: this machine's pace from its last piece; unknown before the first. + estimate(selection = state.selection) { + const found = target(selection); + return found && Number.isFinite(info?.pace) ? info.pace * found.frames / 24 : null; + }, + // First-use download for the bar's "Downloads before generating" line: bytes still to fetch, or 0. + async downloadBytes() { + const pack = await Promise.resolve().then(() => store.musicPack()).catch(() => null); + return pack && !pack.ready && !pack.running ? pack.bytes || 0 : 0; + }, + // Generate on a stretch of the music track, or again on a piece there. {say} answers the press. + // A second press while the first is still checking or starting its job is the same press. + async press(text) { + const found = target(); + if (!found) return null; + if (pressing || api.running()) return {say: t('This part is generating.', '这一段正在生成。')}; + pressing = true; + let id = found.id, words; + try { + const current = info || await refreshInfo(); + if (current && current.available === false) return {say: musicFailureText('unsupported', t)}; + const pack = await Promise.resolve().then(() => store.musicPack()).catch(() => null); + if (pack && !pack.ready && pack.disk_ok === false) return {say: musicFailureText('disk_space', t)}; + const pick = preset(); words = text.trim(); + state.apply(t('Generate', '生成'), record => { + const next = music.rememberPreset(record, pick); + if (id) return music.remakeMusic(next, id, {text: words, ...pick}); + const planned = music.planMusic(next, {start: found.start, frames: found.frames, text: words, ...pick}); + id = planned.id; return planned.record; + }, {selectionAfter: null}); + starting.add(id); + } finally { pressing = false; } + try { return await begin(id, words); } finally { starting.delete(id); } + }, + async cancel(selection = state.selection) { + const found = target(selection); + if (!found?.clip?.job || !api.running(selection)) return false; + deliver(found.clip.id, found.clip.job, await store.cancelMusic(found.clip.job).then(value => value?.job || value)); + return true; + }, + // Pieces still waiting for their job when the edit opens again. + recover() { for (const {id, job} of music.musicJobs(state.record)) watch(id, job); }, + reset() { for (const jobId of [...watching.keys()]) stopWatching(jobId); statuses.clear(); chosen = null; }, + dispose() { disposed = true; api.reset(); }, + }; + return api; +} diff --git a/web/editor_music_model.js b/web/editor_music_model.js new file mode 100644 index 00000000..2f020e99 --- /dev/null +++ b/web/editor_music_model.js @@ -0,0 +1,132 @@ +// A piece of music planned on a stretch of the second sound track, made by the music job and laid back in place. +// Pure edits on the project record; the generation bar decides when to call them. +import * as model from './editor_model.js'; +import {FPS} from './editor_model.js'; + +export const MUSIC_TRACK = 'a2'; +// The longest piece the music model makes in one go. +export const MAX_MUSIC_FRAMES = 600 * FPS; +// The longest description the music job takes. +export const MAX_MUSIC_TEXT = 512; +// The presets the server knows; the user's own words go instead when given. +export const MUSIC_STYLES = ['cinematic', 'pop', 'electronic', 'piano', 'folk', 'ambient']; +export const MUSIC_MOODS = ['calm', 'upbeat', 'tense', 'warm', 'sad']; +const fail = code => { throw new model.EditError(code); }; +const end = clip => clip.start + clip.frames; +const lane = record => record.tracks.find(track => track.id === MUSIC_TRACK) || null; + +// The music track's planned or running pieces: no file yet, waiting for (or made by) a music job. +export const isMusicPart = (record, clip) => !!clip && !clip.source && clip.pending !== null + && !!lane(record)?.clips.some(other => other.id === clip.id); +// A finished piece the music job made: its file sits in the music folder. +export const isGeneratedMusic = clip => clip?.source?.kind === 'generated' && /^freevideo-music\/[a-z]+(-[a-z]+)?-[0-9a-f]{32}\.mp4$/.test(clip.source.id); + +export const DEFAULT_PRESET = {style: 'cinematic', mood: 'calm'}; +const preset = value => MUSIC_STYLES.includes(value?.style) && MUSIC_MOODS.includes(value?.mood) ? {style: value.style, mood: value.mood} : null; +// The style and mood last chosen in this edit; the default pair before any. +export const lastPreset = record => preset(record.settings?.music) || {...DEFAULT_PRESET}; +// Remember the pair for the next piece in this edit. +export function rememberPreset(record, value) { + const chosen = preset(value) || fail('preset'); + if (JSON.stringify(record.settings?.music) === JSON.stringify(chosen)) return model.parse(record); + const next = model.serialize(record); + next.settings = {...(next.settings || {}), music: chosen}; + return model.parse(next); +} +// What a selection on the music track asks for: an empty stretch drawn or clicked there, or one piece of music +// there (made again in place). Null for anything else. +export function musicTarget(record, selection) { + const track = lane(record); + if (!track || !selection) return null; + if (selection.kind === 'range' && selection.tracks?.length === 1 && selection.tracks[0] === MUSIC_TRACK + || selection.kind === 'gap' && selection.track === MUSIC_TRACK) { + const frames = Math.min(selection.frames, MAX_MUSIC_FRAMES); + if (track.clips.some(clip => clip.start < selection.start + frames && end(clip) > selection.start)) return null; + return {start: selection.start, frames}; + } + if (selection.kind !== 'clip' || selection.ids.length !== 1) return null; + const clip = track.clips.find(clip => clip.id === selection.ids[0]); + if (!clip || !(isMusicPart(record, clip) || isGeneratedMusic(clip))) return null; + return {start: clip.start, frames: clip.frames, id: clip.id, clip}; +} + +// A drag on the music track: the stretch from start for frames, which must be empty there. The track is added when missing. +export function planMusic(record, {start, frames, text = '', style, mood}) { + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(frames) || frames < 1) fail('range'); + if (frames > MAX_MUSIC_FRAMES) fail('too_long'); + if (typeof text !== 'string') fail('text'); + const chosen = style === undefined && mood === undefined ? {...DEFAULT_PRESET} : preset({style, mood}) || fail('preset'); + const next = model.serialize(lane(record) ? record : model.addAudioTrack(record)), track = next.tracks.find(track => track.id === MUSIC_TRACK); + if (track.locked) fail('locked'); + if (track.clips.some(clip => clip.start < start + frames && end(clip) > start)) fail('overlap'); + const id = `c${++next.serial}`; + track.clips.push({id, start, frames, in: 0, length: frames, source: null, link: null, text, part: null, pending: 'draft', + reason: null, job: null, extend: false, versions: [], volume: 1, music: chosen, duck: true}); + track.clips.sort((a, b) => a.start - b.start); + return {record: model.parse(next), id}; +} +// Made again in place: a finished, failed or stopped piece becomes a planned one over the same stretch, with the +// words and preset given; its volume, fades and lowering stay. One undo brings the previous piece back. +export function remakeMusic(record, id, {text = '', style, mood}) { + if (typeof text !== 'string') fail('text'); + const chosen = preset({style, mood}) || fail('preset'); + const next = model.serialize(record), clip = next.tracks.find(track => track.id === MUSIC_TRACK)?.clips.find(clip => clip.id === id); + if (!clip || !(isGeneratedMusic(clip) || !clip.source && clip.pending !== null)) fail('missing'); + if (next.tracks.find(track => track.id === MUSIC_TRACK).locked) fail('locked'); + if (['queued', 'running'].includes(clip.pending)) fail('busy'); + Object.assign(clip, {source: null, in: 0, length: clip.frames, text, pending: 'draft', reason: null, job: null, versions: [], music: chosen}); + delete clip.splice; + return model.parse(next); +} + +// What the music job is asked for: the piece's length, its words when there are any, and its preset. +export function musicRequest(record, id, {seed} = {}) { + const clip = model.find(record, id); + if (!isMusicPart(record, clip)) fail('missing'); + const body = {frames: clip.frames, ...(clip.music || DEFAULT_PRESET)}; + if (clip.text.trim()) body.text = clip.text.trim(); + if (Number.isSafeInteger(seed)) body.seed = seed; + return body; +} + +function change(record, id, job, edit) { + const next = model.serialize(record), clip = next.tracks.find(track => track.id === MUSIC_TRACK)?.clips.find(clip => clip.id === id); + if (!clip || clip.source || clip.pending === null) return null; + if (job !== undefined && clip.job !== job) return null; + edit(clip); + return model.parse(next); +} +// The job was accepted: the piece waits for it. +export function assignMusicJob(record, id, job) { + if (typeof job !== 'string' || !/^m-[0-9a-f]{32}$/.test(job)) fail('job'); + return change(record, id, undefined, clip => { clip.job = job; clip.pending = 'queued'; clip.reason = null; }) ?? fail('missing'); +} +// What the job says while it runs; a piece already finished, removed or given to another job stays as it is. +export function musicProgress(record, id, job, state) { + if (!['queued', 'running'].includes(state)) return model.parse(record); + return change(record, id, job, clip => { clip.pending = state; }) ?? model.parse(record); +} +// The file is made: it goes into the planned stretch. A shorter file shortens the piece; nothing else moves. +export function finishMusic(record, id, job, {source, frames}) { + const [kind, path] = typeof source === 'string' ? [source.slice(0, source.indexOf(':')), source.slice(source.indexOf(':') + 1)] : []; + if (kind !== 'generated' || !path || !Number.isSafeInteger(frames) || frames < 1) fail('source'); + return change(record, id, job, clip => { + Object.assign(clip, {source: {kind, id: path}, in: 0, length: frames, frames: Math.min(clip.frames, frames), pending: null, reason: null, job: null}); + }) ?? model.parse(record); +} +// Failed or stopped: the piece stays where it was drawn, with its reason, ready to try again. +export function failMusic(record, id, job, reason = 'music_failed') { + const state = reason === 'cancelled' ? 'stopped' : 'failed'; + const code = typeof reason === 'string' && /^[a-z_]{1,80}$/.test(reason) ? reason : 'music_failed'; + return change(record, id, job, clip => { clip.pending = state; clip.reason = state === 'failed' ? code : null; }) ?? model.parse(record); +} +// Try again: back to planned, with the same stretch and description. +export function retryMusic(record, id) { + return change(record, id, undefined, clip => { + if (['queued', 'running'].includes(clip.pending)) fail('busy'); + clip.pending = 'draft'; clip.reason = null; clip.job = null; + }) ?? fail('missing'); +} +// The pieces still waiting for a job after the page opens again, to watch or to mark failed. +export const musicJobs = record => (lane(record)?.clips || []).filter(clip => !clip.source && clip.job && ['queued', 'running'].includes(clip.pending)) + .map(clip => ({id: clip.id, job: clip.job})); diff --git a/web/editor_player.js b/web/editor_player.js index 0524d7ca..3a7cb783 100644 --- a/web/editor_player.js +++ b/web/editor_player.js @@ -27,7 +27,7 @@ export function selectAudioChain(chains, track, clip, position, volumePreview = && !c.clips.some(id => volumePreview.has(id) && volumePreview.get(id) !== 1)); return chain ? {chain, seconds: (position - chain.start) / 24 + (chain.offset || 0)} : null; } -export function createEditorPlayer(host, transport, {state, sources, api, t, error = () => {}, readAudioSamples = null, preservePreview = () => false, prepareAudio = () => true}) { +export function createEditorPlayer(host, transport, {state, sources, api, t, error = () => {}, readAudioSamples = null, preservePreview = () => false, prepareAudio = () => true, speech = null}) { const frame = el('div', 'fv-ed-picture'); frame.setAttribute('aria-label', t('Video preview', '视频预览')); const empty = el('div', 'fv-ed-empty-preview'), pictureIcon = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); pictureIcon.setAttribute('viewBox', '0 0 24 24'); pictureIcon.setAttribute('aria-hidden', 'true'); @@ -54,7 +54,7 @@ export function createEditorPlayer(host, transport, {state, sources, api, t, err const chainSeconds = (chain, position) => Math.max(0, (position - chain.start) / 24 + (chain.offset || 0)); const playback = createPlaybackClock(); const gains = new WeakMap(), volumePreview = new Map(); - const audioPreview = createAudioPreview({state, sources, api, volumePreview, readSamples: readAudioSamples, error}); + const audioPreview = createAudioPreview({state, sources, api, volumePreview, readSamples: readAudioSamples, error, speech}); const clipVolume = clip => volumePreview.get(clip?.id) ?? clip?.volume ?? 1; function volume(media, value) { // HTMLMediaElement stops at 1. A GainNode implements the record's 0–2 range. diff --git a/web/editor_references.js b/web/editor_references.js index 1dfc1d6d..87479359 100644 --- a/web/editor_references.js +++ b/web/editor_references.js @@ -7,13 +7,18 @@ const MAX = 32; // The engine keeps three references of a part that continues: the first three, or, with two or more characters, // the last picture of each (the face) and then the others (continuation.pick_pictures). const CONTINUE_MAX = 3; -const personOf = ({row}) => typeof row.cast?.id === 'string' && row.cast.id ? row.cast.id : null; +const personOf = ({row}) => typeof row.cast?.id === 'string' && row.cast.id && row.cast.setting !== true ? row.cast.id : null; +const placeOf = ({row}) => row.cast?.setting === true; function pickContinued(list) { + // A scene's picture keeps one place after the others (continuation.pick_pictures). + const places = list.filter(placeOf), place = places[places.length - 1]; + if (place) list = list.filter(entry => !placeOf(entry)); + const room = CONTINUE_MAX - (place ? 1 : 0); const last = new Map(); list.forEach((entry, at) => { const person = personOf(entry); if (person) last.set(person, at); }); - if (last.size < 2) return list.slice(0, CONTINUE_MAX); - const faces = [...last.values()].sort((a, b) => a - b).map(at => list[at]); - return [...faces, ...list.filter(entry => !personOf(entry))].slice(0, CONTINUE_MAX); + const chosen = last.size < 2 ? list.slice(0, room) + : [...[...last.values()].sort((a, b) => a - b).map(at => list[at]), ...list.filter(entry => !personOf(entry))].slice(0, room); + return place ? [...chosen, place] : chosen; } export function createEditorReferences({t, source, size = () => null, error = () => {}, notify = () => {}}) { const root = el('div', 'fv-ed-refs'), chips = el('div', 'fv-ed-ref-chips'); diff --git a/web/editor_store.js b/web/editor_store.js index 9590f9bd..87622971 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -174,7 +174,7 @@ export function mediaURL(api, value) { } export function createEditorStore(api, {t = en => en, notify = () => {}, onGenerationReady = () => {}, onGenerationChecking = () => {}, clock = clockDefault, pollMs = 2000, standIn: injected = null} = {}) { - const sources = new Map(), sourceReads = new Map(), watchers = new Set(), pollers = new Set(), once = new Set(); + const sources = new Map(), sourceReads = new Map(), lineage = new Map(), lineageReads = new Map(), watchers = new Set(), pollers = new Set(), once = new Set(); let recovered = null, capabilityPoll = null; // The length a new continuation starts with where the engine names one (a device that continues in short parts). let continueSeconds = null; @@ -381,6 +381,34 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener } await sourceReads.get(signature); return sources; }, + lineage(works) { + const unique = new Map(works.filter(work => work && typeof work.key === 'string' && work.reference && !lineage.has(work.key)) + .map(work => [work.key, work])); + const entries = [...unique.values()]; + if (!entries.length) return Promise.resolve(lineage); + const pending = entries.filter(work => !lineageReads.has(work.key)); + for (let offset = 0; offset < pending.length; offset += 16) { + const batch = pending.slice(offset, offset + 16), keys = batch.map(work => work.key); + const task = (async () => { + try { + const value = await json('/freevideo/chain/lineage', post({works: batch.map(work => work.reference)}), {lineage: []}); + batch.forEach((work, index) => { + const item = value?.lineage?.[index]; + lineage.set(work.key, item && typeof item.known === 'boolean' + ? {parts: item.parts, short: item.short, known: item.known} + : {parts: 1, short: false, known: false}); + }); + } catch { + for (const work of batch) lineage.set(work.key, {parts: 1, short: false, known: false}); + /* Unknown history preserves the editor's existing preflight behavior. */ + } + finally { for (const key of keys) lineageReads.delete(key); } + })(); + for (const key of keys) lineageReads.set(key, task); + } + return Promise.all(entries.map(work => lineageReads.get(work.key)).filter(Boolean)).then(() => lineage); + }, + knownLineage() { return Object.fromEntries(lineage); }, subscribeSources(fn) { watchers.add(fn); return () => watchers.delete(fn); }, async waveform(key, points = 200) { return (await json('/freevideo/editor/waveform?' + new URLSearchParams({key, points}), undefined, {peaks: []})).peaks || []; @@ -514,6 +542,33 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener installEars: () => json('/freevideo/editor/subtitle-pack', post({action: 'install', language: true})), cancelEarsPack: () => json('/freevideo/editor/subtitle-pack', post({action: 'cancel'})), watchEarsPack(fn) { return watch(() => json('/freevideo/editor/subtitle-pack'), fn); }, + // Music for a stretch of the sound track. {available, ready}; null where the server is older. + musicInfo: () => json('/freevideo/editor/music'), + // body: {frames, text?} or {frames, style, mood}, optional seed. Refusals keep the server's reason. + async startMusic(body) { + const response = await request('/freevideo/editor/music', post(body)); + const value = await response.json().catch(() => null); + if (!response.ok) throw Object.assign(errorFor(response), {reason: value?.reason || (response.status === 404 ? 'unsupported' : 'failed')}); + return value?.job || null; + }, + // A job the server no longer knows (it was restarted) reads as failed, so the planned piece does not wait forever. + watchMusic(id, fn, lost) { + return watch(async () => (await json('/freevideo/editor/music?' + new URLSearchParams({id}), undefined, {job: {id, state: 'failed', reason: 'missing'}}))?.job || null, fn, lost); + }, + cancelMusic: id => json('/freevideo/editor/music/cancel', post({id})), + // The music model is downloaded on first use, with the download source chosen in Settings. + musicPack: () => json('/freevideo/editor/music-pack'), + // Speech spans of one source in seconds, for "Lower during speech": {spans}, {state: 'pending'} or {spans: null, reason}. + speech: key => json('/freevideo/editor/speech?' + new URLSearchParams({source: key})), + async installMusic() { + const response = await request('/freevideo/editor/music-pack', post({action: 'install'})); + const value = await response.json().catch(() => null); + // 507: not enough room on the disk; the state that comes back says how much is needed. + if (!response.ok && response.status !== 507) throw Object.assign(errorFor(response), {reason: value?.reason || 'failed'}); + return value; + }, + cancelMusicPack: () => json('/freevideo/editor/music-pack', post({action: 'cancel'})), + watchMusicPack(fn) { return watch(() => json('/freevideo/editor/music-pack'), fn); }, dispose() { disposed = true; activateProject(null).catch(() => {}); session.dispose(); for (const poller of pollers) poller.dispose(); pollers.clear(); watchers.clear(); for (const upload of [...uploads]) { upload.release(); upload.xhr.abort(); upload.reject(new Error('editor_closed')); } }, }; diff --git a/web/editor_tracks.css b/web/editor_tracks.css index 335b7f1c..dd581c82 100644 --- a/web/editor_tracks.css +++ b/web/editor_tracks.css @@ -7,6 +7,10 @@ .fv-ed-track-view:focus-visible{outline:none} .fv-ed-track-view :focus-visible{outline:2px solid var(--fv-accent);outline-offset:2px} .fv-ed-track-heads{position:absolute;inset:0 auto 40px 0;width:var(--fv-ed-head-width);border-right:1px solid var(--fv-border);z-index:2;background:var(--fv-bg)} +/* Clip vertically while allowing narrow track switches to open to the right. */ +.fv-ed-track-view[data-vertical-overflow=true] .fv-ed-track-heads{clip-path:inset(0 -100vw 0 0)} +.fv-ed-track-view[data-vertical-overflow=true] .fv-ed-track-heads::after{content:"";position:absolute;inset:0 0 auto;height:24px;background:var(--fv-bg)} +.fv-ed-track-view[data-vertical-overflow=true] .fv-ed-track-heads>.fv-ed-track-head,.fv-ed-track-view[data-vertical-overflow=true] .fv-ed-track-heads>.fv-ed-add-audio{translate:0 var(--fv-ed-track-offset,0px)} .fv-ed-track-view:where(:has(.fv-ed-track-row[data-track=a2])){height:260px} .fv-editor .fv-ed-track-heads,.fv-editor .fv-ed-track-scroll{bottom:0} .fv-ed-track-view .fv-ed-add-audio{position:absolute;left:0;width:100%;height:28px;padding:0 4px;border:0;border-radius:0;background:transparent;color:var(--fv-muted);font:400 13px/1.4 var(--fv-font,system-ui);text-align:center;white-space:nowrap;overflow:hidden;cursor:pointer} @@ -14,9 +18,10 @@ .fv-ed-track-view .fv-ed-add-audio:disabled{opacity:.4;cursor:default} .fv-ed-track-view .fv-ed-track-head{position:absolute;left:0;width:100%;display:flex;align-items:center;justify-content:center;gap:6px;font-size:13px;color:var(--fv-ink)} .fv-ed-track-view .fv-ed-track-head svg{width:16px;height:16px;flex:none;fill:none;stroke:currentColor;stroke-width:1.4;stroke-linejoin:round;stroke-linecap:round} -.fv-ed-track-scroll{position:absolute;left:var(--fv-ed-head-width);top:0;right:0;bottom:40px;overflow-x:auto;overflow-y:hidden;scrollbar-width:thin;scrollbar-color:var(--fv-border) transparent;overscroll-behavior:contain} +.fv-ed-track-scroll{position:absolute;left:var(--fv-ed-head-width);top:0;right:0;bottom:40px;overflow:auto;scrollbar-width:thin;scrollbar-color:var(--fv-border) transparent;overscroll-behavior:contain} .fv-ed-track-content{position:relative;min-height:134px;touch-action:none;user-select:none} .fv-ed-ruler{position:absolute;left:0;top:0;right:0;height:24px;overflow:hidden;border-bottom:1px solid var(--fv-border);cursor:crosshair;touch-action:none} +.fv-ed-track-view[data-vertical-overflow=true] .fv-ed-ruler{position:sticky;left:auto;right:auto;background:var(--fv-bg);z-index:6} .fv-ed-tick{position:absolute;bottom:0;height:5px;border-left:1px solid var(--fv-border);pointer-events:none} .fv-ed-tick-major{height:8px} .fv-ed-tick-major span{position:absolute;left:4px;bottom:6px;white-space:nowrap;font-size:11px;color:var(--fv-muted);font-variant-numeric:tabular-nums} @@ -70,7 +75,7 @@ .fv-ed-track-zoom button:hover{background:var(--fv-card);color:var(--fv-ink)} .fv-ed-zoom-slider{width:64px;height:24px;margin:0;accent-color:var(--fv-accent);cursor:pointer} .fv-ed-track-zoom .fv-ed-fit{padding:0 7px;white-space:nowrap;font-size:12px;margin-left:4px} -.fv-ed-drag-label{position:absolute;z-index:9;padding:4px 7px;background:var(--fv-card);color:var(--fv-ink);border:1px solid var(--fv-border);border-radius:6px;font-size:12px;white-space:pre;line-height:1.5;pointer-events:none} +.fv-ed-drag-label,.fv-ed-duration-hint{position:absolute;z-index:9;padding:4px 7px;background:var(--fv-card);color:var(--fv-ink);border:1px solid var(--fv-border);border-radius:6px;font-size:12px;white-space:pre;line-height:1.5;pointer-events:none} .fv-ed-clip-popover{position:absolute;z-index:10;width:280px;max-width:calc(100% - 16px);padding:10px;border:1px solid var(--fv-border);border-radius:8px;background:var(--fv-card)} .fv-ed-clip-popover p{margin:0 0 8px;font-size:13px;overflow-wrap:anywhere} .fv-ed-clip-popover button{min-height:32px;padding:4px 10px;border:1px solid var(--fv-border);border-radius:8px;background:var(--fv-raised);cursor:pointer} diff --git a/web/editor_tracks.js b/web/editor_tracks.js index 703bb896..66f94e73 100644 --- a/web/editor_tracks.js +++ b/web/editor_tracks.js @@ -13,6 +13,9 @@ import {attachAudioTracks} from './editor_audio.js'; import {seconds as secondsText} from './editor_audio_notes.js'; import {renderSpeedBadge} from './editor_speed.js'; import {speedEdited} from './editor_speed_model.js'; +import {MUSIC_TRACK, MAX_MUSIC_FRAMES} from './editor_music_model.js'; +// A range drawn on the music track alone: music for that stretch. +const musicRange = selection => selection?.kind === 'range' && selection.tracks?.length === 1 && selection.tracks[0] === MUSIC_TRACK; export const SOURCE_TYPE = 'application/x-freevideo-source'; export const SNAP_DISTANCE = 8; @@ -84,10 +87,10 @@ export function edgeParameters(clip, edge, frame, points, zoom, {canExtend = tru const sourceEnd = clip.start + clip.length - clip.in; if (speedEdited(clip)) return {kind: 'trim', edge, frame: snapped.frame, guide: snapped.guide}; if (edge === 'end' && canExtend && clip.extend && snapped.frame > sourceEnd) { - // What continues a video is at least followSeconds long; the preview already shows the raised length. + // What is added at the tail is at least lastSeconds long; the preview already shows the raised length. const wanted = Math.round((snapped.frame - sourceEnd) / model.FPS), seconds = model.continuedSeconds(wanted, maxFollowSeconds); return {kind: 'extend', frame: sourceEnd, frames: seconds * model.FPS, guide: snapped.guide, raised: seconds !== wanted, - least: wanted < model.RULES.followSeconds && seconds >= model.RULES.followSeconds}; + least: wanted < model.RULES.lastSeconds && seconds >= model.RULES.lastSeconds}; } return {kind: 'trim', edge, frame: snapped.frame, guide: snapped.guide}; } @@ -280,11 +283,13 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = } // A range cannot be drawn longer than one video may be; `fromEnd` keeps its end where it is. function capRange(selection, fromEnd = false, continues = false) { - const most = model.maxRangeFrames(state.record, {...actions.generationLimits?.(), continues}); + const most = musicRange(selection) ? MAX_MUSIC_FRAMES : model.maxRangeFrames(state.record, {...actions.generationLimits?.(), continues}); if (selection.frames <= most) return selection; return fromEnd ? {...selection, start: selection.start + selection.frames - most, frames: most} : {...selection, frames: most}; } function rangeParts(selection, record = state.record) { + // A stretch of the music track is one piece, never split into video parts. + if (musicRange(selection)) return [selection.frames]; return model.rangeParts(record, {...selection, ...actions.generationLimits?.()}); } function boundaryLine(frame, index) { @@ -401,6 +406,20 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = } ruler.replaceChildren(...marks); } + function syncVerticalScroll() { + const top = scroll.scrollTop; + element.dataset.verticalOverflow = String(scroll.scrollHeight > scroll.clientHeight); + element.style.setProperty('--fv-ed-track-offset', `${-top}px`); + // Keep the handle beside the fixed ruler; the line still ends at the last row. + playhead.style.top = `${18 + top}px`; + playhead.style.height = `${Math.max(0, parseFloat(content.style.height) - 34 - 18 - top)}px`; + } + function setTrackHeight(height) { + // The heads' 6px gap and 28px audio button belong to the same scroll extent. + content.style.height = `${height + 34}px`; + overlays.style.height = `${height}px`; + syncVerticalScroll(); + } function layout() { // Defer the initial 90-second view until a detached shell is mounted. if (!mountedStates.has(state) && scroll.clientWidth) { @@ -416,7 +435,7 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = item.element.classList.toggle('fv-ed-tiny', pixels(item.clip.frames) < 2); item.status.hidden = pixels(item.clip.frames) < 24; } - renderRuler(); renderSelection(); renderPlayhead(); slider.value = String(state.zoom); + syncVerticalScroll(); renderRuler(); renderSelection(); renderPlayhead(); slider.value = String(state.zoom); } function waveform(item, key) { const viewKey = `${key}/${item.clip.in}/${item.clip.frames}/${item.clip.length}`; @@ -566,7 +585,7 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = addAudio.dataset.sign = addAudio.textContent.slice(0, 1); // Phone widths show the sign alone. addAudio.title = ''; addAudio.setAttribute('aria-label', addAudio.textContent); - content.style.height = `${top - 6}px`; playhead.style.height = `${top - 24}px`; + setTrackHeight(top - 6); layout(); const failure = openFailure && clips.get(openFailure); if (failure && model.clipState(failure.clip) === 'failed' && state.selection?.ids?.includes(openFailure)) failedPopover(failure); @@ -792,7 +811,8 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = const aligned = snap(frame, points, state.zoom); // Where it was pressed, drawn to a clip edge, marker or the playhead nearby. gesture.from ??= snap(gesture.anchor, points, state.zoom).frame; - const drawn = rangeParameters(gesture.from, aligned.frame); + // Drawn on the music track, the range is that track's alone: music for that stretch. + const drawn = rangeParameters(gesture.from, aligned.frame, gesture.track === MUSIC_TRACK ? [MUSIC_TRACK] : undefined); // Drawn from a clip's end, the range starts exactly there. if (gesture.flush != null && aligned.frame > gesture.flush) Object.assign(drawn, {start: gesture.flush, frames: Math.max(2 * model.FPS, Math.round((aligned.frame - gesture.flush) / model.FPS) * model.FPS)}); @@ -962,8 +982,8 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = const frames = result.frame - finished.clip.start - finished.clip.frames + result.frames; if (extendClip(finished.id, frames) && result.raised) { // Say the new length (tenths, rounded down like other durations) so the change is visible. - const least = model.RULES.followSeconds, length = secondsText(result.frames, model.FPS); - actions.notify?.(result.least ? t(`Lengthened to ${length} s. Each continued part must be at least ${least} seconds.`, `已加长到 ${length} 秒。接着生成的每一段至少 ${least} 秒。`) + const least = model.RULES.lastSeconds, length = secondsText(result.frames, model.FPS); + actions.notify?.(result.least ? t(`Lengthened to ${length} s. The last part must be at least ${least} seconds.`, `已加长到 ${length} 秒。最后一段至少 ${least} 秒。`) : t(`Lengthened to ${length} s.`, `已加长到 ${length} 秒。`)); } } else attempt(t('Trim', '裁短'), record => model.trim(record, finished.id, finished.edge, result.frame)); @@ -1055,13 +1075,13 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = const name = t('Audio 2', '声音 2'); head.title = name; head.append(icon('audio'), create('span', '', name)); row.setAttribute('aria-label', name); row.style.top = head.style.top = `${top}px`; row.style.height = head.style.height = '40px'; - const oldName = first.head.title, oldTop = addAudio.style.top, oldHeight = content.style.height, oldPlayhead = playhead.style.height; + const oldName = first.head.title, oldTop = addAudio.style.top, oldHeight = content.style.height; sourcePreview.restore = () => { first.head.title = oldName; first.head.querySelector('span').textContent = oldName; - addAudio.style.top = oldTop; content.style.height = oldHeight; playhead.style.height = oldPlayhead; + addAudio.style.top = oldTop; setTrackHeight(parseFloat(oldHeight) - 34); }; first.head.title = t('Audio 1', '声音 1'); first.head.querySelector('span').textContent = first.head.title; - addAudio.style.top = `${top + 46}px`; content.style.height = `${top + 40}px`; playhead.style.height = `${top + 22}px`; + addAudio.style.top = `${top + 46}px`; setTrackHeight(top + 40); heads.insertBefore(head, addAudio); content.insertBefore(row, overlays); nodes.push(head, row); } for (const {track, clip} of added) { @@ -1192,7 +1212,15 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = }); on(scroll, 'scroll', renderRuler, {passive: true}); on(scroll, 'scroll', renderEmptyHints, {passive: true}); + on(scroll, 'scroll', syncVerticalScroll, {passive: true}); on(scroll, 'wheel', wheel, {passive: false}); + on(heads, 'wheel', event => { + if (event.ctrlKey || event.metaKey) { wheel(event); return; } + event.preventDefault(); event.stopPropagation(); + const unit = event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? scroll.clientHeight : 1; + if (event.shiftKey) scroll.scrollLeft += (event.deltaX || event.deltaY) * unit; + else { scroll.scrollLeft += event.deltaX * unit; scroll.scrollTop += event.deltaY * unit; } + }, {passive: false}); on(scroll, 'dragover', dropOver); on(scroll, 'drop', dropSource); on(document, 'dragstart', event => { clearSourcePreview(); diff --git a/web/freevideo.js b/web/freevideo.js index 4b696b98..dd2e1db0 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -161,7 +161,8 @@ function mediaPanel(node, mount = null) { role.value = 'reference'; toolbar.append(add, generateImage, addAudio, role, picker, audioPicker); panel.append(toolbar, mode, note, audioHelp, connections, cards); const cast = createCastLibrary({api, text, viewURL, uploadOne}); - panel.prepend(cast.element); + const places = createCastLibrary({api, text, viewURL, uploadOne, kind: 'setting'}); + panel.prepend(cast.element, places.element); const read = () => { const rows = JSON.parse(serialized.value || "[]"); if (!Array.isArray(rows) || rows.length > 32) throw new Error(text("Use at most 32 media items", "最多使用 32 个素材")); @@ -419,7 +420,7 @@ function mediaPanel(node, mount = null) { return JSON.stringify(requestRows(swapState(node))); } // The run takes only what it uses (and which character a picture belongs to); the editing data of a picture stays in the workflow. - try { return JSON.stringify(read().map(({file, role, enabled, cast, sound, depth, use, range}) => ({file, role, enabled, ...(use === AUDIO_USE.original ? {use, ...(range && typeof range === 'object' ? {range} : {})} : {}), ...(sound === false && mediaKindOrNull(file) === 'video' ? {sound: false} : {}), ...(depth === true && role === 'reference' && mediaKindOrNull(file) === 'video' ? {depth: true} : {}), ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id}} : {})}))); } catch { return serialized.value; } + try { return JSON.stringify(read().map(({file, role, enabled, cast, sound, depth, use, range}) => ({file, role, enabled, ...(use === AUDIO_USE.original ? {use, ...(range && typeof range === 'object' ? {range} : {})} : {}), ...(sound === false && mediaKindOrNull(file) === 'video' ? {sound: false} : {}), ...(depth === true && role === 'reference' && mediaKindOrNull(file) === 'video' ? {depth: true} : {}), ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id, ...(cast.setting === true ? {setting: true} : {})}} : {})}))); } catch { return serialized.value; } }; async function upload(files) { if (node.isUploading || !files.length) return; @@ -446,7 +447,7 @@ function mediaPanel(node, mount = null) { panel.ondragover = event => { event.preventDefault(); event.stopPropagation(); }; panel.ondrop = event => { event.preventDefault(); event.stopPropagation(); upload(event.dataTransfer.files); }; panel.onpaste = event => { if (event.clipboardData?.files.length) { event.preventDefault(); event.stopPropagation(); upload(event.clipboardData.files); } }; - const dispose = () => { window.removeEventListener('freevideo-media', changed); cast.destroy(); }; + const dispose = () => { window.removeEventListener('freevideo-media', changed); cast.destroy(); places.destroy(); }; if (mount) { mount.append(panel); render(); return dispose; } node.freevideoCreateMediaEditor = target => mediaPanel(node, target); node.addDOMWidget("freevideo_media", "freevideo_media", panel, {serialize: false, getMinHeight: () => 300, getMaxHeight: () => 520}); diff --git a/web/howto.js b/web/howto.js index cafdb7ee..0850b573 100644 --- a/web/howto.js +++ b/web/howto.js @@ -35,8 +35,8 @@ const HOWTOS = [ zh: '填写更长的时长，会自动分段生成。'}, {id: 'generate', anchor: '[data-fv-guide="generate"]', en: 'Choose "Generate video".', zh: '点“生成视频”。'}, {id: 'extend', anchor: '.fv-action-extend', image: 'extend-button', - en: 'Choose "Extend" below a finished video to continue it. Each continued part is at least 5 seconds; with a part shorter than 10 seconds, one video has at most 4 parts.', - zh: '在生成好的视频下方点“接着生成”，继续下一段。接着生成的每一段至少 5 秒；有短于 10 秒的段时，一条视频最多 4 段。'}, + en: 'Choose "Extend" below a finished video to continue it. Each continued part is at least 5 seconds, and the last one can be as short as 2 seconds. With a part shorter than 10 seconds, one video has at most 4 parts.', + zh: '在生成好的视频下方点“接着生成”，继续下一段。接着生成的每一段至少 5 秒，最后一段可短至 2 秒。有短于 10 秒的段时，一条视频最多 4 段。'}, {id: 'longer', anchor: '[data-view="editor"]', en: 'Parts can also be added one by one in "Edit".', zh: '也可在“剪辑视图”中逐段接续。'}]}, {id: 'editor-extend-refs', view: 'editor', title: ['Add reference images when extending in Edit', '在剪辑视图延长时加参考图'], steps: [ diff --git a/web/image_generation.js b/web/image_generation.js index fbb81bdb..23581160 100644 --- a/web/image_generation.js +++ b/web/image_generation.js @@ -110,7 +110,7 @@ export function createImageGenClient(fetchImpl = globalThis.fetch.bind(globalThi async historyDeleteMany(files) { const answer = await post('history-delete-many', {files}); return {items: answer.items || [], failed: answer.failed || []}; }, historyOpen: () => post('history-open', {}), // The prompts and sizes of a new character's five pictures (the Agent's cards use the same ones). - castViews: async description => (await post('cast-views', {description})).views, + castViews: async (description, kind = 'character') => (await post('cast-views', {description, kind})).views, async upload(file) { // ComfyUI's existing image upload stores this at input/. // FreeVideo's media upload uses subdirectories the image worker rejects. diff --git a/web/prompt_references.css b/web/prompt_references.css index b92ca2cf..a4285def 100644 --- a/web/prompt_references.css +++ b/web/prompt_references.css @@ -22,6 +22,7 @@ .fv-reference-group{padding:6px 8px 3px;font-size:11px;color:var(--fv-muted,#a9b3c4)} .fv-reference-person{min-height:36px;padding:5px 8px;gap:10px} .fv-reference-face{width:24px;height:24px;flex:none;border-radius:50%;object-fit:cover;object-position:50% 15%;object-view-box:inset(0 20% 40% 20%);background:var(--fv-raised,#252e39)} +.fv-reference-face[data-place]{width:40px;height:24px;border-radius:4px;object-position:50% 50%;object-view-box:none} .fv-reference-person strong{font-size:13px;font-weight:550;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} .fv-reference-person[aria-disabled=true]{opacity:.5;cursor:not-allowed} .fv-mention-mirror{position:absolute;box-sizing:border-box;overflow:hidden;pointer-events:none;white-space:pre-wrap;overflow-wrap:break-word;word-break:normal;color:var(--fv-ink,#eee);z-index:1} diff --git a/web/prompt_references.js b/web/prompt_references.js index 83919bd0..fbddd0a0 100644 --- a/web/prompt_references.js +++ b/web/prompt_references.js @@ -214,7 +214,9 @@ export function attachReferencePicker(input, {items, t = en => en, view, host, a let available = [], invalid = false; try { available = items(); } catch { invalid = true; } const media = available.filter(item => `${item.name} ${item.token} ${t(item.kind, {image:'图片',video:'视频',audio:'音频'}[item.kind])} ${item.number}`.toLowerCase().includes(range.query)); - const people = everyone.filter(person => person.name.toLowerCase().includes(range.query)); + const found = everyone.filter(person => person.name.toLowerCase().includes(range.query)); + const people = [...found.filter(person => !person.setting), ...found.filter(person => person.setting)]; + const plain = people.filter(person => !person.setting).length; matches = [...people.map(character => ({character})), ...media]; list.replaceChildren(); if (!matches.length) { @@ -229,17 +231,18 @@ export function attachReferencePicker(input, {items, t = en => en, view, host, a } } const heading = (en, zh) => { const h = document.createElement('div'); h.className = 'fv-reference-group'; h.setAttribute('role', 'presentation'); h.textContent = t(en, zh); list.append(h); }; - if (people.length) heading('Characters', '角色'); + if (plain) heading('Characters', '角色'); for (const [index, item] of matches.entries()) { + if (index === plain && index < people.length) heading('Scenes', '场景'); if (index === people.length && people.length && media.length) heading('Media', '素材'); const option = document.createElement('div'); option.id = `${list.id}-${index}`; option.className = 'fv-reference-option'; option.setAttribute('role', 'option'); if (item.character) { const person = item.character, face = document.createElement('img'); - face.className = 'fv-reference-face'; face.alt = ''; face.src = person.thumb || ''; + face.className = 'fv-reference-face'; face.alt = ''; if (person.setting) face.dataset.place = ''; face.src = person.thumb || ''; const name = document.createElement('strong'); name.textContent = person.name; option.className += ' fv-reference-person'; option.append(face, name); - if (person.blocked) { option.setAttribute('aria-disabled', 'true'); option.title = person.blocked; } + if (person.blocked) { option.setAttribute('aria-disabled', 'true'); if (typeof person.blocked === 'string') option.title = person.blocked; } option.onpointerdown = e => e.preventDefault(); option.onclick = () => choose(index); option.onpointermove = () => { if (usable(index)) select(index); }; list.append(option); continue; diff --git a/web/studio.js b/web/studio.js index e30040a1..0e0c1363 100644 --- a/web/studio.js +++ b/web/studio.js @@ -698,7 +698,7 @@ export function openStudio(node, {extendFrom = null} = {}) { prompt.disabled = linked(node, 'text'); prompt.oninput = () => { set(node, 'text', prompt.value); placeholder(); }; const promptEditor = el('div', null, 'fv-prompt-editor'); promptEditor.append(prompt); // Library characters offered in the @ list; a mention puts the character's pictures in the media list. - let castLibrary = null; + let castLibrary = null, sceneLibrary = null; const castAssets = () => { const owner = upstream(node, 'media'); const field = owner && widget(owner, 'assets'); return field ? {owner, field} : null; }; // The library's own upload of a picture goes to the same streaming upload the media panel uses. const castUpload = file => uploadMedia(api, file); @@ -749,20 +749,23 @@ export function openStudio(node, {extendFrom = null} = {}) { function castChoices() { if (!castLibrary || linked(node, 'text') || !castAssets()) return []; const rows = mediaRows(node); - return castLibrary.list().filter(card => card.name).map(card => { + const places = (sceneLibrary?.list() || []).map(card => ({...card, kind: 'setting'})); + return [...castLibrary.list(), ...places].filter(card => card.name).map(card => { const state = mentionState(rows, card, Math.min(MAX_PICTURES_EACH, Number(card.images) || 0), {continued: Boolean(extension.source)}); return {id: card.id, name: card.name, handle: mentionHandle(rows, card, castPending), thumb: card.thumbnail?.file ? view(card.thumbnail.file) : card.thumbnail?.url || '', - blocked: state.ok ? '' : castBlocked(state.reason)}; + ...(card.kind === 'setting' ? {setting: true} : {}), blocked: state.ok ? '' : castBlocked(state.reason) || true}; }); } // The name goes into the text at once; the card's pictures follow when the full card has been read. function mentionCharacter(person) { if (!castLibrary) return false; if (castInRows(mediaRows(node)).some(entry => entry.id === person.id)) return true; - if (!mentionState(mediaRows(node), {id: person.id}, 1, {continued: Boolean(extension.source)}).ok) return false; + const library = person.setting ? sceneLibrary : castLibrary; + if (!library) return false; + if (!mentionState(mediaRows(node), {id: person.id, ...(person.setting ? {kind: 'setting'} : {})}, 1, {continued: Boolean(extension.source)}).ok) return false; const typed = person.handle || person.name; castPending.set(person.id, typed); - castLibrary.get(person.id).then(card => { + library.get(person.id).then(card => { if (disposed) return; const rows = mediaRows(node), target = castAssets(); if (!target || !mentionState(rows, card, undefined, {continued: Boolean(extension.source)}).ok) return; @@ -816,9 +819,11 @@ export function openStudio(node, {extendFrom = null} = {}) { const castChanged = event => { if (String(event.detail) === String(mediaNode?.id)) { castOverlay.refresh(); references.refresh(); announceCast(); } }; window.addEventListener('freevideo-media', castChanged); node.freevideoContinues = () => Boolean(extension.source); - cleanup.push(() => { castLibrary?.destroy?.(); castLibrary = null; window.removeEventListener('freevideo-media', castChanged); prompt.removeEventListener('input', announceCast); castOverlay.dispose(); delete node.freevideoContinues; }); + cleanup.push(() => { castLibrary?.destroy?.(); castLibrary = null; sceneLibrary?.destroy?.(); sceneLibrary = null; window.removeEventListener('freevideo-media', castChanged); prompt.removeEventListener('input', announceCast); castOverlay.dispose(); delete node.freevideoContinues; }); openCastLibrary({api, text: t, viewURL: view, uploadOne: castUpload, onChange: () => { castOverlay.refresh(); references.refresh(); }}) .then(library => { if (disposed) library?.destroy?.(); else castLibrary = library; }); + openCastLibrary({api, text: t, viewURL: view, uploadOne: castUpload, kind: 'setting', onChange: () => { references.refresh(); }}) + .then(library => { if (disposed) library?.destroy?.(); else sceneLibrary = library; }); let requestAssets = null, rewriteSubjects = []; const enhancer = createPromptEnhancer({node, input: prompt, editor: promptEditor, api, t, starter, issueButton, // One rewrite runs at a time, so the subjects of the text it was given are the ones its numbers refer to. diff --git a/web/tutorial_book_content.js b/web/tutorial_book_content.js index c4090870..36c7261e 100644 --- a/web/tutorial_book_content.js +++ b/web/tutorial_book_content.js @@ -199,8 +199,8 @@ export const TUTORIAL_CHAPTERS = [ settings: {}, }, limits: { - zh: ['接着生成的每一段至少 5 秒。一个连续镜头最多 8 段，有短于 10 秒的段时最多 4 段；创作面板中时长最长 60 秒；分镜剧本可在剪辑视图中逐个镜头添加，或用 AI 全自动。', '分段生成和接着生成需要 NVIDIA 显卡或 Mac。', '段数越多，衔接处越容易出现变化；适合切换镜头处，可直接开始新的一段。'], - en: ['Each continued part is at least 5 seconds. One continuous shot has at most 8 parts, or 4 when a part is shorter than 10 seconds, and the creation panel accepts up to 60 seconds. For a storyboard, add shots one by one in Edit, or use Autopilot.', 'Generating in parts and Extend need an NVIDIA graphics card or a Mac.', 'The more parts are joined, the more the joins may change. Where a cut fits, cut to a new shot and start a fresh part.'], + zh: ['接着生成的每一段至少 5 秒，最后一段可短至 2 秒。一个连续镜头最多 8 段，有短于 10 秒的段时最多 4 段；创作面板中时长最长 60 秒；分镜剧本可在剪辑视图中逐个镜头添加，或用 AI 全自动。', '分段生成和接着生成需要 NVIDIA 显卡或 Mac。', '段数越多，衔接处越容易出现变化；适合切换镜头处，可直接开始新的一段。'], + en: ['Each continued part is at least 5 seconds, and the last one can be as short as 2 seconds. One continuous shot has at most 8 parts, or 4 when a part is shorter than 10 seconds, and the creation panel accepts up to 60 seconds. For a storyboard, add shots one by one in Edit, or use Autopilot.', 'Generating in parts and Extend need an NVIDIA graphics card or a Mac.', 'The more parts are joined, the more the joins may change. Where a cut fits, cut to a new shot and start a fresh part.'], }, }, { diff --git a/web/view_navigation.js b/web/view_navigation.js index 6e7b66bc..1d3f84f0 100644 --- a/web/view_navigation.js +++ b/web/view_navigation.js @@ -26,6 +26,8 @@ export function preferredView() { let collapsing = false; export function viewChanged(view, node) { + // A local count for the diagnostic report; the page never waits for it. + if (view === 'nodes' && currentView !== 'nodes') try { api.fetchApi('/freevideo/node-view-opened', {method: 'POST'}).catch(() => {}); } catch { /* Counting never blocks the view. */ } currentView = view; if (node) selected = node; // Making way for the user's own workflow is not a choice of view: the FreeVideo workflow reopens as it was left.