FreeVideo 0.4.28

MANIFEST.in

@@ -65,4 +65,8 @@ include web/assets/tour/c8-s1.mp4
 include web/assets/tour/c8-s1.webp
 include web/assets/tour/c9-s1.mp4
 include web/assets/tour/c9-s1.webp
+include web/assets/tour/c10-person.webp
+include web/assets/tour/c10-s1.mp4
+include web/assets/tour/c10-s1.webp
+include web/assets/tour/c10-source.webp
 include web/assets/tour/fox-sheet.webp

__init__.py

@@ -61,4 +61,6 @@ async def comfy_entrypoint():
         logging.warning('[FreeVideo] Agent routes not registered: %s', error)
     from .freevideo_engine.comfy_nodes import FreeVideoExtension, register_prompt_cleanup
     register_prompt_cleanup()
+    from .freevideo_engine.comfy_nodes import schedule_startup_prefetch
+    schedule_startup_prefetch()
     return FreeVideoExtension()

docs/zh-CN/使用说明.md

@@ -1,18 +1,28 @@
 # 使用说明
 
-这一页说明创作面板里三件常被问到的事：用参考图生成视频、首帧尾帧和参考图的区别、做更长的视频。
+这一页说明创作面板里常用的操作：用参考图生成视频、替换人物、区分首帧尾帧和参考图、筛选作品，以及做更长的视频。
 
 ## 参考图生成视频
 
 1. 打开创作面板，展开“素材”。
 2. 点“上传文件”，选择图片、视频或音频。上传的每张图片可以指定用途：首帧、尾帧或参考。要让画面里出现这张图里的人物、物品或场景，选“参考”。
 3. 在描述中输入 `@`，从列表里选一张素材，描述里会出现 `<Picture 1>` 这样的标记。视频和音频素材对应 `<Video 1>`、`<Audio 1>`，各类素材分别编号。
-4. 在描述中写明这张图在画面中的作用，例如“`<Picture 1>` 里的女孩在海边奔跑”，然后点“生成视频”。
+4. 音频卡片上选“原样使用”时，成片直接使用这段声音，时长随音频而定；一次只能有一段音频原样使用。
+5. 在描述中写明这张图在画面中的作用，例如“`<Picture 1>` 里的女孩在海边奔跑”，然后点“生成视频”。
 
 素材可以暂时关闭而不删除：取消勾选的素材不参与这次生成，其余素材的编号会随之重排，描述里的标记会同步更新。
 
 参考图生成视频目前仍是实验能力。能够运行不等于每次都能保持人物外观一致，清晰的正面图效果更稳定。
 
+## 替换人物
+
+1. 展开创作面板的“素材”，打开“替换人物”。
+2. 在“源视频”中点“添加视频”，在“人物图”中点“添加图片”。
+3. 需要源视频的声音时，勾选“保留源视频声音”。
+4. 描述可以留空，也可以补充画面要求；然后点“生成视频”。
+
+替换人物不能与首帧、尾帧同时使用。首次使用时会自动下载替换人物模型（约 80 MB），面板中显示进度，下载完成后才能生成。源视频至少 1 秒；源视频没有声音时，不能保留声音。
+
 ## 首帧、尾帧和参考的区别
 
 | 用途 | 作用 | 适合 |
@@ -25,13 +35,21 @@
 - 首帧、尾帧暂不能与参考图、参考视频同时使用；同时给了两类，先保留其中一类。
 - 参考图保持原有比例，按输出像素面积缩小。参考视频只取与成片等长的一段，最长 15 秒，生成用时约为只用图片时的两倍。
 
+## 作品库和收藏
+
+在作品详情中点爱心图标收藏；列表上方选“收藏”只看收藏的作品，选“全部”看所有作品。
+
 ## 在描述中用 @ 引用素材
 
 - 输入 `@` 后会列出当前可用的素材，选中后插入对应的标记。
 - 也可以直接写标记，如 `<Picture 2>`、`<Video 1>`、`<Audio 1>`。
 - 编号顺序是：先是上传的素材，再是从上到下连接的图片，最后是音频。参考视频的声音也占一个音频编号。
 - 删除或关闭某张素材后，描述里指向它的标记会改成“[素材已移除]”，其余标记按新编号更新。
 
+## 出错了怎么办
+
+全部下载源都失败时，错误卡片会逐个列出下载源和失败原因。
+
 ## 用一句话做白模
 
 未连接外部 Agent 时，可开启聊天框中的白模按钮，用一句话描述镜头。有合适的现成场景时，会先生成白模供您确认。
@@ -42,7 +60,7 @@
 
 1. **直接把时长填长。** 超过一次能生成的长度时，FreeVideo 自动分成几段依次生成，再接成一条视频。
 2. **接着生成。** 在生成好的视频结果卡上点“接着生成”，继续下一段；也可在“剪辑视图”中逐段修改。适合边看边调。
-3. **用 AI 全自动。** 打开“AI 全自动”，把想法交给 Agent，它会按镜头拆分、逐镜生成并拼接。耗时随镜头数和每镜时长而定，每镜按普通生成计时。
+3. **用 AI 全自动。** 打开“AI 全自动”，把想法交给 Agent，它会按镜头拆分、逐镜生成并拼接。耗时随镜头数和每镜时长而定，每镜按普通生成计时。在对话列表中点对话旁的“更多”，选“重命名”可修改对话名称；Agent 开拍整部片前、重拍耗时较长时是否先询问，可在“设置”的“AI Agent”中选择。
 
 ### 在剪辑视图里加参考图
 
@@ -51,6 +69,7 @@
 ### 限制
 
 - 两种做法一样，一条视频最多 8 段（第一段加 7 次接着生成），有短于 10 秒的段时最多 4 段；创作面板里时长最多填 60 秒。接续的每一段至少 5 秒，每段都是 10 秒以上时整条约 110 秒。剪辑视图里逐段生成同样适用。段数越多，衔接处越容易出现变化，适合切换镜头处，可直接切镜头，用对应的参考图开始新的一段。
+- 每张显卡同一时间处理一个请求，其余请求在队列中等待。
 - 接着生成使用轻量档，需要 NVIDIA 显卡或 Mac，也不能和“继续二采”同时使用。
 - 单次生成的最长时长随画幅和显卡而不同，在创作面板的“可生成范围”中可查看各尺寸的最长时长和预计用时。
 - 分段生成同样需要 NVIDIA 显卡或 Mac。

freevideo_engine/adaptive.py

@@ -41,6 +41,37 @@ def commit_exhausted(error):
     return found(error)
 
 
+def commit_limited_allocation(error, failure):
+    """A driver-level CUDA allocation failure on Windows with VRAM to spare.
+
+    WDDM charges device allocations to the system commit limit. When that limit
+    is spent the driver refuses to map more device memory although the card has
+    room: the error is the bare "CUDA error: out of memory" of the runtime, not
+    the caching allocator's "CUDA out of memory. Tried to allocate", and the
+    device is not near its budget. A worker that keeps serving afterwards
+    still holds the commit it charged (about 6 GiB private commit after a
+    cleaned-up failure, against 0.03 GiB in a new process) and its device
+    mappings are inconsistent: the retry in it failed with "resource already
+    mapped" and the worker could not be stopped. WDDM non-local exhaustion is a
+    separate case with its own recovery (``nonlocal_exhausted``).
+    """
+    from .system import windows
+    if not windows() or failure.get('kind') != 'gpu_oom' or nonlocal_exhausted(failure):
+        return False
+    gpu = failure.get('gpu') if isinstance(failure.get('gpu'), dict) else {}
+    free = gpu.get('free_bytes')
+    if type(free) is not int or free < 2 * 2**30:
+        return False
+    seen = set()
+    while error is not None and id(error) not in seen:
+        seen.add(id(error))
+        text = str(error).lower()
+        if 'cuda error: out of memory' in text and 'cuda out of memory' not in text:
+            return True
+        error = error.__cause__ or error.__context__
+    return False
+
+
 # A CUDA error that may leave the device itself unusable; never retried.
 DEVICE_STATE_ERRORS = ('cuda error: unknown error', 'device is lost', 'device lost', 'gpu is lost',
     'illegal memory access', 'device-side assert', 'unspecified launch failure', 'driver shutting down',

freevideo_engine/agent/cast.py

@@ -679,7 +679,11 @@ def generate(self, identity, value=None, *, by='builtin', ref=None, revision=Non
             if value['face'] and card['kind'] != 'character':
                 raise spec.SpecError([schema.diagnostic('/face', 'false for a setting card', True)])
             ready = value.get('source') != 'video' and getattr(self.jobs.port, 'image_ready', lambda: False)()
-            source = value.get('source', 'image' if ready else 'video')
+            from ..image_gen.loras import validate_selection
+            loras = validate_selection(value.get('loras', []))
+            source = value.get('source', 'image' if ready or loras else 'video')
+            if loras and source != 'image':
+                spec.fail('/loras', 'LoRAs for reference pictures require source=image')
             # An explicit image request goes through the image service even when
             # its pack is absent: that failure owns a diagnostic report and a
             # durable job/card, just like a picture requested directly.
@@ -689,7 +693,7 @@ def generate(self, identity, value=None, *, by='builtin', ref=None, revision=Non
                 from ..image_gen.cast_views import VIEWS, SETTING_VIEWS
                 # A picture takes seconds, so a character always gets every view, the face close-up included.
                 views = list(SETTING_VIEWS if card['kind'] == 'setting' else VIEWS)
-                items = self._image_items(card, views[:1], seed)
+                items = self._image_items(card, views[:1], seed, loras=loras)
             else:
                 parts = [('turn', SCENE if card['kind'] == 'setting' else TURN, False)]
                 if value['face']:
@@ -708,12 +712,14 @@ def generate(self, identity, value=None, *, by='builtin', ref=None, revision=Non
                 card['_generating'].update(source=source, stage='front', views=views[1:],
                                             job_views={jobs[0]: views[0]}, name=card['name'],
                                             description=card['description'])
+                if loras:
+                    card['_generating']['loras'] = loras
             result = dict(job=jobs[0], jobs=jobs, card=view(card))
             self._remember(record, ref, digest, result, client)
             self._save(record)
             return result
 
-    def _image_items(self, card, views, seed, front=None, from_photo=False):
+    def _image_items(self, card, views, seed, front=None, from_photo=False, loras=None):
         from ..image_gen.cast_views import size, uses_front, view_prompt, photo_prompt
         # A picture somebody put on the card (the user, or an agent passing on the user's photo) is the person.
         photo = next((row['media'] for row in card.get('images', []) if not row.get('generated')), None)
@@ -726,7 +732,8 @@ def _image_items(self, card, views, seed, front=None, from_photo=False):
                       view_prompt(dict(card, photo=True) if from_photo else card, part))
             items.append(dict(kind='image', prompt=prompt, width=width, height=height, seed=seed,
                               images=images, label=('Cast · %s · %s' % (card['name'], part))[:120],
-                              metadata=dict(cast_card=card['id'], cast_part=part)))
+                              metadata=dict(cast_card=card['id'], cast_part=part),
+                              **({'loras': loras} if loras else {})))
         return items
 
     def _image_views(self, identity, generating, hidden, rows):
@@ -742,7 +749,7 @@ def _image_views(self, identity, generating, hidden, rows):
             card = record['card']
             basis = dict(id=identity, kind=hidden['kind'], name=hidden['name'], description=hidden['description'])
             items = self._image_items(basis, hidden['views'], hidden['seed'], dict(file=front['result']['file']),
-                                      from_photo=bool(front['spec'].get('images')))
+                                      from_photo=bool(front['spec'].get('images')), loras=hidden.get('loras'))
             jobs, _ = self.jobs.submit_request(dict(jobs=items, ref='cast-' + hidden['ref'] + '-views'),
                 client='cast:' + identity, parent=dict(cast=identity, ref=hidden['ref']))
             card['generating']['jobs'].extend(row['id'] for row in jobs)

freevideo_engine/agent/comfy.py

@@ -322,7 +322,11 @@ def vlm_cancel(self, key):
     # Local image generation ----------------------------------------------------------------------------
     def _image(self):
         from ..image_gen.service import get_service
-        return get_service(self.server)
+        service = get_service(self.server)
+        if (service is not None and getattr(service, 'folders', None) is None
+                and getattr(self, 'folders', None) is not None):
+            service.folders = self.folders
+        return service
 
     @property
     def image(self):

freevideo_engine/agent/errors.py

@@ -7,6 +7,7 @@
 import math
 import re
 import uuid
+from ..image_gen.loras import ERRORS as LORA_ERRORS, MESSAGE_EN as LORA_MESSAGE_EN, MESSAGE_ZH as LORA_MESSAGE_ZH
 
 REQUEST_ID = ContextVar('agent_request_id', default=None)
 # The work session a token request belongs to (sessions.py); jobs and productions record it.
@@ -155,13 +156,14 @@ def image_failure(code, report_id=None):
     en, zh = IMAGE_REASONS.get(code, IMAGE_REASONS['invalid_request'] if code.startswith('invalid_')
                              or code == 'too_many_images' else IMAGE_REASONS['generation_failed'])
     valid = isinstance(report_id, str) and re.fullmatch(r'[0-9a-f]{32}', report_id)
-    return dict(reason=dict(en=en, zh=zh), report_id=report_id if valid and code not in (
+    return dict(reason=dict(en=en, zh=zh), report_id=report_id if valid and code not in LORA_ERRORS and code not in (
         'busy', 'cancelled', 'invalid_request', 'invalid_media', 'invalid_prompt', 'invalid_dimensions',
         'invalid_seed', 'too_many_images', 'disk_full') else None)
 
 
 # Recovery is a function of the committed task and phase, not just an error code.
 IMAGE_ERRORS = {
+    **LORA_MESSAGE_EN,
     'invalid_request': 'The picture request contains invalid fields.',
     'invalid_prompt': 'The picture prompt must contain text and be at most 2000 characters.',
     'invalid_dimensions': 'Picture width and height must be integers from 256 to 2048.',
@@ -183,6 +185,7 @@ def image_failure(code, report_id=None):
     'disk_full': IMAGE_REASONS['disk_full'][0],
     'cancelled': 'The picture job was cancelled.',
 }
+IMAGE_REASONS.update({code: (LORA_MESSAGE_EN[code], LORA_MESSAGE_ZH[code]) for code in LORA_ERRORS})
 for _code, _message in IMAGE_ERRORS.items():
     if _code not in ERRORS:
         ERRORS[_code] = (503 if _code in ('pack_missing', 'closed', 'image_unavailable') else 422,
@@ -267,6 +270,7 @@ def safe_details(value):
 # One public sentence shared by conversation cards, HTTP and MCP. Backend text
 # is classified, never echoed: it can contain prompts, filenames or credentials.
 FAILURE_REASONS_ZH = {
+    **LORA_MESSAGE_ZH,
     'lan_unavailable': '局域网设置需要启动器安装记录',
     'out_of_memory': '生成时显存或内存不足',
     'insufficient_memory': '可用显存或内存不够生成这段内容',
@@ -316,6 +320,7 @@ def safe_details(value):
     'internal_error': '没有完成',
 }
 FAILURE_REASONS_EN = {
+    **LORA_MESSAGE_EN,
     'lan_unavailable': 'Phone access requires FreeVideo installed with the launcher',
     'out_of_memory': 'Graphics or system memory ran out during generation',
     'insufficient_memory': 'There is not enough free graphics or system memory for this generation',

freevideo_engine/agent/jobs.py

@@ -1999,7 +1999,7 @@ def wait_image(self, row, code, *, waiting_for=None):
     def start_image(self, row):
         spec = row['spec']
         row['_phase'] = 'preparing images'
-        value = {k: spec[k] for k in ('width', 'height', 'seed') if k in spec}
+        value = {k: spec[k] for k in ('width', 'height', 'seed', 'loras') if k in spec}
         if spec.get('people'):
             # A first frame: the people's pictures at the asked places, in the scene shown or described.
             action = 'compose'

freevideo_engine/agent/schema.py

@@ -245,12 +245,21 @@ def vlm(*, local=False):
         ('kind', 'questions'), description='x-job-kind also checks the sum of images and sampled video frames across questions (at most 24).', **{'x-job-kind': 'vlm'})
 
 
+def image_loras():
+    from ..image_gen.loras import MAX_LORAS, MAX_NAME, MIN_STRENGTH, MAX_STRENGTH
+    # The name's rules (extension, folders, control characters) are checked by validate_selection with its own messages.
+    return array(obj(dict(name=dict(type='string', minLength=1, maxLength=MAX_NAME), strength=number(MIN_STRENGTH, MAX_STRENGTH, default=1.),
+                          enabled=dict(type='boolean', default=True)), ('name',)), high=MAX_LORAS,
+                 **{'x-image-loras': True})
+
+
 def image(*, local=False):
     return obj(dict(kind={'const': 'image', 'type': 'string'},
         prompt=dict(type='string', minLength=1, maxLength=2000),
         width=number(256, 2048, integer=True), height=number(256, 2048, integer=True),
         aspect=enum(IMAGE_ASPECTS, description='Instead of width and height; 1:1 when all three are omitted.'),
         seed=nullable(number(-2**63, 2**64 - 1, integer=True)),
+        loras=image_loras(),
         images=array({'oneOf': [FILE, obj(dict(file=FILE), ('file',))]}, high=4, description='Pictures to change '
             'or combine: input files, such as result.file of an earlier picture job. The prompt calls them image 1, image 2.'),
         people=array(obj(dict(image=FILE, cast=CAST, name=string(40), x=number(0, 1), height=number(.05, 1),
@@ -613,7 +622,8 @@ def cast_schema(name, *, local=False):
         return obj(dict(revision=number(1, integer=True), ref=REF))
     if name == 'generate':
         return obj(dict(face=dict(type='boolean', default=False, description='Also make one face close-up (video; the image model always does).'),
-                        source=enum(('image', 'video'), description='Defaults to image when the local image model is ready, otherwise video.'),
+                        loras=image_loras(),
+                        source=enum(('image', 'video'), description='Defaults to image when LoRAs are selected or the local image model is ready, otherwise video.'),
                         replace=dict(type='boolean', default=False,
                                      description='Replace pictures the user did not add; default fills empty slots.'),
                         revision=number(1, integer=True), ref=REF))
@@ -1227,6 +1237,12 @@ def cross_issues(schema, value, path):
     errors = []
     def bad(field, allowed, got=None):
         errors.append(diagnostic(path + '/' + field, allowed, got))
+    if schema.get('x-image-loras'):
+        from ..image_gen.loras import MESSAGE_ZH, validate_selection
+        try:
+            validate_selection(value)
+        except ValueError as error:
+            errors.append(diagnostic(path, MESSAGE_ZH[str(error)], None))
     flags = schema.get('x-media')
     if flags is not None:
         item = {'file': value} if isinstance(value, str) else value
@@ -1551,7 +1567,7 @@ def openapi():
     description='Work session; omit for latest.')
 
 
-SHAPE_MIN_BYTES = 400
+SHAPE_MIN_BYTES = 200
 RECIPE_TOOL = 'freevideo_edit_image'
 
 

freevideo_engine/agent/spec.py

@@ -271,6 +271,10 @@ def normalize(value, *, port=None):
             result['questions'].append(item)
         result.update({k: value[k] for k in ('format', 'max_tokens', 'system') if k in value})
     elif kind == 'image':
+        from ..image_gen.loras import validate_selection
+        loras = validate_selection(original.get('loras', []))
+        if loras:
+            result['loras'] = loras
         width, height = ((value['width'], value['height']) if 'width' in value else
                          contract.IMAGE_ASPECTS[value.get('aspect', '1:1')])
         result.update(prompt=value.get('prompt', ''), width=width, height=height,