FreeVideo 0.4.20

docs/zh-CN/使用说明.md

@@ -46,7 +46,7 @@
 
 ### 限制
 
-- 两种做法一样，一条视频最多 5 段（第一段加 4 次接着生成），所以创作面板里时长最多填 60 秒。接续的每一段至少 10 秒，整条约 70 秒。剪辑视图里逐段生成同样适用。段数越多，衔接处越容易出现变化，适合切换镜头处，可直接切镜头，用对应的参考图开始新的一段。
+- 两种做法一样，一条视频最多 8 段（第一段加 7 次接着生成）；创作面板里时长最多填 60 秒。接续的每一段至少 10 秒，整条约 110 秒。剪辑视图里逐段生成同样适用。段数越多，衔接处越容易出现变化，适合切换镜头处，可直接切镜头，用对应的参考图开始新的一段。
 - 接着生成使用轻量档，需要 NVIDIA 显卡或 Mac，也不能和“继续二采”同时使用。
 - 单次生成的最长时长随画幅和显卡而不同，在创作面板的“可生成范围”中可查看各尺寸的最长时长和预计用时。
 - 分段生成同样需要 NVIDIA 显卡或 Mac。

freevideo_engine/adaptive.py

@@ -41,6 +41,32 @@ def commit_exhausted(error):
     return found(error)
 
 
+# A CUDA error that may leave the device itself unusable; never retried.
+DEVICE_STATE_ERRORS = ('cuda error: unknown error', 'device is lost', 'device lost', 'gpu is lost',
+    'illegal memory access', 'device-side assert', 'unspecified launch failure', 'driver shutting down',
+    'device has been removed', 'cuda error: launch timeout', 'the launch timed out',
+    'cudaerrorillegaladdress', 'cudaerrorunknown')
+
+
+def second_pass_resumable(error, failure, metrics, attempts):
+    """A rejected, non-sticky CUDA call in the second pass, first pass retained.
+
+    The retry is one fresh process that loads the retained first pass and
+    repeats only the second pass with the same placement. A sticky device
+    error, an unconfirmed worker stop, a failure after sampling finished and
+    any earlier resumed attempt stay final; out-of-memory keeps its own path.
+    """
+    from .decode_resume import completed_sampling, refine_checkpoint_complete
+    if failure.get('kind') != 'code_error' or not isinstance(metrics, dict):
+        return False
+    recorded = str(metrics.get('error', '')).lower()
+    if not (cuda_runtime_error(error) or 'cuda error' in recorded) or 'out of memory' in recorded:
+        return False
+    if not refine_checkpoint_complete(metrics) or completed_sampling(metrics):
+        return False
+    return not any(row.get('decision', {}).get('second_pass_resume') for row in attempts)
+
+
 def classify_failure(error, metrics=None):
     """Only positively identified resource failures are retriable.
 
@@ -55,10 +81,7 @@ def classify_failure(error, metrics=None):
     attached = getattr(error, 'cleanup_errors', [])
     cleanup += attached if isinstance(attached, list) else [attached]
     text = '\n'.join([str(error), str(metrics.get('error', ''))] + [str(item) for item in cleanup]).lower()
-    if any(part in text for part in ('cuda error: unknown error', 'device is lost', 'device lost',
-            'gpu is lost', 'illegal memory access', 'device-side assert', 'unspecified launch failure',
-            'driver shutting down', 'device has been removed', 'cuda error: launch timeout',
-            'the launch timed out', 'cudaerrorillegaladdress', 'cudaerrorunknown')) or details.get('kind') == 'cuda_error':
+    if any(part in text for part in DEVICE_STATE_ERRORS) or details.get('kind') == 'cuda_error':
         kind, outcome, retry = 'cuda_error', 'discarded', False
     elif getattr(error, 'worker_stop_confirmed', None) is False:
         kind, outcome, retry = 'worker_stop_unconfirmed', 'discarded', False
@@ -628,17 +651,25 @@ def execute_attempts(history, identity, profile, canvas, launch, *,
             history.finish(attempt, failure['outcome'], details=details)
             row.update(state=failure['outcome'], failure=failure)
             persist()
-            if not automatic or not failure['retryable'] or index == max_retries:
+            resume = (automatic and index < max_retries and not failure['retryable']
+                      and second_pass_resumable(error, failure, metrics, report['resource_attempts']))
+            if not automatic or not (failure['retryable'] or resume) or index == max_retries:
                 raise
             previous_ram_retries = sum(r.get('failure', {}).get('kind') == 'ram_pressure'
                                        for r in report['resource_attempts'][:-1])
             previous_gpu_retries = sum(r.get('failure', {}).get('kind') == 'gpu_oom'
                                        for r in report['resource_attempts'][:-1])
             from .decode_resume import completed_sampling
-            updated, decision = next_placement(current, failure, (metrics or {}).get('phase', 'sample'), canvas,
-                                               previous_ram_retries=previous_ram_retries,
-                                               previous_gpu_retries=previous_gpu_retries,
-                                               sampling_complete=completed_sampling(metrics))
+            if resume:
+                updated, decision = copy.deepcopy(current), dict(
+                    reason='A CUDA call was rejected in the second pass; repeat only the second pass '
+                           'from the retained first pass with the same placement.',
+                    numerical_class='unchanged', second_pass_resume=True)
+            else:
+                updated, decision = next_placement(current, failure, (metrics or {}).get('phase', 'sample'), canvas,
+                                                   previous_ram_retries=previous_ram_retries,
+                                                   previous_gpu_retries=previous_gpu_retries,
+                                                   sampling_complete=completed_sampling(metrics))
             if updated is None:
                 row['recovery_note'] = decision['reason']
                 persist()

freevideo_engine/agent/cast.py

@@ -196,6 +196,7 @@ def __init__(self, jobs):
         self.directory = jobs.directory / 'cast'
         self.records = {}
         self.deleted = {}
+        self._unreadable = set()  # Bad cards whose quarantine rename failed.
         self.changed = threading.Condition(jobs.lock)
         self.landing = set()  # Cards whose reference pictures a tick is landing now (under the lock).
         self.load()
@@ -204,6 +205,8 @@ def __init__(self, jobs):
     def load(self):
         for folder, target in ((self.directory / '.deleted', self.deleted), (self.directory, self.records)):
             for path in sorted(folder.glob('*.json')):
+                if not CARD.fullmatch(path.stem):
+                    continue
                 try:
                     record = json.loads(path.read_text(encoding='utf-8'))
                     _check(record, path.stem)
@@ -212,11 +215,16 @@ def load(self):
                         path.unlink()
                         continue
                     target[path.stem] = record
+                    self._unreadable.discard(path)
+                except FileNotFoundError:
+                    continue  # A missing record is not an unreadable card.
                 except (ValueError, KeyError, TypeError, AttributeError, IndexError, RecursionError, OSError):
                     try:
                         path.replace(path.with_name(path.name + '.bad-' + uuid.uuid4().hex))
+                        self._unreadable.discard(path)
                     except OSError:
-                        pass
+                        if target is self.records:
+                            self._unreadable.add(path)
                     logging.getLogger(__name__).warning('Isolated an invalid cast card; other cards remain available')
 
     def _write(self, folder, record):
@@ -235,6 +243,7 @@ def _write(self, folder, record):
     def _save(self, record):
         self._write(self.directory, record)
         self.records[record['card']['id']] = record
+        self._unreadable.discard(self.directory / (record['card']['id'] + '.json'))
         self.changed.notify_all()
 
     def _record(self, identity):
@@ -411,7 +420,11 @@ def list(self, *, q=None, kind=None, origin=None, production=None, limit=50):
                     and (not production or production in c['used_by'] or c['made_for'] == production)
                     and (not text or text in c['name'].casefold() or text in (c['description'] or '').casefold())]
             rows.sort(key=lambda c: (c['updated_at'], c['id']), reverse=True)
-            return dict(cards=[row_view(c) for c in rows[:limit]], total=len(rows))
+            unreadable = 0
+            if production is None:
+                quarantined = {path for path in self.directory.glob('*.json.bad-*') if path.is_file()}
+                unreadable = len(quarantined | {path for path in self._unreadable if path.exists()})
+            return dict(cards=[row_view(c) for c in rows[:limit]], total=len(rows), unreadable=unreadable)
 
     def update(self, identity, value, *, by='builtin', ref=None, revision=None, local=False, client=None):
         shape = schema.cast_schema('update', local=local)

freevideo_engine/agent/errors.py

@@ -66,7 +66,7 @@
     'chain_unavailable': (503, True, 'The continuous generation backend is unavailable.', 'Start FreeVideo with the continuous generation backend enabled, then retry.'),
     'chain_missing': (409, True, 'The saved chain receipt is unavailable.', 'Restore the chain receipt and retry resume, or submit a new job.'),
     'join_failed': (409, True, 'The generated blocks could not be joined; completed blocks are retained.', 'POST /freevideo/agent/v1/jobs/{id}/resume to retry joining the completed blocks.'),
-    'chain_too_long': (422, False, 'This video can have up to 5 parts. Make it shorter.', 'Submit a new job with a smaller seconds; continue_from counts the parts of the video it continues.'),
+    'chain_too_long': (422, False, 'One continuous shot can have up to 8 parts. Make it shorter.', 'Submit a new job with a smaller seconds, or cut to a new shot with a new job without continue_from; continue_from counts the parts of the video it continues.'),
     'chain_rejected': (409, False, 'The continuous backend rejected this generation.', 'Read GET /freevideo/agent/v1/capabilities and submit a supported generation request.'),
     'invalid_wait': (422, False, 'wait must be a finite nonnegative number of seconds.', 'Use wait=0 or wait=25; larger valid values are clamped to 25 seconds.'),
     'invalid_resume': (422, False, 'resume accepts an empty object or an optional ref only.', 'Send {} or {"ref":"resume-1"}; submit a new job to change generation parameters.'),

freevideo_engine/agent/guide.py

@@ -246,8 +246,8 @@
     'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should '
     'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a '
     'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and '
-    'continuing again and again is not recommended. One video has at most 5 parts (the first and 4 '
-    'continuations), each continued part at least 10 seconds, about 70 seconds in all; filling that is not '
+    'continuing again and again is not recommended. One video has at most 8 parts (the first and 7 '
+    'continuations), each continued part at least 10 seconds, about 110 seconds in all; filling that is not '
     'recommended. Past it, cut to a new shot.',
     'A question is not a request. When the user asks about progress or status ("is it working?", "what happened?") '
     'answer it in one sentence from what you know or from the job and white-model status, and start nothing; never '
@@ -282,7 +282,7 @@
     '先用 people（角色图和各自的位置）做出首帧，再用它生成视频。',
     '默认切镜头。能切镜头的地方就切：新镜头直接用对应的参考图（人物卡、场景图）生成，不接上一段视频。'
     '只有不能切镜的长镜头、必须连贯的动作、必须连贯的声音，才接着上一段生成，而且不建议一段接一段地连续接。'
-    '一条视频最多 5 段（第一段加 4 次接着生成），接着生成的每一段至少 10 秒，总长约 70 秒；不建议接满，超过就切镜头。',
+    '一条视频最多 8 段（第一段加 7 次接着生成），接着生成的每一段至少 10 秒，总长约 110 秒；不建议接满，超过就切镜头。',
     '问话不是要求。用户问进度或状态（“在做吗”“怎么了”）时，用一句话照实回答，依据是你所知道的、任务和白模的状态，'
     '什么都不要开始；不要把问句当提示词。某条 inbox 带有 whitebox 时，说明用户刚启动了白模场景，'
     '它还在准备或已经失败：照实说是哪一种，再等用户回话。',
@@ -568,7 +568,7 @@ def connection_text(value):
     'before uploading. Unknown fields are rejected and every batch is validated before it is recorded.',
     'POST /plan gives the length supported for the current size and quality. Without continuation, only a '
     'single block can run; unknown device limits use 15 s. Multiple blocks return continuation_unavailable.',
-    'One video has at most 5 parts (60 s in one request). continue_from counts the parts of the video it '
+    'One video has at most 8 parts (60 s in one request). continue_from counts the parts of the video it '
     'continues; plan and submit refuse a longer one and name the longest seconds that still fits.',
     'features in the status reply names what this installation offers; false means not usable now (for example '
     'chain.auto while continuation is false), and a missing name means not offered here.',

freevideo_engine/agent/jobs.py

@@ -480,6 +480,7 @@ def __init__(self, port, directory, *, clock=now):
         self.directory = Path(directory)
         self.clock = clock
         self.jobs = {}
+        self.unreadable = 0
         self.lock = threading.RLock()
         self._receipts = _ReceiptWrites(self.directory) if windows() else None
         self.cpu = threading.Semaphore(1)
@@ -507,28 +508,68 @@ def __init__(self, port, directory, *, clock=now):
 
     # Records -------------------------------------------------------------------------------------------
     def load(self):
+        self.unreadable = 0
         if not self.directory.is_dir():
             return
         # One atomic batch receipt is the commit point. Later per-job records
         # override its initial rows. A crash never exposes a partial submission.
         initial = {}
+        unreadable_ids, unreadable_batches = set(), set()
+
+        def readable(row):
+            return (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id')))
+                    and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDS
+                    and row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict)
+                    and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at'])
+                    and (self._receipts is None or (type(row.get('_receipt_revision', 0)) is int
+                         and row.get('_receipt_revision', 0) >= 0)))
+
         for path in sorted(self.directory.glob('b*.json')):
+            if not re.fullmatch(r'b[0-9a-f]{32}', path.stem):
+                continue
             try:
                 batch = json.loads(path.read_text(encoding='utf-8'))
-                initial.update({r['id']: r for r in batch['jobs']})
+                if not isinstance(batch, dict) or not isinstance(batch.get('jobs'), list):
+                    raise ValueError
+            except FileNotFoundError:
+                continue
             except (OSError, ValueError, KeyError, TypeError):
+                unreadable_batches.add(path)
+                logging.getLogger(__name__).warning('Skipped an unreadable batch file')
                 continue
+            bad = False
+            for row in batch['jobs']:
+                if readable(row):
+                    initial[row['id']] = row
+                else:
+                    bad = True
+                    identity = row.get('id') if isinstance(row, dict) else None
+                    if isinstance(identity, str) and specs.JOB_ID.fullmatch(identity):
+                        unreadable_ids.add(identity)
+                    else:
+                        # No task id can be recovered: count this receipt once.
+                        unreadable_batches.add(path)
+            if bad:
+                logging.getLogger(__name__).warning('Skipped an unreadable batch file')
         records = list(initial.values())
         for path in sorted(self.directory.glob('j*.json')):
+            if not specs.JOB_ID.fullmatch(path.stem):
+                continue
             try:
-                records.append(json.loads(path.read_text(encoding='utf-8')))
-            except (OSError, ValueError):
+                row = json.loads(path.read_text(encoding='utf-8'))
+                if not readable(row) or row['id'] != path.stem:
+                    raise ValueError
+                records.append(row)
+            except FileNotFoundError:
                 continue
+            except (OSError, ValueError):
+                unreadable_ids.add(path.stem)
+                logging.getLogger(__name__).warning('Skipped an unreadable job file')
         if self._receipts is not None:
             for path in sorted(self.directory.glob('j*.pending')):
                 try:
                     row = json.loads(path.read_text(encoding='utf-8'))
-                    if isinstance(row, dict) and path.name.startswith(str(row.get('id')) + '.'):
+                    if readable(row) and path.name.startswith(str(row.get('id')) + '.'):
                         records.append(row)
                 except (OSError, ValueError):
                     continue  # An interrupted staging write is not a receipt.
@@ -545,10 +586,7 @@ def load(self):
                     latest[row['id']] = row
             records = list(latest.values())
         for row in records:
-            if (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id')))
-                    and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDS
-                    and row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict)
-                    and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at'])):
+            if readable(row):
                 if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'image_edit') + GPU_KINDS:
                     # This process restarted while the work ran in it; nothing continues it.
                     self.finish(row, 'failed', error=dict(code='interrupted',
@@ -564,6 +602,7 @@ def load(self):
                     # Before device-aware Agent planning, every accepted job used the unknown limit.
                     row['spec'].setdefault('single_max_frames', None)
                 self.jobs[row['id']] = row
+        self.unreadable = len(unreadable_batches) + len(unreadable_ids - self.jobs.keys())
         for row in self.jobs.values():
             if (row['kind'] != 'generate' or row['spec'].get('refine_from')
                     or row['status'] in DONE and not row.get('resume_pending')