From 0026f90c911c2232c7e8039c2252af31b4720651 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:17:56 +0000 Subject: [PATCH 1/8] Retain resident model components by recent requests and keep the usage habit across restarts --- freevideo_engine/comfy_nodes.py | 17 ++- freevideo_engine/encode_worker.py | 4 +- freevideo_engine/media_encoding.py | 14 ++- freevideo_engine/resident_models.py | 164 ++++++++++++++++++++++++++- freevideo_engine/resident_process.py | 123 ++++++++++++++++---- freevideo_engine/resident_worker.py | 19 ++++ freevideo_engine/usage_habit.py | 83 ++++++++++++++ freevideo_engine/worker.py | 6 - 8 files changed, 384 insertions(+), 46 deletions(-) create mode 100644 freevideo_engine/usage_habit.py diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index 2ff56df0..dcd442ed 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -4,7 +4,7 @@ from comfy_api.latest import ComfyExtension, InputImpl, io, ui -from . import comfy_bridge, workflow_repairs +from . import comfy_bridge, usage_habit, workflow_repairs from .comfy_continuation import node_frames, optional_text, part_steps, request_options from .default_prompt import DEFAULT_PROMPT from .original_audio import audio_use_option @@ -397,8 +397,8 @@ def busy(): gpu = comfy_bridge.installation()[1].get('gpu_uuid') except (OSError, ValueError, KeyError, TypeError): gpu = None - OWNER.finished(next_part=next_part, gpu=gpu, warm=True, - agent=server is not None and str(getattr(server, 'client_id', '') or '').startswith('agent:')) + kept = OWNER.finished(next_part=next_part, gpu=gpu, warm=True, + agent=server is not None and str(getattr(server, 'client_id', '') or '').startswith('agent:')) from .comfy_environment import executor_device if target and not result_reused[0] and not executor_device(): client = server.client_id if server is not None else None @@ -411,12 +411,19 @@ def warmed(value): IDLE.schedule(path, output.with_suffix('.prewarm.json'), ram_budget=measured.get('profile', {}).get('inference_ram_budget_gb', 0) * 1e9, busy=busy, notify=warmed, purpose='upscale') + elif (isinstance(kept, dict) and kept.get('action') == 'kept' and next_part is None + and usage_habit.dominant((usage_habit.load() or {}).get('recent')) == 'prompt'): + # Prompts change often: place the encoder on the card while the kept worker + # idles. It yields to any new task, and only what fits free memory is placed. + root, machine = comfy_bridge.installation() + environment = comfy_bridge.engine_environment(root, comfy_bridge.source_root()) + environment.update(FREEVIDEO_HOME=str(root), PYTHONPATH=str(comfy_bridge.source_root())) + OWNER.warm_encoder(output, machine['python'], environment, busy=busy, notify=warmed) else: - # File cache only: a kept worker must not grow on the card while it waits. IDLE.schedule(path, output.with_suffix('.prewarm.json'), ram_budget=measured.get('profile', {}).get('inference_ram_budget_gb', 0) * 1e9, busy=busy, notify=warmed) - except (OSError, RuntimeError): + except (OSError, RuntimeError, ValueError, KeyError, TypeError): pass # A saved video remains successful if idle warming cannot start. return io.NodeOutput(InputImpl.VideoFromFile(str(output)), report, ui=preview) diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index fc3a534e..4ceba4ec 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -576,8 +576,8 @@ def attempt(index, failure): save(request['metrics'], {**loading, 'success': False, 'phase': 'keyframe_vae', 'load_seconds': load_seconds, 'prompt_encode_seconds': prompt_encode_seconds}) # Only the encoder is read; the decoder in the shared shards is skipped. - vae, _ = load_video_encoder(request['base'], before_upload=None if resident is None else - lambda model: resident.input_vae_room(model, dict(width=width, height=height))) + vae = (resident.input_video_encoder(request['base'], dict(width=width, height=height)) + if resident is not None else load_video_encoder(request['base'])[0]) torch.cuda.synchronize() keyframe_metrics['keyframe_vae_load_seconds'] = time.perf_counter() - vae_started vae_started = time.perf_counter() diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index c2da401e..c3a7a446 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -351,9 +351,12 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu on_mps = torch.device(device).type == 'mps' # On MPS the weights stay in the checkpoint mapping and each encoder # stage uploads its own while it runs (bounded_encoder). - vae, _ = load_video_encoder(base, before_upload=None if resident is None else - lambda model: resident.input_vae_room(model, canvas), - device='cpu' if on_mps else device) + if resident is not None and not on_mps: + vae = resident.input_video_encoder(base, canvas, device=device) + else: + vae, _ = load_video_encoder(base, before_upload=None if resident is None else + lambda model: resident.input_vae_room(model, canvas), + device='cpu' if on_mps else device) lifetime = nullcontext() if on_mps: from .backends.mps_vae_encode import bounded_encoder @@ -382,8 +385,9 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu empty_cache() if any(row.get('audio') for row in normalized): print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding reference audio'}), flush=True) - vae, _ = load_audio_vae(base, before_upload=None if resident is None else - lambda model: resident.input_vae_room(model, canvas, audio=True), device=device) + vae = (resident.input_audio_encoder(base, canvas, device=device) + if resident is not None and torch.device(device).type != 'mps' else + load_audio_vae(base, device=device)[0]) mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1) std = torch.tensor(vae.config.latents_std).view(1, 1, -1) if vae.config.sampling_rate != 32000: diff --git a/freevideo_engine/resident_models.py b/freevideo_engine/resident_models.py index b55c139c..7f03200a 100644 --- a/freevideo_engine/resident_models.py +++ b/freevideo_engine/resident_models.py @@ -1,17 +1,83 @@ """Request-aware model residency inside the isolated interactive GPU worker.""" import gc +import hashlib import json from pathlib import Path import sys import time import torch +from . import usage_habit from .torch_compat import empty_host_cache, host_cache_release_supported from .device_index import device_properties GiB = 1 << 30 # Where a preview's sampling peak is kept, apart from its complete request's. PREVIEW_STAGE = 'engine_preview' +HABIT_WINDOW = 4 + + +def retention_order(history): + """Cache roles to evict first; two thirds of recent transitions wins. + + History contains at most four successful requests' digests, never inputs. + Image changes win ties with prompt changes, then same-prompt reruns. With + fewer than two successes (or no dominant habit), use the old engine-first + order. Active stages still use make_room's existing admission rules. + """ + return eviction_order(usage_habit.dominant(usage_habit.transitions(history))) + + +def eviction_order(habit): + """Eviction order for a habit kind from usage_habit.dominant (None: default).""" + if habit == 'images': + keep = ('input_video_vae', 'encoder', 'conditioning', 'engine', + 'input_audio_vae', 'video_vae', 'audio_vae') + elif habit == 'prompt': + keep = ('encoder', 'conditioning', 'engine', 'input_video_vae', + 'input_audio_vae', 'video_vae', 'audio_vae') + elif habit == 'rerun': + keep = ('conditioning', 'engine', 'video_vae', 'audio_vae', + 'input_video_vae', 'input_audio_vae', 'encoder') + else: + return ('engine', 'encoder', 'audio_vae', 'video_vae', + 'input_audio_vae', 'input_video_vae', 'conditioning') + return tuple(reversed(keep)) + + +def _input_digest(value): + """Hash CPU conditioning without retaining inputs or copying mapped payloads.""" + digest = hashlib.sha256() + def visit(item): + if isinstance(item, torch.Tensor): + tensor = item.detach().cpu().contiguous() + digest.update(str((str(tensor.dtype), list(tensor.shape))).encode()) + digest.update(memoryview(tensor.reshape(-1).view(torch.uint8).numpy())) + elif isinstance(item, dict): + for name in sorted(item): + visit(name) + visit(item[name]) + elif isinstance(item, (tuple, list)): + for child in item: + visit(child) + else: + digest.update(json.dumps(item, sort_keys=True).encode()) + digest.update(b'\0') + visit(value) + return digest.hexdigest() + + +class ConditioningMemo(torch.nn.Module): + """The engine's bounded CPU refined-text results, independently evictable.""" + def __init__(self, memo): + super().__init__() + self.memo = memo + + def to(self, device): + if device != 'meta': + raise ValueError('Conditioning memo stays on the CPU') + self.memo.clear() + return self def key(value): @@ -156,6 +222,11 @@ def __init__(self): from .streamed_weights import ConvertedStore # Converted decoder layers streamed from the checkpoint; rebuilt on demand. self.converted_store = ConvertedStore() + self.history = [] + # Transition kinds kept across restarts (see usage_habit); history + # holds only this process's digests. + stored = usage_habit.load() + self.recent = stored['recent'] if stored else [] # Set by a request that can read the whole device: bytes no process # holds, and the growth reserve that request keeps out of them. The # planning budget already excludes that reserve; the device reading @@ -200,6 +271,11 @@ def drop(self, role, reason): import comfy.model_management encoder_discarded = release_encoder(model, comfy.model_management) elif role == 'engine': + cached = self.entries.get('conditioning') + if cached and getattr(model, 'refined_text_memo', None) is cached['model'].memo: + # Engine.close clears its dictionary; detach the independently + # owned results so an engine miss need only reload weights. + model.refined_text_memo = {} model.close() else: model.to('meta') @@ -268,14 +344,15 @@ def ram_fits(self, need=2*GiB): (self.ram_budget is None or used + need <= self.ram_budget)) def settle(self, pinned=0): - """End of a request: keep the engine and encoder only if the next cold encoder still fits. + """Keep as many independent components as the next cold start permits. The RAM guard counts the whole session, so retained engine weights plus the next encoder's start (about 8 GiB) must stay inside the budget, and so must the host weights a fresh plan would pin beyond those the engine already holds: measured, a retained copy left no room to - pin and sampling took 95 s instead of 38 s. Otherwise the engine goes - as it did with one worker per request. + pin and sampling took 95 s instead of 38 s. Evict lower-priority + components first, checking live RAM again after each release. This does + not change an engine's layer layout. """ # A retained engine's pinned weights are reused by the next request # rather than pinned again, so they count once, in the session's use. @@ -283,8 +360,11 @@ def settle(self, pinned=0): need = 8*GiB + max(0, int(pinned) - held) if not self.ram_fits(need): self.converted_store.clear() - for role in ('engine', 'encoder'): - if role in self.entries and not self.ram_fits(need): + order = eviction_order(self.habit()) + for role in (*order, *(role for role in self.entries if role not in order)): + if self.ram_fits(need): + break + if role in self.entries: self.drop(role, 'Retained weights leave no host room for the next request') if role == 'engine': need = 8*GiB + int(pinned) @@ -295,6 +375,37 @@ def settle(self, pinned=0): except (OSError, AttributeError): pass + def succeeded(self, request): + """Observe video successes only, locally; encoding/preloads are not jobs. + + Read the existing conditioning with mmap after generation has returned, + so failures and cancellations do not train the policy. An unreadable + or legacy external input leaves the previous habit unchanged. + """ + path = Path(request['conditioning']) + value = torch.load(path, map_location='cpu', weights_only=True, mmap=True) + prompt = value.get('prompt') + if prompt is None: + prompt = value['prompt_embeds'][value['text_token_tags'] == 1] + images = dict(anchors=value.get('keyframe_anchors'), latents=value.get('condition_latents'), + references=[{k: row[k] for k in ('kind', 'latent') if k in row} + for row in value.get('references', []) if row['kind'] != 'audio']) + row = dict(prompt=_input_digest(prompt), images=_input_digest(images), + run=key(dict(seed=request.get('seed'), options=request.get('engine_options'), + geometry=request.get('geometry'), sampling=request.get('sampling_plan')))) + kinds = usage_habit.transition(self.history[-1], row) if self.history else None + self.history = (self.history + [row])[-HABIT_WINDOW:] + if kinds is not None: + self.recent = (self.recent + [kinds])[-(HABIT_WINDOW - 1):] + try: + usage_habit.record(kinds) + except OSError: + pass # A habit that cannot be saved is only forgotten at restart. + + def habit(self): + """The dominant kind of change, from stored transitions or this process's digests.""" + return usage_habit.dominant(self.recent or usage_habit.transitions(self.history)) + def input_vae_room(self, model, canvas, *, audio=False): # Input encoding has its own live workspace; it cannot assume an output # decoder's memory footprint. Charge actual retained CPU model tensors @@ -311,7 +422,41 @@ def input_vae_room(self, model, canvas, *, audio=False): # model's host copy is released unless the live budget covers the # stage as well. Plentiful hosts keep it for reuse. host = weights + (2*GiB if audio else max(8*GiB, 26*GiB*area)) - self.make_room('input_audio_vae' if audio else 'input_video_vae', weights + workspace, ram_need=host) + role = 'input_audio_vae' if audio else 'input_video_vae' + retained = model if self.entries.get(role, {}).get('model') is model else None + self.make_room(role, weights + workspace, model=retained, ram_need=host) + + def input_video_encoder(self, base, canvas, *, device='cuda'): + """Reuse just the image/video encoder; never load the output decoder.""" + from .vae_weights import load_video_encoder + directory = Path(base) / 'vae' + identity = key(dict(files=[file_identity(path) for path in + sorted([directory/'config.json', *directory.glob('*.safetensors')])], + device=str(device))) + found = self.take('input_video_vae', identity) + if found is not None: + self.input_vae_room(found, canvas) + return found + found, _ = load_video_encoder(base, before_upload=lambda model: self.input_vae_room(model, canvas), + device=device) + self.put('input_video_vae', identity, found) + return found + + def input_audio_encoder(self, base, canvas, *, device='cuda'): + """Reuse the reference-audio VAE the way the image/video encoder is reused.""" + from .vae_weights import load_audio_vae + directory = Path(base) / 'audio_vae' + identity = key(dict(files=[file_identity(path) for path in + sorted([directory/'config.json', *directory.glob('*.safetensors')])], + device=str(device))) + found = self.take('input_audio_vae', identity) + if found is not None: + self.input_vae_room(found, canvas, audio=True) + return found + found, _ = load_audio_vae(base, before_upload=lambda model: self.input_vae_room(model, canvas, audio=True), + device=device) + self.put('input_audio_vae', identity, found) + return found def device_ceiling(self): """What this process's allocator can hold before the device itself is full. @@ -450,6 +595,13 @@ def engine(self, request, factory): if found is None: found = factory() self.put('engine', identity, found) + memo = getattr(found, 'refined_text_memo', None) + if isinstance(memo, dict): + cached = self.take('conditioning', identity) + if cached is None: + cached = ConditioningMemo(memo) + self.put('conditioning', identity, cached) + found.refined_text_memo = cached.memo return found, hit def encoder_room(self, clip, tokens, fallback_peak): diff --git a/freevideo_engine/resident_process.py b/freevideo_engine/resident_process.py index 6003da8f..8337bdce 100644 --- a/freevideo_engine/resident_process.py +++ b/freevideo_engine/resident_process.py @@ -23,17 +23,17 @@ ENV = 'FREEVIDEO_RESIDENT_SESSION' GiB = 2**30 -# A finished request keeps the worker for the next one while host memory is ample -# (for as long as nothing needs the memory, unless KEEP_IDLE_SECONDS sets a limit), and it is given up at once when anything else -# needs the memory: another program's GPU use grows, host memory runs low, an +# A successful request settles individual caches inside the worker, then keeps +# it without an idle limit unless KEEP_IDLE_SECONDS sets one. It gives up the +# worker at once when another program's GPU use grows, host memory runs low, an # image or prompt model starts (those release it themselves), or the agent asks -# for the card back. A request that is not followed by a queued part and fails, -# or a machine without the room, releases it as before; a next part already on -# its way is held for HOLD_SECONDS. +# for the card back. A failed request without a queued part releases it as +# before; a next part already on its way is held for HOLD_SECONDS. HOLD_SECONDS = 90. AGENT_HOLD_SECONDS = 30. KEEP_IDLE_SECONDS = None OTHER_GPU_GROWTH = 512 * 2**20 +PRELOAD_CONTEXT_BYTES = 1536 * 2**20 # what the preload's process adds beside its weights (context, allocator, staging; measured 1.4 GiB) HOST_FLOOR = 4 * GiB # Below this much host memory (free plus the idle worker's own), a kept worker's # base leaves the next part less room than a fresh one; measured on 16 GiB. @@ -363,6 +363,7 @@ def __init__(self): self.identity = None self.active = False self.prewarm = None + self.gpu = self.baseline = self.own = None self.mutex = threading.RLock() self.generation = 0 self.idle_since = None @@ -384,7 +385,7 @@ def take_decisions(self): def finished(self, *, next_part=None, agent=False, gpu=None, warm=False): """Release the idle worker now, or keep it for the next request. - A successful request (warm) keeps it while host memory is ample and the card's use can be read, so + A successful request (warm) keeps its settled caches above HOST_FLOOR while the card's use can be read, so another program's growth is seen; a next part on its way keeps it for a short hold. Anything else gives the memory back now. """ @@ -398,6 +399,8 @@ def finished(self, *, next_part=None, agent=False, gpu=None, warm=False): warm = False if not next_part or (hold is not None and hold <= 0): return self.release('finished' if not next_part else 'no_hold') + # The worker settles individual caches against its live RAM budget. + # A small host alone is no reason to discard everything it kept. reason = pressure(gpu, None) or self._tight() if reason: return self.release(reason) @@ -406,21 +409,12 @@ def finished(self, *, next_part=None, agent=False, gpu=None, warm=False): baseline = gpu_used(gpu) if warm and baseline is None: return self.release('gpu_unreadable') - threading.Thread(target=self._watch, args=(self.generation, self.idle_since, hold, gpu, baseline), + self.gpu, self.baseline = gpu, baseline + self.own = gpu_own(gpu) if warm else None + threading.Thread(target=self._watch, args=(self.generation, self.idle_since, hold, gpu), name='freevideo-resident-hold', daemon=True).start() return self.record(action='kept', reason=next_part, seconds=hold, pid=self.process.pid) - def release_idle(self, reason='requested'): - """Give the card and host memory back on a caller's request; never touches a running request.""" - with self.mutex: - if self.active: - return dict(released=False, reason='busy') - if self.process is None or self.process.poll() is not None: - return dict(released=False, reason='nothing_held') - row = self.release(reason) - return dict(released=True, reason=reason, **{k: v for k, v in (row or {}).items() - if k in ('idle_seconds', 'worker_rss_bytes')}) - def _tight(self): from .system import windows if windows(): @@ -436,11 +430,79 @@ def _tight(self): pass return None - def _watch(self, generation, since, hold, gpu, baseline): + def preloading(self): + return self.prewarm is not None and self.prewarm.thread is not None and self.prewarm.thread.is_alive() + + def warm_encoder(self, output, python, environment, *, busy=None, notify=None): + """Place the encoder on the card while the kept worker idles, under the worker's own lease. + + Only the models the worker can hold beside what it keeps are placed (it + sizes that from live free memory), and any new request, queued prompt or + memory pressure stops it first. + """ + with self.mutex: + if (self.process is None or self.process.poll() is not None or self.active or + environment.get('FREEVIDEO_ENCODER_PREWARM', 'auto').lower() in ('0', 'off', 'false')): + return False + self.stop_prewarm() + if self.process is None: + return False + from .idle_encoder import IdleEncoder + self.prewarm = IdleEncoder() + return self.prewarm.schedule(self.endpoint, output, python, environment, busy=busy, notify=notify) + + def release_idle(self, reason='requested'): + """Give the card and host memory back on a caller's request; never touches a running request.""" + with self.mutex: + if self.active: + return dict(released=False, reason='busy') + if self.process is None or self.process.poll() is not None: + return dict(released=False, reason='nothing_held') + row = self.release(reason) + return dict(released=True, reason=reason, **{k: v for k, v in (row or {}).items() + if k in ('idle_seconds', 'worker_rss_bytes')}) + + def placed_bytes(self): + """Card bytes the last idle preload accounts for: its reported growth, else its budget, plus the context.""" + try: + report = json.loads(Path(self.prewarm.report).read_text(encoding='utf-8')) + before, after = report.get('gpu_bytes_before'), report.get('gpu_bytes_after') + if type(before) is int and type(after) is int: + known = max(0, after - before) + else: + known = max(0, int(report.get('gpu_budget_bytes') or 0)) + except (AttributeError, OSError, TypeError, ValueError): + known = 0 + return known + PRELOAD_CONTEXT_BYTES + + def _watch(self, generation, since, hold, gpu): + placing = False while True: time.sleep(.5) - reason = ('idle' if hold is not None and time.monotonic() - since >= hold - else pressure(gpu, baseline)) + # Our own idle preload grows the card; only another program's growth counts, + # so the reference is read again once placement has ended. + unreadable = False + if self.preloading(): + placing = True + elif placing: + placing = False + used = gpu_used(gpu) + own = gpu_own(gpu) + if used is None: + unreadable = True # Same as at the start of the hold: never keep without a reading. + elif own is not None and self.own is not None and self.baseline is not None: + # Only the part this tree added moves the reference; another program's share stays visible. + self.baseline += max(0, own - self.own) + self.own = own + elif self.baseline is not None: + # No per-process bytes (WDDM): only what the preload reports it placed, plus its + # context, moves the reference; the rest of the growth is another program's. + self.baseline += min(max(0, used - self.baseline), self.placed_bytes()) + else: + self.baseline = used + reason = ('gpu_unreadable' if unreadable else + 'idle' if hold is not None and time.monotonic() - since >= hold + else pressure(gpu, None if placing else self.baseline)) with self.mutex: if generation != self.generation or self.active or self.process is None: return @@ -564,6 +626,23 @@ def gpu_used(gpu): return None +def gpu_own(gpu): + """Card bytes held by this process tree (the kept worker and its leases), or None if not listed.""" + try: + from .background_room import Gpu, family + device = Gpu(gpu) + try: + rows = device.processes() + finally: + device.close() + if rows is None or any(size is None for size in rows.values()): + return None + pids = family(os.getpid()) + return sum(size for pid, size in rows.items() if pid in pids) + except (OSError, RuntimeError, AttributeError, ValueError, ImportError): + return None + + def pressure(gpu, baseline): """Why a kept idle worker should go now, or None.""" try: diff --git a/freevideo_engine/resident_worker.py b/freevideo_engine/resident_worker.py index 14192674..c74521e9 100644 --- a/freevideo_engine/resident_worker.py +++ b/freevideo_engine/resident_worker.py @@ -68,6 +68,25 @@ def run(message): from .worker import generate bank.configure(request['gpu_budget_bytes'], request.get('ram_budget_bytes')) generate(request, resident=bank) + # Only a fully returned video is a success, including its + # final cleanup. Keep habit data inside this worker. + try: + bank.succeeded(request) + except Exception: + pass # Optional observations cannot fail a finished video. + try: + bank.settle(float(request['engine_options'].get('pin_host_gb') or 0)*1e9) + except Exception: + pass # Best-effort trimming cannot fail a finished video. + if request.get('metrics'): + try: + from .monitoring import save + path = Path(request['metrics']) + metrics = json.loads(path.read_text(encoding='utf-8')) + metrics.update(resident_models=bank.snapshot(), resident_admission=list(bank.decisions)) + save(path, metrics) + except (OSError, ValueError, TypeError): + pass else: from .encode_worker import encode bank.configure(request['gpu_budget_gb']*1e9, request.get('ram_budget_bytes')) diff --git a/freevideo_engine/usage_habit.py b/freevideo_engine/usage_habit.py new file mode 100644 index 00000000..334a7bed --- /dev/null +++ b/freevideo_engine/usage_habit.py @@ -0,0 +1,83 @@ +"""Recent usage habit kept across restarts: operation types and counts only. + +Nothing derived from a prompt, image or seed is stored. A transition between +two successive successful videos is one of three kinds of change, and the file +keeps the last three transitions (the four-request window) plus running counts. +A file that is missing, unreadable or malformed counts as no habit at all. +""" +import json +import os +from pathlib import Path + +WINDOW = 4 +KINDS = ('images', 'prompt', 'rerun') + + +def path(): + override = os.environ.get('FREEVIDEO_USAGE_HABIT') + if override: + return Path(override) + from .system import install_root + return install_root() / '.freevideo' / 'usage-habit.json' + + +def transition(previous, current): + """Kinds of change between two requests' digests (never stored).""" + kinds = [] + if previous['images'] != current['images']: + kinds.append('images') + if previous['prompt'] != current['prompt']: + kinds.append('prompt') + elif not kinds: + kinds.append('rerun') + return kinds + + +def transitions(history): + recent = history[-WINDOW:] + return [transition(a, b) for a, b in zip(recent, recent[1:])] + + +def dominant(recent): + """The kind that two thirds of the recent transitions share, or None. + + Image changes win ties with prompt changes, then same-prompt reruns. + """ + if not recent: + return None + for kind in KINDS: + if sum(kind in kinds for kinds in recent) * 3 >= len(recent) * 2: + return kind + return None + + +def load(file=None): + """The stored habit, or None when there is nothing trustworthy to read.""" + try: + value = json.loads(Path(file or path()).read_text(encoding='utf-8')) + if value['version'] != 1 or value['window'] != WINDOW: + return None + recent = value['recent'] + counts = value['counts'] + if (not isinstance(recent, list) or len(recent) > WINDOW - 1 + or any(not isinstance(kinds, list) or any(kind not in KINDS for kind in kinds) for kinds in recent) + or not isinstance(counts, dict) + or any(type(counts.get(name)) is not int or counts[name] < 0 for name in (*KINDS, 'requests'))): + return None + return dict(version=1, window=WINDOW, recent=[list(kinds) for kinds in recent], + counts={name: counts[name] for name in (*KINDS, 'requests')}) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + return None + + +def record(kinds, file=None): + """Append one transition (None for the first request after a restart) and save.""" + from .monitoring import save + value = load(file) or dict(version=1, window=WINDOW, recent=[], counts=dict.fromkeys((*KINDS, 'requests'), 0)) + value['counts']['requests'] += 1 + if kinds is not None: + value['recent'] = (value['recent'] + [list(kinds)])[-(WINDOW - 1):] + for kind in kinds: + value['counts'][kind] += 1 + save(Path(file or path()), value) + return value diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index 5a365d58..ea7fa7b4 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -763,12 +763,6 @@ def keep_audio_lead_in(value): resident.device_free, resident.device_reserve = None, 0 if live_budget is not None: live_budget.close() - if resident is not None and metrics['success'] and engine is not None: - try: - resident.settle(float(request['engine_options'].get('pin_host_gb') or 0)*1e9) - except Exception: - # Trimming is best effort; it must not turn a finished request into a failure. - pass if resident is not None: metrics['resident_admission'] = list(resident.decisions) from .adaptive import classify_failure From 2dccfc710e91de9a0f5c666bf9c0a7ada7793a86 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:17:59 +0000 Subject: [PATCH 2/8] Prefetch the first request's model files after engine start, as far as free memory allows --- __init__.py | 2 ++ freevideo_engine/comfy_nodes.py | 20 ++++++++++++ freevideo_engine/diagnostic_resources.py | 2 +- freevideo_engine/encoder_prewarm.py | 39 +++++++++++++++++++----- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/__init__.py b/__init__.py index 69298cab..886ebffe 100644 --- a/__init__.py +++ b/__init__.py @@ -61,4 +61,6 @@ async def comfy_entrypoint(): logging.warning('[FreeVideo] Agent routes not registered: %s', error) from .freevideo_engine.comfy_nodes import FreeVideoExtension, register_prompt_cleanup register_prompt_cleanup() + from .freevideo_engine.comfy_nodes import schedule_startup_prefetch + schedule_startup_prefetch() return FreeVideoExtension() diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index dcd442ed..5e259dc2 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -56,6 +56,26 @@ def register_prompt_cleanup(): server.add_on_prompt_handler(clean_prompt) +def schedule_startup_prefetch(delay=15.): + """Once after the engine starts, read what the first request reads into the OS file cache, as far as + free memory goes. A request, or the busy queue, cancels it through the same stop as the idle read.""" + from .encoder_prewarm import IDLE, startup_files + try: + from server import PromptServer + server = PromptServer.instance + root, machine = comfy_bridge.installation() + habit = usage_habit.load() + files = startup_files(machine, usage_habit.dominant(habit['recent'] if habit else None)) + files = [item for item in files if item.is_file()] + if not files: + return False + except (ImportError, OSError, RuntimeError, ValueError, KeyError, TypeError, AttributeError): + return False # Optional preparation cannot stop the engine from starting. + def busy(): + return server is not None and server.prompt_queue.get_tasks_remaining() > 0 + return IDLE.schedule(files, root / 'startup-prewarm.json', busy=busy, purpose='startup', delay=delay) + + def step_cache_option(value): """Bridge keyword for the Step cache widget; a workflow without it, with 'none' or with a value that is not an option adds nothing. Workflows saved before the widget existed hold '' in its position.""" diff --git a/freevideo_engine/diagnostic_resources.py b/freevideo_engine/diagnostic_resources.py index ea57cffa..1200afdc 100644 --- a/freevideo_engine/diagnostic_resources.py +++ b/freevideo_engine/diagnostic_resources.py @@ -143,7 +143,7 @@ def encoder_prewarm(value): for key in ('gpu_preload', 'worker_released', 'receipt_available'): if type(value.get(key)) is bool: result[key] = value[key] - if value.get('purpose') in ('encoder', 'upscale'): + if value.get('purpose') in ('encoder', 'upscale', 'startup'): result['purpose'] = value['purpose'] # Why idle preparation stopped, as fixed words only: the receipt's reason # is free text and may name a path. A bare "failed" explained nothing. diff --git a/freevideo_engine/encoder_prewarm.py b/freevideo_engine/encoder_prewarm.py index 153bdc15..a2af164c 100644 --- a/freevideo_engine/encoder_prewarm.py +++ b/freevideo_engine/encoder_prewarm.py @@ -18,15 +18,23 @@ CHUNK = 8 * 1024 * 1024 -def plan(size, available, *, ram_budget=None, complete=True): +MINIMUM_PARTIAL = 256 * 1024 * 1024 + + +def plan(size, available, *, ram_budget=None, complete=True, partial=False): + """Choose how much of the files to read. A sequential read of more than the cache holds evicts its own + earlier pages, so a partial read stops at the memory that is really free instead of exceeding it.""" reserve = 2 * GiB limit = max(0, available - reserve) if ram_budget is not None: limit = min(limit, max(0, int(ram_budget))) - return dict(enabled=bool(complete and size > 0 and size <= limit), bytes=size, + whole = bool(complete and size > 0 and size <= limit) + part = bool(partial and complete and size > limit >= MINIMUM_PARTIAL) + return dict(enabled=whole or part, partial=part, bytes=size, read_bytes=size if whole else (limit if part else 0), memory_allowance_bytes=limit, system_reserve_bytes=reserve, scope='Reclaimable OS file cache; no private model copy or GPU allocation', - reason=('Idle RAM covers the checkpoint plus system headroom' if complete and 0 < size <= limit + reason=('Idle RAM covers the checkpoint plus system headroom' if whole + else 'Idle RAM covers the first part of the files, read in the order of use' if part else 'Leave memory for active applications; encoder loads on demand')) @@ -45,6 +53,16 @@ def warm_target(report): return ('encoder', encoder) if encoder else None +def startup_files(machine, habit): + """Files the first request after an engine start reads first: a rerun reads the video model blocks, + any other habit, or none yet, starts with the text encoder.""" + if habit == 'rerun': + files = sorted(item for item in Path(machine['checkpoint']).rglob('*') if item.is_file()) + if files: + return files + return [Path(machine['encoder_model_root']) / 'text_encoders' / machine['encoder']] + + def available_memory(): memory = system_memory() group = cgroup_memory() @@ -67,12 +85,14 @@ def prefetch(path, selected, cancelled, *, busy=None, notify=None, availability= return result result['state'] = 'reading' buffer = bytearray(CHUNK) + view = memoryview(buffer) last = 0. + limit = selected.get('read_bytes', sum(item.stat().st_size for item in paths)) for item in paths: initial = item.stat() remaining = initial.st_size with item.open('rb', buffering=0) as stream: - while remaining > 0: + while remaining > 0 and result['done_bytes'] < limit: if cancelled.is_set() or busy and busy(): result.update(state='cancelled', reason='A new request takes priority') break @@ -80,7 +100,7 @@ def prefetch(path, selected, cancelled, *, busy=None, notify=None, availability= if not complete or available < selected['system_reserve_bytes'] + CHUNK: result.update(state='released', reason='Live RAM pressure; warmed pages remain reclaimable') break - count = stream.readinto(buffer) + count = stream.readinto(view[:min(CHUNK, limit - result['done_bytes'])]) if not count: raise ValueError('A model file changed during prefetch') remaining -= count @@ -92,6 +112,9 @@ def prefetch(path, selected, cancelled, *, busy=None, notify=None, availability= last = elapsed if result['state'] != 'reading': break + if remaining > 0: + result.update(state='partial', reason='The first part of the files is read; the rest loads on demand') + break current = item.stat() if (current.st_size, current.st_mtime_ns) != (initial.st_size, initial.st_mtime_ns): raise ValueError('A model file changed during prefetch') @@ -116,7 +139,7 @@ def stop(self): from .encoder_diagnostics import prewarm_receipt return prewarm_receipt(self.report, gpu=False) - def schedule(self, path, report, *, ram_budget=None, busy=None, notify=None, purpose='encoder'): + def schedule(self, path, report, *, ram_budget=None, busy=None, notify=None, purpose='encoder', delay=2.): """Warm the encoder checkpoint, or the files an upscale of a preview reads.""" paths = [Path(item) for item in path] if isinstance(path, (list, tuple)) else [Path(path)] with self.guard: @@ -130,11 +153,11 @@ def work(): result = {'state': 'cancelled', 'reason': 'A new request takes priority', 'purpose': purpose} try: # Return the completed video to the user before doing work. - if cancel.wait(2.) or busy and busy(): + if cancel.wait(delay) or busy and busy(): return available, complete = available_memory() selected = plan(sum(item.stat().st_size for item in paths), available, - ram_budget=ram_budget, complete=complete) + ram_budget=ram_budget, complete=complete, partial=purpose == 'startup') selected['purpose'] = purpose if not selected['enabled'] and purpose == 'upscale': selected['reason'] = 'Leave memory for active applications; the upscale reads its files on demand' From 15d6187b48fa5bb6e07abb970fcbad9ead07645e Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:18:08 +0000 Subject: [PATCH 3/8] Keep the image worker on the card longer for people who mostly make pictures --- freevideo_engine/comfy_nodes.py | 5 ++ freevideo_engine/comfy_residency.py | 12 +++++ freevideo_engine/image_gen/keep.py | 78 +++++++++++++++++++++++++++ freevideo_engine/image_gen/service.py | 9 +++- freevideo_engine/image_gen/worker.py | 46 ++++++++++++++-- freevideo_engine/usage_habit.py | 45 +++++++++++++--- 6 files changed, 184 insertions(+), 11 deletions(-) create mode 100644 freevideo_engine/image_gen/keep.py diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index 5e259dc2..93a12312 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -258,6 +258,11 @@ def execute(cls, text, width, height, seconds, seed, first=None, last=None, stop_music_for_generation(PromptServer.instance) except Exception as error: logging.warning('[FreeVideo] Could not stop music before the video: %s', error) + try: # the picture worker keeps the card longer only for people who mostly make pictures + from .image_gen.keep import note, path + note('video', path(comfy_bridge.installation_root())) + except Exception as error: + logging.warning('[FreeVideo] Could not record the video for the picture worker: %s', error) prewarm = {} result_reused = [False] result_report = [None] diff --git a/freevideo_engine/comfy_residency.py b/freevideo_engine/comfy_residency.py index aa329952..d9dff576 100644 --- a/freevideo_engine/comfy_residency.py +++ b/freevideo_engine/comfy_residency.py @@ -76,6 +76,16 @@ def release_native(server, *, timeout=10.): reserved_after_bytes=int(torch.cuda.memory_reserved(device))) +def release_image_worker(): + """Let an idle image worker give back the card; one that is making a picture is left alone.""" + try: + from server import PromptServer + from .image_gen.service import release_idle_worker + release_idle_worker(PromptServer.instance) + except Exception: + pass # Native models must load whether or not the image option is installed. + + def install(memory): if getattr(memory.free_memory, '_freevideo_resident_cooperation', False) is True: return @@ -90,6 +100,8 @@ def free_memory(memory_required, device, *args, **kwargs): OWNER.process is not None and OWNER.process.poll() is None): if device is None or memory_required > memory.get_free_memory(device): OWNER.release('comfy_model') + if device is None or memory_required > memory.get_free_memory(device): + release_image_worker() return original(memory_required, device, *args, **kwargs) free_memory._freevideo_resident_cooperation = True diff --git a/freevideo_engine/image_gen/keep.py b/freevideo_engine/image_gen/keep.py new file mode 100644 index 00000000..a00325bc --- /dev/null +++ b/freevideo_engine/image_gen/keep.py @@ -0,0 +1,78 @@ +"""How long an idle image worker keeps the model on the card. + +A worker that made a picture waits IDLE_SECONDS for the next one. For someone who mostly makes +pictures it waits KEEP_SECONDS, but only while the whole model sits on the card, and it lets go at +once when another program starts taking card memory. Video always takes the card first. +""" + +from .. import usage_habit + +IDLE_SECONDS = 45. +KEEP_SECONDS = 600. +# Operations looked at (two thirds of them must be pictures), and how few are enough to judge at all. +WINDOW = 6 +MINIMUM = 3 +STORED = usage_habit.OPS_STORED +# Card memory other programs may take while the worker waits before it lets go. +YIELD_BYTES = 1 << 30 +POLL_SECONDS = .5 + + +def path(root=None): + """Where the habit is kept: the usage habit file under the installation directory.""" + return usage_habit.path(root) + + +def load(file=None): + """The last operations ('image' or 'video'), oldest first; empty when the file cannot be trusted.""" + return usage_habit.ops(file) + + +def note(kind, file=None): + """Remember one operation. Only the kind is stored, nothing from the request.""" + usage_habit.note_op(kind, file) + + +def mostly_images(file=None): + """True when most of the recent operations were pictures and the last one was too.""" + recent = load(file)[-WINDOW:] + return len(recent) >= MINIMUM and recent[-1] == 'image' and recent.count('image') * 3 >= len(recent) * 2 + + +def keep_seconds(file=None): + """The idle limit the worker is asked to use for the request being sent.""" + return KEEP_SECONDS if mostly_images(file) else IDLE_SECONDS + + +def card_free_bytes(device): + """Free memory of the card the worker's model is on, or None when it cannot be read.""" + torch = __import__('sys').modules.get('torch') + if torch is None or getattr(device, 'type', str(device).split(':')[0]) != 'cuda': + return None + try: + return int(torch.cuda.mem_get_info(device)[0]) + except Exception: + return None + + +class Crowding: + """True once other programs have taken YIELD_BYTES of card memory since the wait began. + + The worker's own memory does not change while it waits, so a drop in free memory is theirs.""" + + def __init__(self, read, limit=YIELD_BYTES): + self.read = read + self.limit = limit + self.start = read() + + def __bool__(self): + now = self.read() + return self.start is not None and now is not None and self.start - now >= self.limit + + +def wait_seconds(requested, runner, idle=IDLE_SECONDS): + """Idle limit for the worker after a picture: the longer one only for a model whole on a readable card.""" + if (type(requested) in (int, float) and requested > idle and getattr(runner, 'full_card', False) is True + and card_free_bytes(getattr(runner, 'device', None)) is not None): + return min(float(requested), KEEP_SECONDS) + return idle diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index 3255b9fc..6a71abb0 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -16,7 +16,7 @@ import time import uuid -from . import image_pack +from . import image_pack, keep from ..download_settings import observed_speed from .failure import (Context, HOST_MESSAGE_ZH, host_memory_shortage, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, UNIFIED_MESSAGE_ZH, classify_exception) @@ -135,6 +135,7 @@ def __init__(self, installation, input_directory, *, video_busy=None, idle_secon self.worker_restarts = 0 self.worker_started = False self.queued = None + self.habit_root = None self.queue_thread = None self.queue_wake = threading.Event() self.finished = OrderedDict() @@ -882,6 +883,7 @@ def spawn(self, root, machine, env, descriptor): def work(self, root, machine): from ..locking import runtime_lock from .. import processes, resident_process + self.habit_root = root session = None failure = 'worker_exit' failure_error = None @@ -1082,6 +1084,11 @@ def worker_gone_error(session, heard): def infer(self, session, value): context = self.context or Context('generate', value) heard = False + habit = keep.path(self.habit_root) + if 'keep_seconds' not in value: + keep.note('image', habit) + # How long the worker may wait for the next picture; it applies that only with the whole model on the card. + value['keep_seconds'] = keep.keep_seconds(habit) try: session.send(action='generate', **value) except OSError: diff --git a/freevideo_engine/image_gen/worker.py b/freevideo_engine/image_gen/worker.py index f9c565a3..0b549e3f 100644 --- a/freevideo_engine/image_gen/worker.py +++ b/freevideo_engine/image_gen/worker.py @@ -9,6 +9,7 @@ import threading import time +from . import keep from .failure import Context, classify_exception, exception_details, host_memory_shortage, safe_plan @@ -71,6 +72,35 @@ def _flatten(image): return image.convert('RGB') +def next_request(inbox, wait, crowding=None): + """The next row, or None when the pipe closed, `wait` seconds passed, or the card is crowded. + + Without `crowding` this is one blocking read, as for an ordinary idle wait.""" + if crowding is None: + try: + return inbox.rows.get(timeout=wait) + except queue.Empty: + return None + deadline = time.monotonic() + wait + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + try: + return inbox.rows.get(timeout=min(remaining, keep.POLL_SECONDS)) + except queue.Empty: + if crowding: + return None + + +def hold(runner): + """Start the long idle wait: hand back the allocator's cached memory, then watch the card.""" + torch = sys.modules.get('torch') + if torch is not None: + torch.cuda.empty_cache() + return keep.Crowding(lambda: keep.card_free_bytes(runner.device)) + + def load_runner(pack_dir, device, *, progress=None): # Importing this module in ComfyUI or in CPU protocol tests never loads torch. from .image_runner import ImageRunner @@ -92,12 +122,10 @@ def send(**row): runner = None back_detector = None back_detector_loaded = False + wait, crowding = idle_seconds, None try: while not inbox.closed.is_set(): - try: - value = inbox.rows.get(timeout=idle_seconds) - except queue.Empty: - break + value = next_request(inbox, wait, crowding) if value is None or inbox.closed.is_set(): break started = time.monotonic() @@ -195,6 +223,16 @@ def generate(seed): inbox.busy = False if result is not None: send(**result) + wait, crowding = idle_seconds, None + if result is not None and result.get('phase') == 'result' and runner is not None: + wait = keep.wait_seconds(value.get('keep_seconds'), runner, idle_seconds) + if wait > idle_seconds: + try: + crowding = hold(runner) + except Exception as error: + # The picture is already sent; a hold that cannot start is the normal short wait. + print('image worker hold failed: %s' % type(error).__name__, file=sys.stderr, flush=True) + wait = idle_seconds finally: inbox.closed.set() inbox.cancel.set() diff --git a/freevideo_engine/usage_habit.py b/freevideo_engine/usage_habit.py index 334a7bed..b7f1154b 100644 --- a/freevideo_engine/usage_habit.py +++ b/freevideo_engine/usage_habit.py @@ -4,6 +4,9 @@ two successive successful videos is one of three kinds of change, and the file keeps the last three transitions (the four-request window) plus running counts. A file that is missing, unreadable or malformed counts as no habit at all. + +The same file keeps the last OPS_STORED operations ('image' or 'video') under "ops", which the image +worker reads to decide how long to stay on the card. Reading and saving the habit leave them as they are. """ import json import os @@ -11,14 +14,18 @@ WINDOW = 4 KINDS = ('images', 'prompt', 'rerun') +OPS = ('image', 'video') +OPS_STORED = 8 -def path(): +def path(root=None): override = os.environ.get('FREEVIDEO_USAGE_HABIT') if override: return Path(override) - from .system import install_root - return install_root() / '.freevideo' / 'usage-habit.json' + if root is None: + from .system import install_root + root = install_root() + return Path(root) / '.freevideo' / 'usage-habit.json' def transition(previous, current): @@ -64,20 +71,46 @@ def load(file=None): or not isinstance(counts, dict) or any(type(counts.get(name)) is not int or counts[name] < 0 for name in (*KINDS, 'requests'))): return None + ops = value.get('ops', []) + if not isinstance(ops, list) or len(ops) > OPS_STORED or any(kind not in OPS for kind in ops): + ops = [] return dict(version=1, window=WINDOW, recent=[list(kinds) for kinds in recent], - counts={name: counts[name] for name in (*KINDS, 'requests')}) + counts={name: counts[name] for name in (*KINDS, 'requests')}, ops=list(ops)) except (OSError, ValueError, KeyError, TypeError, AttributeError): return None +def empty(): + return dict(version=1, window=WINDOW, recent=[], counts=dict.fromkeys((*KINDS, 'requests'), 0), ops=[]) + + def record(kinds, file=None): """Append one transition (None for the first request after a restart) and save.""" from .monitoring import save - value = load(file) or dict(version=1, window=WINDOW, recent=[], counts=dict.fromkeys((*KINDS, 'requests'), 0)) + value = load(file) or empty() value['counts']['requests'] += 1 if kinds is not None: value['recent'] = (value['recent'] + [list(kinds)])[-(WINDOW - 1):] for kind in kinds: value['counts'][kind] += 1 - save(Path(file or path()), value) + save(Path(file or path()), value if value['ops'] else {name: item for name, item in value.items() if name != 'ops'}) return value + + +def ops(file=None): + """The last operations ('image' or 'video'), oldest first; empty when nothing can be trusted.""" + value = load(file) + return value['ops'] if value else [] + + +def note_op(kind, file=None): + """Remember one operation next to the habit. Only the kind is stored, nothing from the request.""" + if kind not in OPS: + raise ValueError(kind) + from .monitoring import save + value = load(file) or empty() + value['ops'] = (value['ops'] + [kind])[-OPS_STORED:] + try: + save(Path(file or path()), value) + except OSError: + pass # A habit that cannot be saved is only forgotten at restart. From 815e20be6985f39d6b9ca78f4937904c667bfcbf Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:18:11 +0000 Subject: [PATCH 4/8] Repair replaces pinned Python packages whose files are missing, and a failed hardware probe... Repair replaces pinned Python packages whose files are missing, and a failed hardware probe names them --- freevideo_engine/bootstrap.py | 45 ++++++++++++-- freevideo_engine/failure_details.py | 4 +- freevideo_engine/runtime_libraries.py | 88 +++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index f477ceea..c238093f 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -158,7 +158,7 @@ def reinstall_download_bytes(dependencies, system, *, root=None, frontend_python Windows wheel (torch_download), elsewhere the package's installed size, an upper bound for its wheel. Marks each engine environment's damaged packages.""" from .environments import ENVIRONMENTS - from .runtime_libraries import damaged, incomplete, installed_bytes + from .runtime_libraries import damaged, incomplete, installed_bytes, missing_packages from .torch_download import select total = 0 seen = set() @@ -173,6 +173,9 @@ def reinstall_download_bytes(dependencies, system, *, root=None, frontend_python row['damaged'] = broken = damaged(row['python']) if frontend: broken = sorted(set(broken + incomplete(row['python']))) + else: + # Setup puts back pinned packages whose files are gone (Setup.repairing). + broken = sorted(set(broken + missing_packages(row['python'], constraints_file(name, system)))) pins = ENVIRONMENTS.get(name, {}) wheels = {w['package']: w for w in (select(pins['torch'], pins['cuda']) or [])} if system == 'Windows' and pins else {} for package in broken: @@ -1628,10 +1631,11 @@ def check(): if install(uv, executable, role['torch'], role['cuda'], root=self.root, networking=self.network, env=self.env, ui=self.ui, run=lambda args, env: self.command(label + '-torch', args, env=env), - constraints=['-c', constraints_file(label, self.system)], check=check): + constraints=['-c', constraints_file(label, self.system)], check=check, + reinstall=self.missing_files(executable, label, gpu=True, wheels=role)): return from .runtime_libraries import damaged, reinstall_arguments - fresh = self.check_runtime_libraries(executable, label) + fresh = sorted(set(self.check_runtime_libraries(executable, label)) | set(self.missing_files(executable, label, gpu=True))) result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(fresh), *argv[5:]], group) # Files linked from a changed uv cache arrive with a new environment. if not fresh: @@ -1647,7 +1651,8 @@ def check(): '-c', constraints_file(name, self.system), 'pip', 'ninja', 'packaging', 'wheel', 'setuptools'] if self.layout == 'unified': command += ['-r', SOURCE / 'constraints/encoder-runtime.txt'] - task('runtime', lambda: self.packages(name + '-packages', command), (name + '-torch',), ('environment:' + name,)) + task('runtime', lambda: self.packages(name + '-packages', self.repairing(command, python, name)), + (name + '-torch',), ('environment:' + name,)) # Enqueue downloads as soon as their SDKs exist, before waiting for # source setup, encoder checks or acceleration compilation. task('models', lambda: self.command('models', [python, '-m', 'freevideo_engine.provision', @@ -1657,8 +1662,9 @@ def check(): encoder_after = 'runtime' if self.layout == 'dual': def encoder_packages(): - self.packages('encoder-packages', [uv, 'pip', 'install', '--python', encoder_python, - '-r', SOURCE / 'constraints/encoder-runtime.txt', '-c', constraints_file('encoder', self.system)]) + self.packages('encoder-packages', self.repairing([uv, 'pip', 'install', '--python', encoder_python, + '-r', SOURCE / 'constraints/encoder-runtime.txt', '-c', constraints_file('encoder', self.system)], + encoder_python, 'encoder')) self.packages('encoder-engine', [uv, 'pip', 'install', '--python', encoder_python, '--no-deps', '-e', SOURCE]) task('encoder-runtime', encoder_packages, ('encoder-torch',), ('environment:encoder',)) encoder_after = 'encoder-runtime' @@ -1690,6 +1696,33 @@ def encoder_packages(): tasks['models']['after'] = ('package-cache',) return tasks, pythons, comfy + def missing_files(self, python, role, *, gpu=None, wheels=None): + """Pinned packages of an environment whose files are gone (runtime_libraries.package_files). + + gpu=True keeps the PyTorch family, which is installed from the GPU wheels (and only + those of `wheels`, an environment's spec, when given); gpu=False keeps the rest.""" + from .runtime_libraries import gpu_family, missing_packages, normalize_name + try: + names = missing_packages(python, constraints_file(role, self.system)) + except ValueError: + return [] + if gpu is not None: + names = [name for name in names if gpu_family(name) == gpu] + if wheels is not None: + from .torch_download import select + catalog = {normalize_name(row['package']) for row in select(wheels['torch'], wheels['cuda']) or []} + names = [name for name in names if normalize_name(name) in catalog] + return names + + def repairing(self, argv, python, role): + """An install command that also replaces the pinned packages whose files are gone. + + Installing the pinned version again does nothing when uv finds it installed, + whatever state its files are in. The PyTorch family is replaced in its own step.""" + from .runtime_libraries import reinstall_arguments + names = self.missing_files(python, role, gpu=False) + return [*argv[:5], *reinstall_arguments(names), *argv[5:]] if names else argv + def check_runtime_libraries(self, python, role): from .runtime_libraries import damaged key = role + '-runtime-libraries' diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 7d8ec3d7..af9d23c8 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -756,7 +756,9 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re ('Runtime files have been changed', '运行组件已被更改'), ('Click Repair to reinstall a complete copy of PyTorch. Models and videos are kept.', '请点“修复”，FreeVideo 会重新安装运行组件，模型和视频都会保留。')), - (r'ModuleNotFoundError|No module named|DLL load failed', 'dependencies', *DEPENDENCIES), + # runtime_libraries.summary names packages whose files are gone (a numpy extension that cannot load). + (r'ModuleNotFoundError|No module named|DLL load failed|Files of these Python packages are missing or have changed', + 'dependencies', *DEPENDENCIES), (r'CUDA (?:error: )?out of memory|torch\.OutOfMemoryError|Insufficient currently available memory|crossed its RAM budget|paging file is too small|WinError 1455|os error 1455', 'memory', ('Memory is unavailable for this step', '这一步可用内存不足'), ('Close other memory-heavy applications and retry. Copy the details if it continues.', '关闭占用内存较多的程序后重试；仍失败时可复制详情反馈。')), diff --git a/freevideo_engine/runtime_libraries.py b/freevideo_engine/runtime_libraries.py index 2fc9e804..bb3d43a6 100644 --- a/freevideo_engine/runtime_libraries.py +++ b/freevideo_engine/runtime_libraries.py @@ -167,6 +167,10 @@ def diagnose(site=None): result.update(found) except Exception as error: result['check_error'] = '%s: %s' % (type(error).__name__, error) + try: + result['missing_files'] = {name: files for name, files in list(package_files(site).items())[:6]} + except Exception as error: + result['missing_files_error'] = '%s: %s' % (type(error).__name__, error) try: for path in loaded(): inside = _canonical(os.path.dirname(path)) in directories @@ -187,6 +191,13 @@ def summary(result, repair=REINSTALL): if changed: lines.append("Some of FreeVideo's PyTorch library files do not match the installed version (%s). %s" % (', '.join(changed[:4]) + (', …' if len(changed) > 4 else ''), repair)) + missing = result.get('missing_files') + if isinstance(missing, dict) and missing: + names = ['%s (%s)' % (name, ', '.join(str(file).rsplit('/', 1)[-1] for file in files[:2])) + for name, files in list(missing.items())[:4] if isinstance(files, list) and files] + if names: + lines.append('Files of these Python packages are missing or have changed: %s%s. %s' + % (', '.join(names), ', …' if len(missing) > 4 else '', repair)) outside = sorted({str(Path(p).parent) for p in result.get('outside', [])}) if outside: lines.append('cuDNN was also loaded from outside FreeVideo: %s.' % '; '.join(outside[:3])) @@ -363,6 +374,83 @@ def incomplete(python): return [] +def package_files(site, limit=3): + """Files RECORD lists inside site-packages that are missing or have another size. + + Returns {distribution: [paths relative to site-packages, at most `limit`]}. Only + files below site-packages count: console scripts and data files sit beside it, and + their size depends on where the environment was created. A numpy extension that + cannot load is usually one of these files gone (quarantined, cleaned up, cut short), + which an ordinary install does not notice. No hashing, so it is cheap to run. + Unreadable records and checks that cannot run find nothing. + """ + found = {} + try: + site = Path(site) + # Paths stay as RECORD spells them: a package folder that is a junction + # to another drive is still checked. + base = os.path.normcase(os.path.normpath(str(site))) + os.sep + for distribution in importlib.metadata.distributions(path=[str(io_path(site))]): + try: + rows = list(csv.reader(io.StringIO(distribution.read_text('RECORD') or ''))) + name = distribution.metadata['Name'] + except (OSError, csv.Error, UnicodeDecodeError, KeyError): + continue + if not name: + continue + for row in rows: + if len(row) < 3 or not row[2].isdigit(): + continue + path = os.path.normpath(os.path.join(str(site), row[0])) + if not os.path.normcase(path).startswith(base): + continue + try: + size = os.path.getsize(io_path(path)) + except (FileNotFoundError, NotADirectoryError): + size = None + except OSError: + continue + if size != int(row[2]): + files = found.setdefault(name, []) + if len(files) >= limit: + break + files.append(row[0].replace('\\', '/')) + except Exception: + return {} + return found + + +def gpu_family(name): + """The packages installed from the GPU wheels: PyTorch, its CUDA libraries and its compiler.""" + name = normalize_name(name) + return name in ('torch', 'torchvision', 'torchaudio', 'triton', 'triton-windows') or name.startswith('nvidia-') + + +def pinned_names(constraints): + """Normalized names a constraints file pins, or None when it cannot be read.""" + rows = _requirement_rows(constraints) + if rows is None: + return None + return {normalize_name(match[0]) for match in (re.match(r'[A-Za-z0-9][A-Za-z0-9._-]*', row) for row in rows) if match} + + +def missing_packages(python, constraints): + """Pinned distributions of this environment whose RECORD files are missing or cut short. + + Setup and repair reinstall these: uv treats a package at the pinned version + as installed whatever its files look like. Packages the constraints do not + pin (editable installs, kernels built on this machine) are left alone. + """ + try: + site = site_for(python) + pinned = pinned_names(constraints) + if site is None or pinned is None: + return [] + return sorted(name for name in package_files(site, limit=1) if normalize_name(name) in pinned) + except Exception: + return [] + + def installed_bytes(python, name): """Installed size of one distribution in this Python's environment, from its RECORD.""" try: From 148c2bae6feec46e24f3ee3ebb307141c007a989 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:18:15 +0000 Subject: [PATCH 5/8] Replace the Windows worker after a device allocation the driver refused with VRAM to spare --- freevideo_engine/adaptive.py | 31 +++++++++++++++++++++++++++++ freevideo_engine/resident_worker.py | 8 +++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py index e5355d6e..8a371cd7 100644 --- a/freevideo_engine/adaptive.py +++ b/freevideo_engine/adaptive.py @@ -41,6 +41,37 @@ def commit_exhausted(error): return found(error) +def commit_limited_allocation(error, failure): + """A driver-level CUDA allocation failure on Windows with VRAM to spare. + + WDDM charges device allocations to the system commit limit. When that limit + is spent the driver refuses to map more device memory although the card has + room: the error is the bare "CUDA error: out of memory" of the runtime, not + the caching allocator's "CUDA out of memory. Tried to allocate", and the + device is not near its budget. A worker that keeps serving afterwards + still holds the commit it charged (about 6 GiB private commit after a + cleaned-up failure, against 0.03 GiB in a new process) and its device + mappings are inconsistent: the retry in it failed with "resource already + mapped" and the worker could not be stopped. WDDM non-local exhaustion is a + separate case with its own recovery (``nonlocal_exhausted``). + """ + from .system import windows + if not windows() or failure.get('kind') != 'gpu_oom' or nonlocal_exhausted(failure): + return False + gpu = failure.get('gpu') if isinstance(failure.get('gpu'), dict) else {} + free = gpu.get('free_bytes') + if type(free) is not int or free < 2 * 2**30: + return False + seen = set() + while error is not None and id(error) not in seen: + seen.add(id(error)) + text = str(error).lower() + if 'cuda error: out of memory' in text and 'cuda out of memory' not in text: + return True + error = error.__cause__ or error.__context__ + return False + + # A CUDA error that may leave the device itself unusable; never retried. DEVICE_STATE_ERRORS = ('cuda error: unknown error', 'device is lost', 'device lost', 'gpu is lost', 'illegal memory access', 'device-side assert', 'unspecified launch failure', 'driver shutting down', diff --git a/freevideo_engine/resident_worker.py b/freevideo_engine/resident_worker.py index c74521e9..bd1b01a0 100644 --- a/freevideo_engine/resident_worker.py +++ b/freevideo_engine/resident_worker.py @@ -98,7 +98,7 @@ def run(message): if diagnostics is not None: diagnostics.finish(False) traceback.print_exc() - from .adaptive import classify_failure, commit_exhausted, cuda_runtime_error + from .adaptive import classify_failure, commit_exhausted, commit_limited_allocation, cuda_runtime_error failure = classify_failure(error) if request.get('metrics'): from .monitoring import save @@ -138,6 +138,12 @@ def run(message): if commit_exhausted(error): log.flush() os._exit(75) + # The same holds when the driver refuses a device allocation + # for lack of commit while the card still has free memory: the + # process keeps its charge and its mappings are unreliable. + if commit_limited_allocation(error, failure): + log.flush() + os._exit(75) failed_error = error # ModelBank cannot own tensors held by an exception's traceback. From 184982f2f812d4d21fa9dbf24b9124c34834a815 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sun, 1 Nov 2026 09:18:41 +0000 Subject: [PATCH 6/8] Tutorial book: chapters for the 0.4.25 and 0.4.26 additions, a replace-person chapter and tour... Tutorial book: chapters for the 0.4.25 and 0.4.26 additions, a replace-person chapter and tour card, Chinese line breaks --- MANIFEST.in | 4 + ...77\347\224\250\350\257\264\346\230\216.md" | 25 ++++++- freevideo_source_bootstrap.py | 4 + web/assets/tour/c10-person.webp | Bin 0 -> 3522 bytes web/assets/tour/c10-s1.mp4 | Bin 0 -> 562659 bytes web/assets/tour/c10-s1.webp | Bin 0 -> 11264 bytes web/assets/tour/c10-source.webp | Bin 0 -> 8178 bytes web/feature_tour_content.js | 2 + web/tutorial/replace-person-1.webp | Bin 0 -> 9454 bytes web/tutorial/replace-person-2.webp | Bin 0 -> 10034 bytes web/tutorial_book_content.js | 70 +++++++++++------- 11 files changed, 76 insertions(+), 29 deletions(-) create mode 100644 web/assets/tour/c10-person.webp create mode 100644 web/assets/tour/c10-s1.mp4 create mode 100644 web/assets/tour/c10-s1.webp create mode 100644 web/assets/tour/c10-source.webp create mode 100644 web/tutorial/replace-person-1.webp create mode 100644 web/tutorial/replace-person-2.webp diff --git a/MANIFEST.in b/MANIFEST.in index 35f8afd0..c430037e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -65,4 +65,8 @@ include web/assets/tour/c8-s1.mp4 include web/assets/tour/c8-s1.webp include web/assets/tour/c9-s1.mp4 include web/assets/tour/c9-s1.webp +include web/assets/tour/c10-person.webp +include web/assets/tour/c10-s1.mp4 +include web/assets/tour/c10-s1.webp +include web/assets/tour/c10-source.webp include web/assets/tour/fox-sheet.webp diff --git "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 53e0c03e..76a1d8bc 100644 --- "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -1,18 +1,28 @@ # 使用说明 -这一页说明创作面板里三件常被问到的事：用参考图生成视频、首帧尾帧和参考图的区别、做更长的视频。 +这一页说明创作面板里常用的操作：用参考图生成视频、替换人物、区分首帧尾帧和参考图、筛选作品，以及做更长的视频。 ## 参考图生成视频 1. 打开创作面板，展开“素材”。 2. 点“上传文件”，选择图片、视频或音频。上传的每张图片可以指定用途：首帧、尾帧或参考。要让画面里出现这张图里的人物、物品或场景，选“参考”。 3. 在描述中输入 `@`，从列表里选一张素材，描述里会出现 `` 这样的标记。视频和音频素材对应 `