From 33abd53a0cfedb6aa913d61fdc7a8e5ea4103222 Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 13 Aug 2026 15:04:57 +0500 Subject: [PATCH 1/5] feat(videoprism): add temporal video search capability --- INSTALLATION_GUIDE.md | 5 +- README.md | 12 +- desktop/model-cache-catalog.json | 5 + desktop/src/components/TargetSummary.tsx | 1 + docs/desktop.md | 14 +- pyproject.toml | 10 + src/vidxp/capabilities/registry.py | 3 +- src/vidxp/capabilities/videoprism/__init__.py | 1 + src/vidxp/capabilities/videoprism/config.py | 15 ++ .../capabilities/videoprism/definition.py | 95 ++++++++ src/vidxp/capabilities/videoprism/indexing.py | 216 ++++++++++++++++++ src/vidxp/capabilities/videoprism/models.py | 61 +++++ .../capabilities/videoprism/operations.py | 94 ++++++++ .../capabilities/videoprism/requirements.txt | 6 + src/vidxp/capabilities/videoprism/specs.py | 16 ++ src/vidxp/frontend.py | 4 +- src/vidxp/local_probe.py | 4 +- src/vidxp/runtime.py | 1 + tests/test_capabilities.py | 14 +- tests/test_frontend.py | 2 +- tests/test_local_probe.py | 7 +- tests/test_mcp.py | 2 +- tests/test_models.py | 6 +- tests/test_videoprism.py | 81 +++++++ uv.lock | 127 ++++++++-- 25 files changed, 761 insertions(+), 41 deletions(-) create mode 100644 src/vidxp/capabilities/videoprism/__init__.py create mode 100644 src/vidxp/capabilities/videoprism/config.py create mode 100644 src/vidxp/capabilities/videoprism/definition.py create mode 100644 src/vidxp/capabilities/videoprism/indexing.py create mode 100644 src/vidxp/capabilities/videoprism/models.py create mode 100644 src/vidxp/capabilities/videoprism/operations.py create mode 100644 src/vidxp/capabilities/videoprism/requirements.txt create mode 100644 src/vidxp/capabilities/videoprism/specs.py create mode 100644 tests/test_videoprism.py diff --git a/INSTALLATION_GUIDE.md b/INSTALLATION_GUIDE.md index 3b0243a..38e35b2 100644 --- a/INSTALLATION_GUIDE.md +++ b/INSTALLATION_GUIDE.md @@ -14,7 +14,7 @@ shape needs. | Desktop app | Install the native package | Adopt a compatible local installation or create a private Desktop-managed runtime | | Browser UI in Docker | Published `vidxp` image | One CPU worker/UI container | | Public/self-hosted service | `compose.coolify.yaml` | API/MCP control plane, CPU worker, PostgreSQL, Chroma, tusd | -| Embed one capability | `dialogue`, `scene`, or `actor` extra | Python indexing/retrieval code | +| Embed one capability | `dialogue`, `scene`, `actor`, or `videoprism` extra | Python indexing/retrieval code | Do not install the bare package and expect it to index video. Base `vidxp` provides the lightweight command shell, configuration, and typed contracts. @@ -227,7 +227,8 @@ Extras are composable: | `dialogue` | Storage, transcription, dialogue embeddings | Scene/actor providers | | `scene` | Storage, PyTorch, Transformers, OpenCV, Pillow | Dialogue/actor providers | | `actor` | Storage, OpenCV, YuNet/SFace support | Dialogue/scene providers | -| `all` | Dialogue, scene, and actor | Grounded-query model client and UI | +| `videoprism` | Storage, VideoPrism, PyTorch, Transformers, Torchvision | Other model providers | +| `all` | Every built-in search capability | Grounded-query model client and UI | | `local-worker` | `all` plus grounded-query client | Browser UI, MCP SDK, HTTP server | | `frontend` | Streamlit | Worker providers | | `mcp` | MCP SDK | Worker providers | diff --git a/README.md b/README.md index 609d6f5..3d767ce 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@

- Dialogue search · Scene search · Actor grouping + Dialogue search · Scene search · Temporal video search · Actor grouping

@@ -43,9 +43,13 @@ VidXP makes one video—or an entire collection—searchable by meaning: matching moments. - **Scene search:** describe what appeared on screen and find the closest visual matches. +- **VideoPrism search:** describe an action or event spanning a short clip. - **Actor matching:** find recurring faces within a video and export a highlighted video for a selected group. +Existing indexes remain compatible. Index media with the `videoprism` +capability selected before using temporal video search. + Use it to search years of family videos, add video search to an editing workflow, or let an AI agent answer questions using evidence from your own video library. Your videos can stay on your machine. @@ -138,6 +142,9 @@ vidxp index create # Find a visual moment vidxp search scene "a yellow taxi on a city street" +# Find an action spanning several frames +vidxp search videoprism "a person opens a door and walks outside" + # Find something that was said vidxp search dialogue "the bread just came out of the oven" ``` @@ -197,10 +204,11 @@ approximately 3 GiB. |---|---:| | Dialogue search | 2.64 GiB | | Scene search | 1.43 GiB | +| VideoPrism temporal search | 0.93 GiB | | Actor matching | 37 MiB | A full local Desktop setup with every search capability uses approximately -7.1 GiB. Leave additional temporary space during installation and for indexes, +8.1 GiB. Leave additional temporary space during installation and for indexes, source videos, and exported results. By default, the CLI and desktop app share the same VidXP data directory: diff --git a/desktop/model-cache-catalog.json b/desktop/model-cache-catalog.json index 9bdf1f6..8284bad 100644 --- a/desktop/model-cache-catalog.json +++ b/desktop/model-cache-catalog.json @@ -9,6 +9,11 @@ "label": "google/siglip2-base-patch16-224", "relative_artifact": "models--google--siglip2-base-patch16-224/snapshots/75de2d55ec2d0b4efc50b3e9ad70dba96a7b2fa2/model.safetensors" }, + { + "id": "google/videoprism-lvt-base-f16r288", + "label": "google/videoprism-lvt-base-f16r288", + "relative_artifact": "models--google--videoprism-lvt-base-f16r288/snapshots/fb6de9f0eb7bc285be86bdca1cf7daa3e3ef51ff/model.safetensors" + }, { "id": "Qwen/Qwen3-Embedding-0.6B", "label": "Qwen/Qwen3-Embedding-0.6B", diff --git a/desktop/src/components/TargetSummary.tsx b/desktop/src/components/TargetSummary.tsx index bfb8520..06cee6f 100644 --- a/desktop/src/components/TargetSummary.tsx +++ b/desktop/src/components/TargetSummary.tsx @@ -48,6 +48,7 @@ const CAPABILITY_LABELS: Record = { dialogue: 'Dialogue search', media: 'Video tools', scene: 'Visual scene search', + videoprism: 'Temporal video search', }; interface WorkerFailure { diff --git a/docs/desktop.md b/docs/desktop.md index 2253813..7ed7311 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -135,13 +135,13 @@ Target profiles use a versioned desktop-private schema. Profile content and the selected profile identity are stored separately. No credentials or remote tokens are stored; remote targets are intentionally outside this release. -Users select dialogue, scene, and actor search features independently. Product -choices map to package extras as follows: **Local video processing** adds -`local-worker` and includes all built-in search features, **Browser interface** -adds `frontend`, **AI assistant integration** adds the stdio `mcp` transport, -and **App integration service** adds the loopback API plus Streamable HTTP MCP -through `server`. These package names stay out of the normal product flow. Model preparation -can be deferred, and a native folder picker +Users select dialogue, scene, VideoPrism temporal search, and actor features +independently. Product choices map to package extras as follows: **Local video +processing** adds `local-worker` and includes all built-in search features, +**Browser interface** adds `frontend`, **AI assistant integration** adds the +stdio `mcp` transport, and **App integration service** adds the loopback API +plus Streamable HTTP MCP through `server`. These package names stay out of the +normal product flow. Model preparation can be deferred, and a native folder picker can select a model-cache directory before any model is downloaded. The managed runtime acquires the exact VidXP package from the wheel embedded in the Desktop installer with dependency resolution disabled, then resolves that diff --git a/pyproject.toml b/pyproject.toml index baeb6b7..b93c2df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,11 +95,16 @@ actor = { file = [ "src/vidxp/requirements/storage.txt", "src/vidxp/capabilities/actor/requirements.txt", ] } +videoprism = { file = [ + "src/vidxp/requirements/storage.txt", + "src/vidxp/capabilities/videoprism/requirements.txt", +] } all = { file = [ "src/vidxp/requirements/storage.txt", "src/vidxp/capabilities/dialogue/requirements.txt", "src/vidxp/capabilities/scene/requirements.txt", "src/vidxp/capabilities/actor/requirements.txt", + "src/vidxp/capabilities/videoprism/requirements.txt", ] } local-worker = { file = [ "src/vidxp/requirements/storage.txt", @@ -107,6 +112,7 @@ local-worker = { file = [ "src/vidxp/capabilities/dialogue/requirements.txt", "src/vidxp/capabilities/scene/requirements.txt", "src/vidxp/capabilities/actor/requirements.txt", + "src/vidxp/capabilities/videoprism/requirements.txt", ] } mcp = { file = ["src/vidxp/requirements/mcp.txt"] } slm = { file = ["src/vidxp/requirements/slm.txt"] } @@ -121,6 +127,7 @@ server-worker = { file = [ "src/vidxp/capabilities/dialogue/requirements.txt", "src/vidxp/capabilities/scene/requirements.txt", "src/vidxp/capabilities/actor/requirements.txt", + "src/vidxp/capabilities/videoprism/requirements.txt", ] } test = { file = ["src/vidxp/requirements/test.txt"] } frontend = { file = ["src/vidxp/requirements/frontend.txt"] } @@ -136,6 +143,9 @@ select = ["E4", "E7", "E9", "F"] torch = [ { index = "pytorch-cpu", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] +torchvision = [ + { index = "pytorch-cpu", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] [[tool.uv.index]] name = "pytorch-cpu" diff --git a/src/vidxp/capabilities/registry.py b/src/vidxp/capabilities/registry.py index 5e82f82..681d814 100644 --- a/src/vidxp/capabilities/registry.py +++ b/src/vidxp/capabilities/registry.py @@ -503,8 +503,9 @@ def _builtin_plugins() -> tuple[CapabilityPlugin, ...]: from vidxp.capabilities.actor.definition import PLUGIN as actor from vidxp.capabilities.dialogue.definition import PLUGIN as dialogue from vidxp.capabilities.scene.definition import PLUGIN as scene + from vidxp.capabilities.videoprism.definition import PLUGIN as videoprism - return dialogue, scene, actor + return dialogue, scene, actor, videoprism def _external_entry_points(allowlist: tuple[str, ...]) -> tuple[EntryPoint, ...]: diff --git a/src/vidxp/capabilities/videoprism/__init__.py b/src/vidxp/capabilities/videoprism/__init__.py new file mode 100644 index 0000000..7e891a3 --- /dev/null +++ b/src/vidxp/capabilities/videoprism/__init__.py @@ -0,0 +1 @@ +"""VideoPrism temporal video search capability.""" diff --git a/src/vidxp/capabilities/videoprism/config.py b/src/vidxp/capabilities/videoprism/config.py new file mode 100644 index 0000000..f415e2a --- /dev/null +++ b/src/vidxp/capabilities/videoprism/config.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pydantic import Field + +from vidxp.capabilities.contracts import CapabilityConfig +from vidxp.core.contracts import IndexConfig + + +class VideoPrismConfig(CapabilityConfig): + batch_size: int = Field(default=1, gt=0) + sample_fps: float = Field(default=2.0, gt=0) + + +def videoprism_config(config: IndexConfig) -> VideoPrismConfig: + return VideoPrismConfig.model_validate(config.options_for("videoprism")) diff --git a/src/vidxp/capabilities/videoprism/definition.py b/src/vidxp/capabilities/videoprism/definition.py new file mode 100644 index 0000000..b4acc2e --- /dev/null +++ b/src/vidxp/capabilities/videoprism/definition.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.application_models import CapabilityRole +from vidxp.capabilities.contracts import ( + CapabilityDefinition, + CapabilityExecutor, + CapabilityPlugin, + OperationDefinition, + PreparationContext, + module_import_check, +) +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.capabilities.videoprism.config import VideoPrismConfig +from vidxp.capabilities.videoprism.indexing import VISUAL_PROCESSOR +from vidxp.capabilities.videoprism.models import get_videoprism_model +from vidxp.capabilities.videoprism.operations import search_operation +from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL +from vidxp.capabilities.visual import index_capabilities +from vidxp.core.contracts import IndexConfig, VideoSource +from vidxp.core.indexing_common import ProgressCallback, report_preparation + + +def prepare_models( + context: PreparationContext, + progress: ProgressCallback | None, +) -> tuple[str, ...]: + VideoPrismConfig.model_validate(context.settings) + report_preparation( + progress, + "videoprism_model", + f"Preparing VideoPrism {VIDEOPRISM_MODEL.model_id}", + ) + get_videoprism_model(context.runtime, download=True, progress=progress) + return (VIDEOPRISM_MODEL.model_id,) + + +def model_manifest( + config: IndexConfig, + _sources: tuple[VideoSource, ...], +) -> Mapping[str, Any]: + return {"videoprism": VIDEOPRISM_MODEL.identity()} + + +DEFINITION = CapabilityDefinition( + name="videoprism", + description="Index and search temporal video clips with VideoPrism.", + extra="videoprism", + config_model=VideoPrismConfig, + collection_name="videoprism", + index_stage="visual_indexing", + execution_group="visual", + prepares_models=True, + roles=(CapabilityRole.searchable, CapabilityRole.queryable), + model_specs=(VIDEOPRISM_MODEL,), + operations={ + "search": OperationDefinition( + input_model=SearchInput, + output_model=SearchResult, + ) + }, +) + + +def create_executor() -> CapabilityExecutor: + return CapabilityExecutor( + indexer=index_capabilities, + index_processor=VISUAL_PROCESSOR, + operations={"search": search_operation}, + prepare=prepare_models, + model_manifest=model_manifest, + runtime_checks=( + module_import_check("OpenCV import", "cv2", "VideoCapture"), + module_import_check("Torch import", "torch"), + module_import_check("Torchvision import", "torchvision"), + module_import_check( + "Transformers VideoPrism import", + "transformers", + "VideoPrismClipModel", + "VideoPrismProcessor", + ), + module_import_check( + "Hugging Face Hub import", + "huggingface_hub", + "snapshot_download", + ), + ), + ) + + +PLUGIN = CapabilityPlugin( + definition=DEFINITION, + executor_factory=create_executor, +) diff --git a/src/vidxp/capabilities/videoprism/indexing.py b/src/vidxp/capabilities/videoprism/indexing.py new file mode 100644 index 0000000..6f82ff3 --- /dev/null +++ b/src/vidxp/capabilities/videoprism/indexing.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Sequence + +from vidxp.capabilities.videoprism.config import videoprism_config +from vidxp.capabilities.videoprism.models import ( + VideoPrismModel, + get_videoprism_model, +) +from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL +from vidxp.core.contracts import ( + CancellationToken, + IndexConfig, + StorageRecord, + batched, + stable_source_id, +) +from vidxp.core.indexing_common import ProgressCallback, report_progress +from vidxp.core.video import FrameSample, FrameSampling +from vidxp.ports import IndexStore, ModelRuntimePort + + +CLIP_FRAMES = 16 + + +@dataclass +class VideoPrismIndexState: + provider: VideoPrismModel + pending: list[FrameSample] = field(default_factory=list) + stored_clips: int = 0 + video_info: Any | None = None + + +def videoprism_sampling(config: IndexConfig, info) -> FrameSampling: + return FrameSampling( + source_fps=info.fps, + target_fps=videoprism_config(config).sample_fps, + ) + + +def encode_video_clips( + clips: Sequence[Sequence[FrameSample]], + provider: VideoPrismModel, +) -> list[list[float]]: + import torch + + inputs = provider.processor( + videos=[[sample.frame for sample in clip] for clip in clips], + do_sample_frames=False, + return_tensors="pt", + ) + inputs = {name: value.to(provider.device) for name, value in inputs.items()} + with torch.inference_mode(): + features = provider.model.get_video_features(**inputs).pooler_output + features = torch.nn.functional.normalize(features, dim=-1) + return features.cpu().numpy().tolist() + + +def videoprism_records( + clips: Sequence[Sequence[FrameSample]], + vectors: Sequence[Sequence[float]], + info, + config: IndexConfig, +) -> list[StorageRecord]: + records = [] + cadence = 1 / min(info.fps, videoprism_config(config).sample_fps) + for clip, vector in zip(clips, vectors): + first, last = clip[0], clip[-1] + end = min(info.duration, last.timestamp + cadence) + if end <= first.timestamp: + end = first.timestamp + 1 / info.fps + source_id = stable_source_id( + config.run_id, + str(config.video_id), + "videoprism", + f"f{first.frame_index:012d}-f{last.frame_index:012d}", + generation_id=config.generation_id, + ) + records.append( + StorageRecord( + source_id=source_id, + embedding=list(vector), + metadata={ + **config.record_identity("videoprism", source_id), + "frame_index": first.frame_index, + "end_frame_index": last.frame_index, + "timestamp": first.timestamp, + "start": first.timestamp, + "end": end, + "fps": info.fps, + "duration": info.duration, + "sample_count": len(clip), + }, + ) + ) + return records + + +def _store_clips( + clips: Sequence[Sequence[FrameSample]], + *, + state: VideoPrismIndexState, + info, + config: IndexConfig, + storage: IndexStore, + cancellation: CancellationToken, +) -> None: + settings = videoprism_config(config) + for group in batched(clips, settings.batch_size): + cancellation.raise_if_cancelled() + model_clips = [ + list(clip) + [clip[-1]] * (CLIP_FRAMES - len(clip)) + for clip in group + ] + vectors = encode_video_clips(model_clips, state.provider) + state.stored_clips += storage.upsert( + "videoprism", + videoprism_records(group, vectors, info, config), + batch_size=config.storage_batch_size, + cancellation=cancellation, + ) + + +def process_videoprism_samples( + samples: Sequence[FrameSample], + *, + state: VideoPrismIndexState, + info, + config: IndexConfig, + storage: IndexStore, + cancellation: CancellationToken, +) -> None: + state.video_info = info + state.pending.extend(samples) + complete = len(state.pending) // CLIP_FRAMES + if not complete: + return + consumed = complete * CLIP_FRAMES + clips = [ + state.pending[start : start + CLIP_FRAMES] + for start in range(0, consumed, CLIP_FRAMES) + ] + del state.pending[:consumed] + _store_clips( + clips, + state=state, + info=info, + config=config, + storage=storage, + cancellation=cancellation, + ) + + +class VideoPrismVisualProcessor: + def sampling(self, config: IndexConfig, info) -> FrameSampling: + return videoprism_sampling(config, info) + + def batch_size(self, config: IndexConfig) -> int: + return CLIP_FRAMES * videoprism_config(config).batch_size + + def prepare( + self, + config: IndexConfig, + runtime: ModelRuntimePort, + progress: ProgressCallback | None, + ) -> VideoPrismIndexState: + report_progress( + progress, + "preparing_videoprism_model", + f"Preparing VideoPrism {VIDEOPRISM_MODEL.model_id}.", + ) + return VideoPrismIndexState(get_videoprism_model(runtime)) + + def process( + self, + samples, + *, + state: VideoPrismIndexState, + info, + config: IndexConfig, + storage: IndexStore, + cancellation: CancellationToken, + ) -> None: + process_videoprism_samples( + samples, + state=state, + info=info, + config=config, + storage=storage, + cancellation=cancellation, + ) + + def finalize( + self, + state: VideoPrismIndexState, + *, + config: IndexConfig, + storage: IndexStore, + ) -> tuple[dict[str, Any], int]: + if state.pending: + if state.video_info is None: + raise RuntimeError("VideoPrism indexing is missing video metadata.") + _store_clips( + [state.pending], + state=state, + info=state.video_info, + config=config, + storage=storage, + cancellation=CancellationToken(), + ) + state.pending.clear() + return {"videoprism_clips": state.stored_clips}, state.stored_clips + + +VISUAL_PROCESSOR = VideoPrismVisualProcessor() diff --git a/src/vidxp/capabilities/videoprism/models.py b/src/vidxp/capabilities/videoprism/models.py new file mode 100644 index 0000000..bc53d4d --- /dev/null +++ b/src/vidxp/capabilities/videoprism/models.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable + +from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL +from vidxp.core.indexing_common import report_preparation +from vidxp.model_contracts import loaded_compute_precision +from vidxp.ports import ModelRuntimePort + + +@dataclass(frozen=True) +class VideoPrismModel: + model: Any + processor: Any + device: str + + +def get_videoprism_model( + runtime: ModelRuntimePort, + *, + download: bool = False, + progress: Callable[[dict[str, Any]], None] | None = None, +) -> VideoPrismModel: + device = runtime.device_for("videoprism") + key = VIDEOPRISM_MODEL.key(device) + + def load() -> VideoPrismModel: + from transformers import VideoPrismClipModel, VideoPrismProcessor + + snapshot = runtime.resolve_model( + VIDEOPRISM_MODEL, + download=download, + progress=progress, + ) + report_preparation( + progress, + "loading_model", + f"Loading {VIDEOPRISM_MODEL.model_id}.", + ) + common = { + "local_files_only": True, + } + model = VideoPrismClipModel.from_pretrained(snapshot, **common).to( + device + ) + model.eval() + runtime.record_compute_precision( + VIDEOPRISM_MODEL.capability, + loaded_compute_precision( + model, + fallback=VIDEOPRISM_MODEL.weights_precision, + ), + ) + return VideoPrismModel( + model=model, + processor=VideoPrismProcessor.from_pretrained(snapshot, **common), + device=device, + ) + + return runtime.get_or_load(key, load) diff --git a/src/vidxp/capabilities/videoprism/operations.py b/src/vidxp/capabilities/videoprism/operations.py new file mode 100644 index 0000000..b354199 --- /dev/null +++ b/src/vidxp/capabilities/videoprism/operations.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from vidxp.capabilities.contracts import CapabilityContext +from vidxp.capabilities.schemas import SearchInput, SearchResult +from vidxp.capabilities.search import search_embeddings +from vidxp.capabilities.videoprism.models import get_videoprism_model +from vidxp.core.contracts import IndexConfig +from vidxp.ports import IndexStore, ModelRuntimePort + + +REQUIRED_METADATA = frozenset( + { + "dataset", + "split", + "run_id", + "video_id", + "source_id", + "start", + "end", + "frame_index", + "timestamp", + "fps", + "duration", + "modality", + } +) + + +def videoprism_embedding( + query: str, + runtime: ModelRuntimePort, +) -> list[float]: + import torch + + provider = get_videoprism_model(runtime) + inputs = provider.processor( + text=[query], + padding="max_length", + max_length=64, + truncation=True, + return_tensors="pt", + ) + inputs = {name: value.to(provider.device) for name, value in inputs.items()} + with torch.inference_mode(): + features = provider.model.get_text_features(**inputs).pooler_output + features = torch.nn.functional.normalize(features, dim=-1) + return features.cpu().numpy().tolist()[0] + + +def search_videoprism( + query: str, + *, + config: IndexConfig, + runtime: ModelRuntimePort, + top_k: int = 10, + video_id: str | None = None, + query_id: str | None = None, + filters: Mapping[str, Any] | None = None, + storage: IndexStore, +) -> SearchResult: + cleaned = query.strip() + if not cleaned: + raise ValueError("Search query must not be empty.") + if top_k <= 0: + raise ValueError("top_k must be greater than zero.") + return search_embeddings( + cleaned, + "videoprism", + videoprism_embedding(cleaned, runtime), + config=config, + required_metadata=REQUIRED_METADATA, + top_k=top_k, + video_id=video_id, + query_id=query_id, + filters=filters, + storage=storage, + ) + + +def search_operation( + context: CapabilityContext, + request: SearchInput, +) -> SearchResult: + config = context.require_config() + return search_videoprism( + request.query, + config=config, + top_k=request.top_k, + video_id=request.media_id or config.video_id, + runtime=context.runtime, + storage=context.require_storage(), + ) diff --git a/src/vidxp/capabilities/videoprism/requirements.txt b/src/vidxp/capabilities/videoprism/requirements.txt new file mode 100644 index 0000000..d0bfdf8 --- /dev/null +++ b/src/vidxp/capabilities/videoprism/requirements.txt @@ -0,0 +1,6 @@ +numpy>=2.3,<3 +opencv-python-headless>=5.0.0.93,<6 +torch>=2.13,<3 +torchvision>=0.28,<1 +transformers>=5.14.1,<6 +huggingface-hub>=1.25.1,<2 diff --git a/src/vidxp/capabilities/videoprism/specs.py b/src/vidxp/capabilities/videoprism/specs.py new file mode 100644 index 0000000..f2f9209 --- /dev/null +++ b/src/vidxp/capabilities/videoprism/specs.py @@ -0,0 +1,16 @@ +from vidxp.model_contracts import ModelSpec + + +VIDEOPRISM_MODEL = ModelSpec( + capability="videoprism", + provider="transformers", + model_id="google/videoprism-lvt-base-f16r288", + revision="fb6de9f0eb7bc285be86bdca1cf7daa3e3ef51ff", + download_size_bytes=993_993_146, + weights_file="model.safetensors", + weights_sha256=( + "7d64ac2364d3473c0dd9fde35fb09e3cfb3b43153c3e9af79d7e49f1c8387cf5" + ), + license="Apache-2.0", + weights_precision="float32", +) diff --git a/src/vidxp/frontend.py b/src/vidxp/frontend.py index 0260d7e..cf76a1b 100644 --- a/src/vidxp/frontend.py +++ b/src/vidxp/frontend.py @@ -838,7 +838,9 @@ def run(): st.logo(application_icon, size="large", link=PROJECT_URL) service = _configured_service() st.title("VidXP") - st.caption("Index and search video by dialogue, scene, and actor.") + st.caption( + "Index and search video by dialogue, scene, temporal clips, and actor." + ) st.caption(f"Index repository: {service.layout.root}") if notice := st.session_state.pop(MEDIA_NOTICE_KEY, None): st.success(notice) diff --git a/src/vidxp/local_probe.py b/src/vidxp/local_probe.py index f57697e..1c0bcfa 100644 --- a/src/vidxp/local_probe.py +++ b/src/vidxp/local_probe.py @@ -110,7 +110,9 @@ def _surface_capabilities( return { "worker": _surface_capability( installed=( - {"dialogue", "scene", "actor"}.issubset(search_capabilities) + {"dialogue", "scene", "actor", "videoprism"}.issubset( + search_capabilities + ) and _module_available("pydantic_ai") ), media_ready=media_ready, diff --git a/src/vidxp/runtime.py b/src/vidxp/runtime.py index 97086f2..a02ba49 100644 --- a/src/vidxp/runtime.py +++ b/src/vidxp/runtime.py @@ -293,6 +293,7 @@ def download_snapshot() -> str: revision=spec.revision, cache_dir=str(cache), local_files_only=False, + ignore_patterns=("*.h5", "*.msgpack", "*.npz", "*.ot"), tqdm_class=ReportingTqdm, ) ) diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 3a3c8d8..9edd0d7 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -25,6 +25,7 @@ ) from vidxp.capability_service import CapabilityService from vidxp.capabilities.scene.config import SceneConfig +from vidxp.capabilities.videoprism.config import VideoPrismConfig from vidxp.core.contracts import IndexConfig from vidxp.core.runner import _index_groups @@ -62,12 +63,12 @@ def test_module_import_checks_run_in_an_isolated_process(self): def test_registry_drives_capability_metadata(self): self.assertEqual( self.registry.names(), - ("dialogue", "scene", "actor"), + ("dialogue", "scene", "actor", "videoprism"), ) self.assertEqual(self.registry.index_names(), self.registry.names()) self.assertEqual( self.registry.preparable_names(), - ("dialogue", "scene", "actor"), + ("dialogue", "scene", "actor", "videoprism"), ) self.assertEqual( self.registry.collection_names(), @@ -75,6 +76,7 @@ def test_registry_drives_capability_metadata(self): "dialogue": "dialogue", "scene": "scene", "actor": "actor", + "videoprism": "videoprism", }, ) @@ -140,6 +142,10 @@ def test_built_in_settings_are_owned_and_validated(self): ) self.assertIs(self.registry.get("scene").config_model, SceneConfig) self.assertIs(self.registry.get("actor").config_model, ActorConfig) + self.assertIs( + self.registry.get("videoprism").config_model, + VideoPrismConfig, + ) options = self.registry.validate_options( ("scene",), @@ -207,10 +213,10 @@ def test_operation_only_capability_needs_no_index_metadata(self): def test_visual_execution_group_is_explicit(self): self.assertEqual( _index_groups( - ("dialogue", "scene", "actor"), + ("dialogue", "scene", "actor", "videoprism"), self.registry, ), - (("dialogue",), ("scene", "actor")), + (("dialogue",), ("scene", "actor", "videoprism")), ) self.assertIsNotNone( self.registry.executor("scene").index_processor diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 9991b0a..1070de5 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -277,7 +277,7 @@ def check(command): ): available = frontend._available_index_modalities() - self.assertEqual(available, ("dialogue", "scene")) + self.assertEqual(available, ("dialogue", "scene", "videoprism")) self.assertTrue( all( not call.args[0].include_runtime_checks diff --git a/tests/test_local_probe.py b/tests/test_local_probe.py index 62accbf..2832c40 100644 --- a/tests/test_local_probe.py +++ b/tests/test_local_probe.py @@ -37,7 +37,7 @@ def build(self, **overrides): patch("vidxp.local_probe._module_available", return_value=True), patch( "vidxp.local_probe._installed_search_capabilities", - return_value=["actor", "dialogue", "scene"], + return_value=["actor", "dialogue", "scene", "videoprism"], ), patch( "vidxp.local_probe.media_runtime_is_initialized", @@ -85,7 +85,7 @@ def test_probe_reports_stable_identity_and_contract_compatibility(self): ) self.assertEqual( payload["search_capabilities"], - ["actor", "dialogue", "scene"], + ["actor", "dialogue", "scene", "videoprism"], ) self.assertTrue(all(surface["launchable"] for surface in payload["surfaces"].values())) @@ -247,11 +247,12 @@ def test_non_windows_launcher_resolution_does_not_add_executable_suffix(self): def test_desktop_model_catalog_is_derived_from_canonical_specs(self): catalog = desktop_model_cache_catalog() - self.assertEqual(len(catalog), 5) + self.assertEqual(len(catalog), 6) self.assertEqual( {item["id"] for item in catalog}, { "google/siglip2-base-patch16-224", + "google/videoprism-lvt-base-f16r288", "Qwen/Qwen3-Embedding-0.6B", "dropbox-dash/faster-whisper-large-v3-turbo", "yunet", diff --git a/tests/test_mcp.py b/tests/test_mcp.py index e7f1782..c0d04eb 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -2706,7 +2706,7 @@ async def test_stdio_entrypoint_serves_the_filesystem_aware_surface(self): ) self.assertEqual( [item["name"] for item in result.structured_content["items"]], - ["dialogue", "scene", "actor"], + ["dialogue", "scene", "actor", "videoprism"], ) async def test_streamable_http_works_with_the_official_remote_client(self): diff --git a/tests/test_models.py b/tests/test_models.py index ca3af4a..ea361be 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -281,7 +281,7 @@ def download(**options): with patch( "huggingface_hub.snapshot_download", side_effect=download, - ), patch( + ) as snapshot_download, patch( "huggingface_hub.constants.HF_HUB_DISABLE_XET", False, ): @@ -295,6 +295,10 @@ def download(**options): ) self.assertEqual(resolved, snapshot) + self.assertEqual( + snapshot_download.call_args.kwargs["ignore_patterns"], + ("*.h5", "*.msgpack", "*.npz", "*.ot"), + ) download_events = [ event for event in events if event["stage"] == "downloading_model" ] diff --git a/tests/test_videoprism.py b/tests/test_videoprism.py new file mode 100644 index 0000000..5e7b50a --- /dev/null +++ b/tests/test_videoprism.py @@ -0,0 +1,81 @@ +import unittest +from unittest.mock import Mock, patch + +from pydantic import ValidationError + +from vidxp.capabilities.videoprism.config import VideoPrismConfig +from vidxp.capabilities.videoprism.indexing import ( + CLIP_FRAMES, + VISUAL_PROCESSOR, + VideoPrismIndexState, + process_videoprism_samples, +) +from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL +from vidxp.core.contracts import CancellationToken, IndexConfig +from vidxp.core.video import FrameSample, VideoInfo + + +class VideoPrismTests(unittest.TestCase): + def test_config_rejects_invalid_sampling(self): + with self.assertRaises(ValidationError): + VideoPrismConfig(sample_fps=0) + + def test_streaming_index_groups_clips_and_pads_only_the_tail(self): + config = IndexConfig( + video_id="video-1", + enabled_modalities=("videoprism",), + ) + info = VideoInfo(30.0, 270, 9.0, 2, 2) + samples = [ + FrameSample(index * 15, index / 2, object()) + for index in range(18) + ] + state = VideoPrismIndexState(provider=Mock()) + storage = Mock() + storage.upsert.side_effect = lambda _name, records, **_kwargs: len( + records + ) + + with patch( + "vidxp.capabilities.videoprism.indexing.encode_video_clips", + side_effect=lambda clips, _provider: [[0.1] for _ in clips], + ) as encode: + process_videoprism_samples( + samples, + state=state, + info=info, + config=config, + storage=storage, + cancellation=CancellationToken(), + ) + summary, operations = VISUAL_PROCESSOR.finalize( + state, + config=config, + storage=storage, + ) + + self.assertEqual(summary, {"videoprism_clips": 2}) + self.assertEqual(operations, 2) + self.assertEqual( + [len(call.args[0][0]) for call in encode.call_args_list], + [CLIP_FRAMES, CLIP_FRAMES], + ) + tail = storage.upsert.call_args_list[1].args[1][0] + self.assertEqual(tail.metadata["sample_count"], 2) + self.assertEqual( + (tail.metadata["start"], tail.metadata["end"]), + (8.0, 9.0), + ) + + def test_model_contract_pins_the_pytorch_checkpoint(self): + self.assertEqual( + VIDEOPRISM_MODEL.model_id, + "google/videoprism-lvt-base-f16r288", + ) + self.assertEqual(VIDEOPRISM_MODEL.weights_file, "model.safetensors") + self.assertEqual(VIDEOPRISM_MODEL.license, "Apache-2.0") + self.assertEqual(VIDEOPRISM_MODEL.download_size_bytes, 993_993_146) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index fcfa563..09296c6 100644 --- a/uv.lock +++ b/uv.lock @@ -3452,7 +3452,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3531,7 +3531,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -3894,13 +3894,13 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "filelock", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "fsspec", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "jinja2", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "networkx", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "setuptools", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "sympy", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, - { name = "typing-extensions", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" }, @@ -3921,13 +3921,13 @@ resolution-markers = [ "(python_full_version < '3.12' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform == 'win32')", ] dependencies = [ - { name = "filelock", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "fsspec", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "jinja2", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "networkx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "setuptools", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "sympy", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp311-cp311-linux_s390x.whl", hash = "sha256:6e9817dbdf5ea76789babd46e457eac5bf14ff566cf85f8addbfdff2d56601ce", upload-time = "2026-07-08T19:27:52Z" }, @@ -3955,6 +3955,64 @@ wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:a43376bd094124ef626bfdd3d4c2c62eacb0b5ddc99776f4a32d4fd16f1f3420", upload-time = "2026-07-08T19:31:48Z" }, ] +[[package]] +name = "torchvision" +version = "0.28.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pillow" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b2/1e010052079e4c577007b789db336ea7075f1a426e84d17121fbc3745516/torchvision-0.28.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:83fe6c020866a85acd7d97deccc45ff11d66daf42916d04396a4309c66c0ccb8", size = 1856017, upload-time = "2026-07-08T16:07:55.533Z" }, + { url = "https://files.pythonhosted.org/packages/15/49/c1cab1ecbb3ff1a380a3f99283db1dee61b8afe354f6352c643b65937130/torchvision-0.28.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e9f54c30cd52e3ef7fd034cc69b7bb7e0964e1c8f8743e018ab92e95b40f9eee", size = 1856020, upload-time = "2026-07-08T16:07:52.182Z" }, + { url = "https://files.pythonhosted.org/packages/20/55/08a726c14c67b37c8aca04b077766909f1c7ed23f76116884fe63b9bd033/torchvision-0.28.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d483b4aa3f5237569053f749cd1a2b5bb548ca456e40461a5dd087f21149d123", size = 1856021, upload-time = "2026-07-08T16:07:45.386Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b9/da40eca5bbe9596c12ae9899ab7abaf887f5e20f29d08b924b4633714821/torchvision-0.28.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3bd9dba55224a9db4a2d77f6feaa5651770d8c8e86d3d0ddb0fa6bec54c8712b", size = 1856014, upload-time = "2026-07-08T16:07:44.282Z" }, + { url = "https://files.pythonhosted.org/packages/6a/80/822a6163da716f8a78141cf6678d74e26a572285d4ea866ef8aa657bb307/torchvision-0.28.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:09ce8f56e81f19b9c378ae7bb109f83f6659fd8bc3cd14241a48e4af46e9ed49", size = 1856011, upload-time = "2026-07-08T16:07:33.404Z" }, +] + +[[package]] +name = "torchvision" +version = "0.28.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "(python_full_version >= '3.14' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform == 'win32')", + "(python_full_version == '3.13.*' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform == 'win32')", + "(python_full_version == '3.12.*' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform == 'win32')", + "(python_full_version < '3.12' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform == 'win32')", +] +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pillow" }, + { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" } }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:22958193d72444ed7cbcc665ba4821a31e5279f9c4d1ad08520918b30896b78a", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1dad604dfc0177ecebe0891bd9701fe2c62ec3f7819a247be541b3fb6effee99", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp311-cp311-win_amd64.whl", hash = "sha256:7b6667fd0172463be2a271fb0dbd44b31a7891afd549a66208613ce4cdd79f88", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2f768c4f6d5adf6d5535061fd69ec44827608bac0e96e12114942a6fdfce1107", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b545d46f4d2f9d30381281cf22874bfe1d32a8a7b0ee8396fccde89f30c6a9d9", upload-time = "2026-07-08T12:26:40Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:d88db83abbdfb97199979ec94dd427bc372c1b9ab01f0dbed20af05b0bd644b1", upload-time = "2026-07-08T12:26:40Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:879ae6d4e2e3651582fb7187eafd535601cb5d019595d47e2c874262a000e88e", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c6373ec4c2f922e89f45ac91889404d312ba29a31f205b0ad9a725a3894ca246", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:d63eae114b4d1fca2b294d300cea3f0d6c71b6d132641e0c4cab1aa06a467b0d", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:3a1a76c8decb1d7bbedd3588bccc90fb269944b7321a773db181735b42115422", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:1aa741ae0eb8668b6287dd667548e2dd10179c828db68bfdee1519763b9c5b99", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:d2a0171faa211b506c4dcf3a036942a41077c5d2d3d94883dafbda7b8624a3eb", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:8d8b98608779c770ede5e20609772453ebc7487ebb8697445d1856466c542f45", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:870d0d42f2eb80f4870cd35e51eea52f596a408a671b28136f06a808846f24c5", upload-time = "2026-07-08T12:26:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:82dbffb63d61cd43d9c7a311588e665aa2b21173a05f852b4d384d6782fd88ef", upload-time = "2026-07-08T12:26:39Z" }, +] + [[package]] name = "tqdm" version = "4.70.0" @@ -4151,6 +4209,8 @@ all = [ { name = "sentence-transformers" }, { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.28.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torchvision", version = "0.28.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "transformers" }, ] benchmarks = [ @@ -4180,6 +4240,8 @@ local-worker = [ { name = "sentence-transformers" }, { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.28.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torchvision", version = "0.28.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "transformers" }, ] mcp = [ @@ -4228,6 +4290,8 @@ server-worker = [ { name = "sentence-transformers" }, { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.28.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torchvision", version = "0.28.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "transformers" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -4242,6 +4306,19 @@ test = [ { name = "httpx" }, { name = "pytest" }, ] +videoprism = [ + { name = "chromadb" }, + { name = "huggingface-hub" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "opencv-python-headless" }, + { name = "psutil" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.28.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torchvision", version = "0.28.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "transformers" }, +] [package.metadata] requires-dist = [ @@ -4255,6 +4332,7 @@ requires-dist = [ { name = "chromadb", marker = "extra == 'local-worker'", specifier = ">=1.5.9,<2" }, { name = "chromadb", marker = "extra == 'scene'", specifier = ">=1.5.9,<2" }, { name = "chromadb", marker = "extra == 'storage'", specifier = ">=1.5.9,<2" }, + { name = "chromadb", marker = "extra == 'videoprism'", specifier = ">=1.5.9,<2" }, { name = "chromadb-client", marker = "extra == 'server-worker'", specifier = ">=1.5.9,<2" }, { name = "dbos", specifier = ">=2.28,<3" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.140.13,<0.141" }, @@ -4270,6 +4348,7 @@ requires-dist = [ { name = "huggingface-hub", marker = "extra == 'local-worker'", specifier = ">=1.25.1,<2" }, { name = "huggingface-hub", marker = "extra == 'scene'", specifier = ">=1.25.1,<2" }, { name = "huggingface-hub", marker = "extra == 'server-worker'", specifier = ">=1.25.1,<2" }, + { name = "huggingface-hub", marker = "extra == 'videoprism'", specifier = ">=1.25.1,<2" }, { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.0,<3" }, { name = "mcp", marker = "extra == 'server'", specifier = ">=2.0,<3" }, { name = "numpy", marker = "extra == 'actor'", specifier = ">=2.3,<3" }, @@ -4277,11 +4356,13 @@ requires-dist = [ { name = "numpy", marker = "extra == 'local-worker'", specifier = ">=2.3,<3" }, { name = "numpy", marker = "extra == 'scene'", specifier = ">=2.3,<3" }, { name = "numpy", marker = "extra == 'server-worker'", specifier = ">=2.3,<3" }, + { name = "numpy", marker = "extra == 'videoprism'", specifier = ">=2.3,<3" }, { name = "opencv-python-headless", marker = "extra == 'actor'", specifier = ">=5.0.0.93,<6" }, { name = "opencv-python-headless", marker = "extra == 'all'", specifier = ">=5.0.0.93,<6" }, { name = "opencv-python-headless", marker = "extra == 'local-worker'", specifier = ">=5.0.0.93,<6" }, { name = "opencv-python-headless", marker = "extra == 'scene'", specifier = ">=5.0.0.93,<6" }, { name = "opencv-python-headless", marker = "extra == 'server-worker'", specifier = ">=5.0.0.93,<6" }, + { name = "opencv-python-headless", marker = "extra == 'videoprism'", specifier = ">=5.0.0.93,<6" }, { name = "packaging", specifier = ">=26.2,<27" }, { name = "pillow", marker = "extra == 'all'", specifier = ">=12.3,<13" }, { name = "pillow", marker = "extra == 'local-worker'", specifier = ">=12.3,<13" }, @@ -4300,6 +4381,7 @@ requires-dist = [ { name = "psutil", marker = "extra == 'server'", specifier = ">=7.2.2,<8" }, { name = "psutil", marker = "extra == 'server-worker'", specifier = ">=7.2.2,<8" }, { name = "psutil", marker = "extra == 'storage'", specifier = ">=7.2.2,<8" }, + { name = "psutil", marker = "extra == 'videoprism'", specifier = ">=7.2.2,<8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'server'", specifier = ">=3.3.4,<4" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'server-worker'", specifier = ">=3.3.4,<4" }, { name = "pydantic", specifier = ">=2.13.4,<3" }, @@ -4325,19 +4407,30 @@ requires-dist = [ { name = "torch", marker = "(sys_platform == 'linux' and extra == 'local-worker') or (sys_platform == 'win32' and extra == 'local-worker')", specifier = ">=2.13,<3", index = "https://download.pytorch.org/whl/cpu" }, { name = "torch", marker = "(sys_platform == 'linux' and extra == 'scene') or (sys_platform == 'win32' and extra == 'scene')", specifier = ">=2.13,<3", index = "https://download.pytorch.org/whl/cpu" }, { name = "torch", marker = "(sys_platform == 'linux' and extra == 'server-worker') or (sys_platform == 'win32' and extra == 'server-worker')", specifier = ">=2.13,<3", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torch", marker = "(sys_platform == 'linux' and extra == 'videoprism') or (sys_platform == 'win32' and extra == 'videoprism')", specifier = ">=2.13,<3", index = "https://download.pytorch.org/whl/cpu" }, { name = "torch", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'all'", specifier = ">=2.13,<3" }, { name = "torch", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'local-worker'", specifier = ">=2.13,<3" }, { name = "torch", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'scene'", specifier = ">=2.13,<3" }, { name = "torch", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'server-worker'", specifier = ">=2.13,<3" }, + { name = "torch", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'videoprism'", specifier = ">=2.13,<3" }, + { name = "torchvision", marker = "(sys_platform == 'linux' and extra == 'all') or (sys_platform == 'win32' and extra == 'all')", specifier = ">=0.28,<1", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torchvision", marker = "(sys_platform == 'linux' and extra == 'local-worker') or (sys_platform == 'win32' and extra == 'local-worker')", specifier = ">=0.28,<1", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torchvision", marker = "(sys_platform == 'linux' and extra == 'server-worker') or (sys_platform == 'win32' and extra == 'server-worker')", specifier = ">=0.28,<1", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torchvision", marker = "(sys_platform == 'linux' and extra == 'videoprism') or (sys_platform == 'win32' and extra == 'videoprism')", specifier = ">=0.28,<1", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torchvision", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'all'", specifier = ">=0.28,<1" }, + { name = "torchvision", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'local-worker'", specifier = ">=0.28,<1" }, + { name = "torchvision", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'server-worker'", specifier = ">=0.28,<1" }, + { name = "torchvision", marker = "sys_platform != 'linux' and sys_platform != 'win32' and extra == 'videoprism'", specifier = ">=0.28,<1" }, { name = "transformers", marker = "extra == 'all'", specifier = ">=5.14.1,<6" }, { name = "transformers", marker = "extra == 'local-worker'", specifier = ">=5.14.1,<6" }, { name = "transformers", marker = "extra == 'scene'", specifier = ">=5.14.1,<6" }, { name = "transformers", marker = "extra == 'server-worker'", specifier = ">=5.14.1,<6" }, + { name = "transformers", marker = "extra == 'videoprism'", specifier = ">=5.14.1,<6" }, { name = "typer", specifier = ">=0.27,<1" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.51,<0.52" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'server-worker'", specifier = ">=0.51,<0.52" }, ] -provides-extras = ["storage", "dialogue", "scene", "actor", "all", "local-worker", "mcp", "slm", "server", "server-worker", "test", "frontend", "benchmarks"] +provides-extras = ["storage", "dialogue", "scene", "actor", "videoprism", "all", "local-worker", "mcp", "slm", "server", "server-worker", "test", "frontend", "benchmarks"] [[package]] name = "watchdog" From b231b068ef75e34dba5a83ecef0cb70180f9bd6b Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 13 Aug 2026 15:09:15 +0500 Subject: [PATCH 2/5] feat(frontend): expose VideoPrism temporal search --- .../vidxp/skills/vidxp-ingest-video/SKILL.md | 5 +- src/vidxp/frontend.py | 71 ++++++++++++++++++- tests/test_frontend.py | 45 +++++++++++- tests/test_frontend_app.py | 23 +++++- 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/plugins/vidxp/skills/vidxp-ingest-video/SKILL.md b/plugins/vidxp/skills/vidxp-ingest-video/SKILL.md index 198023f..3a45354 100644 --- a/plugins/vidxp/skills/vidxp-ingest-video/SKILL.md +++ b/plugins/vidxp/skills/vidxp-ingest-video/SKILL.md @@ -10,8 +10,9 @@ description: Use VidXP to upload, import, register, and automatically index vide 1. Resolve the `vidxp` MCP tools and call `get_workspace`. Do not import a video that is already registered or indexed. 2. Choose indexable modalities from the workspace. Use `dialogue` and `scene` - for ordinary content retrieval. Add `actor` only when anonymous recurring-face - clusters are wanted; it does not identify people by name. + for ordinary content retrieval. Add `videoprism` when the request depends on + actions or events spanning multiple frames. Add `actor` only when anonymous + recurring-face clusters are wanted; it does not identify people by name. 3. Call `get_runtime_readiness`. If selected models are missing, submit `prepare_models`, use `wait_job` with its observation token for subsequent bounded waits, then fetch `get_job` once when terminal. diff --git a/src/vidxp/frontend.py b/src/vidxp/frontend.py index cf76a1b..3e66122 100644 --- a/src/vidxp/frontend.py +++ b/src/vidxp/frontend.py @@ -120,6 +120,24 @@ def _settings_from_arguments( 1.0: "Balanced — every second", 2.0: "Detailed — twice per second", } +VIDEOPRISM_SAMPLE_FPS_DEFAULT = 2.0 +VIDEOPRISM_DETAIL_PRESETS = (1.0, VIDEOPRISM_SAMPLE_FPS_DEFAULT, 4.0) +VIDEOPRISM_DETAIL_LABELS = { + 1.0: "Long actions — about 16 seconds per clip", + 2.0: "Balanced — about 8 seconds per clip", + 4.0: "Short actions — about 4 seconds per clip", +} +CAPABILITY_LABELS = { + "actor": "Actor groups", + "dialogue": "Dialogue search", + "natural-language": "Ask a question", + "scene": "Scene search", + "videoprism": "Temporal action search (VideoPrism)", +} + + +def _capability_label(name: str) -> str: + return CAPABILITY_LABELS.get(name, name.replace("-", " ").title()) def _format_bytes(size: int) -> str: @@ -191,7 +209,10 @@ def _render_summary(summary): ( f"Media: {summary.get('media_count', 0):,}", "Capabilities: " - + ", ".join(summary.get("modalities", ())), + + ", ".join( + _capability_label(name) + for name in summary.get("modalities", ()) + ), ) ) ) @@ -347,6 +368,7 @@ def _run_indexing( modalities, *, scene_sample_fps: float | None = None, + videoprism_sample_fps: float | None = None, ): service = _configured_service() temporary_path = None @@ -383,6 +405,15 @@ def _run_indexing( media_id=media_id, modalities=modalities, scene_sample_fps=scene_sample_fps, + capability_options=( + { + "videoprism": { + "sample_fps": videoprism_sample_fps, + } + } + if videoprism_sample_fps is not None + else {} + ), ) ) st.session_state[INDEX_JOB_ID_KEY] = job.job_id @@ -428,6 +459,28 @@ def _scene_sample_fps_control( ) +def _videoprism_sample_fps_control( + modalities: tuple[str, ...], + *, + disabled: bool, +) -> float | None: + if "videoprism" not in modalities: + return None + return float( + st.selectbox( + "Temporal clip length", + VIDEOPRISM_DETAIL_PRESETS, + index=1, + format_func=VIDEOPRISM_DETAIL_LABELS.__getitem__, + disabled=disabled, + help=( + "VideoPrism embeds 16 sampled frames per clip. Longer clips " + "cover slower actions; shorter clips localize quick actions." + ), + ) + ) + + def _run_search(search_type, query, media_id=None): service = _configured_service() try: @@ -654,6 +707,8 @@ def poll_search_job(): result_label = ( "Closest sampled scene" if search_type == "scene" + else "Closest temporal action clip" + if search_type == "videoprism" else "Closest supporting evidence" if search_type == "natural-language" else f"Closest {search_type} match" @@ -665,6 +720,11 @@ def poll_search_job(): "It does not identify the first occurrence and is not reliable " "for counting people." ) + elif search_type == "videoprism": + st.caption( + "VideoPrism ranks short multi-frame clips, making it better suited " + "to actions and events than single-frame scene search." + ) st.video( str(resource.path), start_time=timestamp, @@ -797,6 +857,7 @@ def _search_controls(ready, uploaded_video, available_modalities): search_type = st.selectbox( "Search type", ["natural-language", *available_modalities], + format_func=_capability_label, disabled=not ready, ) with query_column: @@ -813,6 +874,8 @@ def _search_controls(ready, uploaded_video, available_modalities): if search_type == "actor" else "For example: What happens after the taxi arrives?" if search_type == "natural-language" + else "For example: A person opens a door and walks out." + if search_type == "videoprism" else "For example: Chef makes pizza and cuts it up." ), disabled=not ready, @@ -965,6 +1028,7 @@ def run(): "Capabilities", installed_modalities, default=installed_modalities, + format_func=_capability_label, disabled=busy, help="Install another capability extra to make it available here.", ) @@ -973,6 +1037,10 @@ def run(): selected_modalities, disabled=busy, ) + videoprism_sample_fps = _videoprism_sample_fps_control( + selected_modalities, + disabled=busy, + ) model_readiness = ( service.model_readiness(selected_modalities) if selected_modalities @@ -1211,6 +1279,7 @@ def poll_index_status(): status, selected_modalities, scene_sample_fps=scene_sample_fps, + videoprism_sample_fps=videoprism_sample_fps, ) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 1070de5..71fe7f9 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -89,12 +89,12 @@ def test_query_modalities_use_real_capability_service_contracts(self): return_value=service, ): available = frontend._available_query_modalities( - ("dialogue", "scene", "actor"), + ("dialogue", "scene", "actor", "videoprism"), ) self.assertEqual( available, - ("dialogue", "scene", "actor"), + ("dialogue", "scene", "actor", "videoprism"), ) def tearDown(self): @@ -339,6 +339,47 @@ def test_indexing_submits_selected_scene_sample_rate(self): self.assertEqual(command.scene_sample_fps, 2.0) service.require_models.assert_not_called() + def test_videoprism_clip_control_is_conditional_and_configures_index(self): + with patch.object( + frontend.st, + "selectbox", + return_value=4.0, + ) as selectbox: + selected = frontend._videoprism_sample_fps_control( + ("videoprism",), + disabled=False, + ) + + self.assertEqual(selected, 4.0) + self.assertEqual( + selectbox.call_args.args[:2], + ("Temporal clip length", (1.0, 2.0, 4.0)), + ) + self.assertEqual(selectbox.call_args.kwargs["index"], 1) + + jobs = Mock() + jobs.submit_index.return_value = SimpleNamespace(job_id="job-1") + session_state = {frontend.MEDIA_ID_KEY: MEDIA_ID} + with ( + patch.object(frontend, "_configured_service", return_value=Mock()), + patch.object(frontend, "_configured_jobs", return_value=jobs), + patch.object(frontend.st, "session_state", session_state), + patch.object(frontend.st, "query_params", {}), + patch.object(frontend.st, "rerun"), + ): + frontend._run_indexing( + None, + {}, + ("videoprism",), + videoprism_sample_fps=selected, + ) + + command = jobs.submit_index.call_args.args[0] + self.assertEqual( + command.capability_options, + {"videoprism": {"sample_fps": 4.0}}, + ) + def test_indexing_omits_scene_sample_rate_without_scene(self): jobs = Mock() jobs.submit_index.return_value = SimpleNamespace(job_id="job-1") diff --git a/tests/test_frontend_app.py b/tests/test_frontend_app.py index 833d2fa..0b74bcd 100644 --- a/tests/test_frontend_app.py +++ b/tests/test_frontend_app.py @@ -161,7 +161,7 @@ def ready_status() -> IndexStatus: snapshot_id=SNAPSHOT_ID, media_count=1, media_ids=(MEDIA_ID,), - modalities=("dialogue", "scene", "actor"), + modalities=("dialogue", "scene", "actor", "videoprism"), ), ) @@ -269,6 +269,27 @@ def test_ready_page_rejects_empty_search_without_disabling_form(self): self.assertEqual(app.warning[-1].value, "Enter a search query.") self.assertEqual(jobs.submitted_searches, []) + def test_ready_page_exposes_videoprism_as_temporal_action_search(self): + service = FrontendApplicationStub(self.root, ready_status()) + jobs = FrontendJobStub() + app = self.app(service, jobs).run() + + capability_picker = self.widget(app.multiselect, "Capabilities") + self.assertIn( + "Temporal action search (VideoPrism)", + capability_picker.options, + ) + temporal_control = self.widget(app.selectbox, "Temporal clip length") + self.assertEqual(temporal_control.value, 2.0) + + search_type = self.widget(app.selectbox, "Search type") + search_type.select("videoprism") + app.text_input(key="video_search_query").input("a person walks out") + self.widget(app.button, "Search").click() + app.run() + + self.assertEqual(jobs.submitted_searches[0].modalities, ("videoprism",)) + def test_running_index_keeps_one_preview_and_disables_mutations(self): service = FrontendApplicationStub(self.root, ready_status()) jobs = FrontendJobStub() From b9226e8374f98050469e261331ee10450fa74b67 Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 13 Aug 2026 15:16:31 +0500 Subject: [PATCH 3/5] fix(videoprism): flatten pooled embeddings --- src/vidxp/capabilities/videoprism/indexing.py | 3 ++- src/vidxp/capabilities/videoprism/models.py | 6 ++++++ src/vidxp/capabilities/videoprism/operations.py | 7 +++++-- tests/test_videoprism.py | 10 ++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/vidxp/capabilities/videoprism/indexing.py b/src/vidxp/capabilities/videoprism/indexing.py index 6f82ff3..53df5f4 100644 --- a/src/vidxp/capabilities/videoprism/indexing.py +++ b/src/vidxp/capabilities/videoprism/indexing.py @@ -7,6 +7,7 @@ from vidxp.capabilities.videoprism.models import ( VideoPrismModel, get_videoprism_model, + normalize_pooled_output, ) from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL from vidxp.core.contracts import ( @@ -53,7 +54,7 @@ def encode_video_clips( inputs = {name: value.to(provider.device) for name, value in inputs.items()} with torch.inference_mode(): features = provider.model.get_video_features(**inputs).pooler_output - features = torch.nn.functional.normalize(features, dim=-1) + features = normalize_pooled_output(features) return features.cpu().numpy().tolist() diff --git a/src/vidxp/capabilities/videoprism/models.py b/src/vidxp/capabilities/videoprism/models.py index bc53d4d..dde6e0f 100644 --- a/src/vidxp/capabilities/videoprism/models.py +++ b/src/vidxp/capabilities/videoprism/models.py @@ -16,6 +16,12 @@ class VideoPrismModel: device: str +def normalize_pooled_output(features: Any) -> Any: + import torch + + return torch.nn.functional.normalize(features.flatten(start_dim=1), dim=-1) + + def get_videoprism_model( runtime: ModelRuntimePort, *, diff --git a/src/vidxp/capabilities/videoprism/operations.py b/src/vidxp/capabilities/videoprism/operations.py index b354199..dc7b804 100644 --- a/src/vidxp/capabilities/videoprism/operations.py +++ b/src/vidxp/capabilities/videoprism/operations.py @@ -5,7 +5,10 @@ from vidxp.capabilities.contracts import CapabilityContext from vidxp.capabilities.schemas import SearchInput, SearchResult from vidxp.capabilities.search import search_embeddings -from vidxp.capabilities.videoprism.models import get_videoprism_model +from vidxp.capabilities.videoprism.models import ( + get_videoprism_model, + normalize_pooled_output, +) from vidxp.core.contracts import IndexConfig from vidxp.ports import IndexStore, ModelRuntimePort @@ -45,7 +48,7 @@ def videoprism_embedding( inputs = {name: value.to(provider.device) for name, value in inputs.items()} with torch.inference_mode(): features = provider.model.get_text_features(**inputs).pooler_output - features = torch.nn.functional.normalize(features, dim=-1) + features = normalize_pooled_output(features) return features.cpu().numpy().tolist()[0] diff --git a/tests/test_videoprism.py b/tests/test_videoprism.py index 5e7b50a..6b55cd5 100644 --- a/tests/test_videoprism.py +++ b/tests/test_videoprism.py @@ -10,12 +10,22 @@ VideoPrismIndexState, process_videoprism_samples, ) +from vidxp.capabilities.videoprism.models import normalize_pooled_output from vidxp.capabilities.videoprism.specs import VIDEOPRISM_MODEL from vidxp.core.contracts import CancellationToken, IndexConfig from vidxp.core.video import FrameSample, VideoInfo class VideoPrismTests(unittest.TestCase): + def test_provider_pooler_dimension_is_removed(self): + import torch + + embedding = normalize_pooled_output(torch.tensor([[[3.0, 4.0]]])) + + self.assertEqual(tuple(embedding.shape), (1, 2)) + self.assertAlmostEqual(embedding[0, 0].item(), 0.6) + self.assertAlmostEqual(embedding[0, 1].item(), 0.8) + def test_config_rejects_invalid_sampling(self): with self.assertRaises(ValidationError): VideoPrismConfig(sample_fps=0) From f0392fb260fd09012125bfc9ee4278aa055de57c Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 13 Aug 2026 15:51:08 +0500 Subject: [PATCH 4/5] fix(ci): expect VideoPrism in minimal wheel --- utils/verify_runtime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/verify_runtime.py b/utils/verify_runtime.py index ec20bd0..0a8d63b 100644 --- a/utils/verify_runtime.py +++ b/utils/verify_runtime.py @@ -16,7 +16,8 @@ def verify_minimal(executable: str) -> None: from vidxp.capabilities.registry import create_capability_registry require( - create_capability_registry().names() == ("dialogue", "scene", "actor"), + create_capability_registry().names() + == ("dialogue", "scene", "actor", "videoprism"), "minimal wheel does not expose the expected capability registry", ) help_text = subprocess.check_output([executable, "--help"], text=True) From f19b6991062747e9c922438a0bb5591dd391e8fe Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 13 Aug 2026 15:52:53 +0500 Subject: [PATCH 5/5] fix(ci): preserve minimal capability baseline --- utils/verify_runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/verify_runtime.py b/utils/verify_runtime.py index 0a8d63b..1ca099b 100644 --- a/utils/verify_runtime.py +++ b/utils/verify_runtime.py @@ -15,9 +15,9 @@ def require(condition: bool, message: str) -> None: def verify_minimal(executable: str) -> None: from vidxp.capabilities.registry import create_capability_registry + required_capabilities = {"dialogue", "scene", "actor"} require( - create_capability_registry().names() - == ("dialogue", "scene", "actor", "videoprism"), + required_capabilities.issubset(create_capability_registry().names()), "minimal wheel does not expose the expected capability registry", ) help_text = subprocess.check_output([executable, "--help"], text=True)