diff --git a/open_vocabulary_segmentation/geolangsplat/.gitignore b/open_vocabulary_segmentation/geolangsplat/.gitignore
new file mode 100644
index 0000000..f76fb49
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/.gitignore
@@ -0,0 +1,56 @@
+# Python
+__pycache__/
+*.pyc
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.mypy_cache/
+
+# Local query/segmentation outputs and large data artifacts (never commit splats)
+*.ply
+*.npz
+*.csv
+*.png
+*.zip
+*.glscache
+outputs/
+runs/
+
+# ...but do ship the README example figures
+!examples/
+!examples/*.png
+
+# Local benchmark / evaluation run directories
+eval_out/
+fast_bench/
+id_bench/
+robust_out/
+routed_bench/
+frgs_logs/
+
+# Local design notes (not shipped)
+ARCHITECTURE.md
+SLIDES.md
+INTEGRATION.md
+BENCHMARKS.md
+
+# Deferred identity field (seed/click grouping) -- lands in a later change
+geolangsplat/identity.py
+tests/test_identity.py
+
+# Local benchmark / evaluation code and scripts (not part of this example)
+geolangsplat/eval/
+scripts/bench_inference.py
+scripts/bench_fast_query.py
+scripts/bench_identity.py
+scripts/bench_robustness.py
+scripts/bench_scenes_aerial_sat.json
+scripts/eval_scenes.py
+scripts/eval_benchmark.py
+scripts/run_lerf_eval.sh
+scripts/try_instances.py
+scripts/safety_park_demo.sh
+tests/test_eval_align.py
+tests/test_eval_lerf_ovs.py
+tests/test_eval_metrics.py
diff --git a/open_vocabulary_segmentation/geolangsplat/README.md b/open_vocabulary_segmentation/geolangsplat/README.md
new file mode 100644
index 0000000..470ef16
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/README.md
@@ -0,0 +1,310 @@
+# GeoLangSplat
+
+**Training-free, open-vocabulary 3D segmentation for Gaussian splats.** Give it a text
+prompt (`"house"`, `"road"`, `"tree"`, …) and a splat; it returns which Gaussians belong to
+that concept — as a segmented `.ply`, a highlight overlay, or a per-Gaussian report. One
+engine serves **aerial** (oblique drone), **satellite** (near-nadir), and **object/interior**
+(360° inward) captures; the default `auto` recipe reads the scene geometry and synthesizes its
+own views, so the common case is `gls segment scene.ply "thing"` with no flags.
+
+
+
+
+
+gls catalog scene.ply building --> every building becomes its own instance, then lifts out as a .ply
+
+---
+
+## How it works
+
+```
+text prompt
+ |
+ v
+SAM3 segments each view --> alpha-weighted back-projection --> per-Gaussian scores
+ ^ |
+views of the splat competition + cleanup <--+
+(auto-synthesized, or your real photos) |
+ v
+ segmented .ply / overlay .ply / per-Gaussian report
+```
+
+- **Training-free.** No per-scene optimization, no CLIP. We run SAM3 over views of the splat
+ and lift its per-pixel scores into 3D --> one continuous score per Gaussian. A brand-new
+ scene is usable in seconds; quality just tracks SAM3 and the reconstruction.
+- **Views are the quality knob.** The model can only label what SAM3 sees. The `auto` recipe
+ reads the scene and picks the views for you --> flat/top-down captures get a nadir + oblique
+ ladder, rooms and objects get a gravity-up dome. Or point it at the scene's real photos with
+ `--view-source images --sfm`.
+- **One code path, two speeds.** One-shot (default) streams the lift one chunk at a time
+ (encode --> score --> project --> evict), so peak VRAM stays flat no matter how big the scene
+ is. Keep an engine warm with `gls serve` and repeat queries come back in ~1-2 s.
+
+---
+
+## Installation
+
+GeoLangSplat runs inside an existing **fVDB** environment (the fvdb monorepo build, with
+`torch` + `fvdb-reality-capture`; not on PyPI). It is light --> it adds only `numpy`, `pillow`,
+`tyro`, and the `gls` command.
+
+1. **Install into your fVDB env:**
+
+ ```bash
+ source /path/to/fvdb-env/bin/activate
+ pip install -e . # registers the `gls` command
+ ```
+
+2. **Add SAM3** — the 2D segmenter used by `segment`/`bake`. It is not on PyPI: install it from
+ source so that `import sam3` works. It loads lazily, so it is *not* needed to import the
+ package, run the tests (SAM3 is mocked), or run `gls check`.
+
+3. **Point at a checkpoint and run the preflight:**
+
+ ```bash
+ export GEOLANGSPLAT_SAM_CKPT=/path/to/sam3.1_multiplex.pt
+ gls doctor # checks GPU/bf16, fvdb, SAM3, checkpoint
+ ```
+
+For a one-shot provision (e.g. baking a lab image), `setup.sh` downloads the checkpoint
+(`GEOLANGSPLAT_SAM_URL=...`), installs the package, and runs `gls doctor` so the environment
+ships ready — the notebook/first run then needs no installs.
+
+---
+
+## Python / notebook
+
+```python
+from geolangsplat import segment, GeoLangSplatConfig
+
+# single prompt -> per-Gaussian selection, written to an overlay .ply
+res = segment(
+ "scene.ply", "building",
+ config=GeoLangSplatConfig(low_vram=True), # bound VRAM (~6-7 GB)
+ output="ply_overlay", out_path="building.ply",
+)
+print(res.num_selected, "/", res.scores.shape[0])
+
+# multiple prompts -> multi-class labelling (argmax + confidence/ambiguity gates)
+res = segment("scene.ply", ["building", "tree", "grass", "road"], recipe="satellite")
+res.to_report("labels.csv") # index,x,y,z,label_id,label,score (+ legend.json)
+```
+
+`segment(...)` returns a `SegmentResult` with per-Gaussian `scores`, a boolean `selected`
+mask, and (multi-prompt) `label_ids`, all in **input `.ply` order**. Pass a `GaussianSplat3d`
+or a path; every `GeoLangSplatConfig` field is settable (explicit fields beat a recipe). Reuse
+`GeoLangSplatEngine` to build once and query many times.
+
+### Use as a post-processing step
+
+GeoLangSplat is a **self-contained post-processing stage** over any reconstruction: in goes a
+Gaussian splat (a `.ply` or in-memory `fvdb.GaussianSplat3d`, plus an optional COLMAP scene),
+out come per-Gaussian labels in input order. With no per-scene training and no required upstream
+state, it drops in after a reconstruction step (e.g. fvdb-reality-capture) — consume
+`result.selected` / `result.label_ids` directly or persist `to_ply_*` / `to_report`.
+
+Display inline in a notebook without an interactive viewer:
+
+```python
+from IPython.display import Image
+import subprocess; subprocess.run(["gls", "render", "building.ply", "-o", "building.png"])
+Image("building.png")
+```
+
+The overlay `.ply` also opens in the `fvdb.viz` viewer (`gls show building.ply`).
+
+### Segment catalog (notebook object database)
+
+`build_catalog` runs a whole vocabulary in one pass and turns the result into a small,
+browsable database of individual objects: each prompt is segmented, split into spatial
+objects (3D connected components), and objects hit by several prompts (e.g. `car` and
+`vehicle`) are merged and given a stable id. The catalog is a table you browse and pull
+single objects from — handy for picking a specific segment to export downstream.
+
+```python
+from geolangsplat import build_catalog
+
+cat = build_catalog("scene.ply", ["building", "car", "tree", "road"], recipe="satellite")
+cat # rich HTML table inline (id, label, n_gaussians, size, prompts)
+cat.browse() # interactive picker: preview + select objects -> export .ply
+cat.table # the same data as a pandas DataFrame (filter / sort)
+cat.show() # top-down render with each object's id drawn on it
+cat.extract(3, "obj3.ply") # pull one object out as its own .ply
+cat.export_all("scene_catalog/") # objects/_.ply + catalog.csv + labeled .ply
+cat.save("scene_catalog/") # persist; reload with SegmentCatalog.load(dir, model)
+```
+
+`cat.browse()` is the no-id-juggling way to grab segments in a notebook: it shows a live
+preview beside a checklist of every object — click objects to highlight them in the render,
+then hit **Export selected** to write their `.ply`s. If the catalog was built from a warm
+engine, a query box re-runs the vocabulary in place (needs `ipywidgets`). The table is built
+from plain row dicts, so swapping pandas for `cudf` (GPU frame) is a one-liner if a lab needs
+it. Reuse a warm `GeoLangSplatEngine` (`engine.catalog(vocab)`) to build several catalogs of
+the same scene without re-baking.
+
+---
+
+## CLI
+
+```bash
+gls segment scene.ply "road" -O ply_overlay -o road.ply # auto recipe, synthesized views
+gls bake scene.ply --vocab house tree grass road -o labels/ # fixed-vocab multi-class
+gls catalog scene.ply --vocab building car tree road -o catalog/ # ID'd per-object .ply database
+gls check scene.ply # readiness + detected capture
+gls render out.ply -o out.png # montage PNG (good over SSH)
+gls show out.ply # 3D viewer (fvdb.viz)
+```
+
+### Flags
+
+Shared (`segment`/`bake`/`check`/`serve`): `-r/--recipe auto|satellite|satellite_dense|aerial`
+(default `auto`; `aerial` segments **real photos** and needs `-s/--sfm /path/scene`) ·
+`--view-source render|globe|images` · `-u/--up auto|+z|-z|+y|-y|+x|-x` · `--max-views N` ·
+`--vram-budget-gb G` · `-d/--device`.
+
+`segment` query: `-O/--output mask|ply_segmented|ply_overlay|report` · `-o/--out-path` ·
+`-t/--select ` (the main knob) · `--peak <0..1>` · `--compete` (suppress near-synonyms) ·
+`--inside-out` (interior scenes) · `--profile`.
+
+VRAM / streaming: `--low-vram`/`--no-low-vram` · `--stream-chunk N` (time↔VRAM) ·
+`--stream-early-stop auto|on|off` · `--cache-dtype auto|amp|fp16|bf16`.
+
+Other commands: `render` adds `-n/--n-views`, `-z/--zoom` (<1 pulls in, >1 backs out),
+`--globe/--no-globe`; `serve` adds `-b/--background`, `--fast-views N`; `bake` and `catalog`
+take `--vocab w1 w2 …` (or `--vocab-file`), and `catalog` adds `--iou <0..1>` (cross-prompt
+merge threshold) and `-o/--out` (output folder). Bash completion:
+`echo "source $(pwd)/completions/gls.bash" >> ~/.bashrc`.
+
+### Execution modes
+
+One-shot (default) builds, answers, and exits with bounded VRAM:
+
+```bash
+gls segment scene.ply "building" -O ply_overlay -o b.ply
+```
+
+Memory-bound? Lower `--stream-chunk`, drop `lift_res`, or pick a recipe with fewer views.
+
+Keep an engine warm for rapid repeated queries:
+
+```bash
+gls serve scene.ply -r satellite -b # build once, detach
+gls segment scene.ply "building" -o b.ply # instant (auto-attaches)
+gls status # what's running
+gls stop scene.ply # free it (or `gls stop` = all)
+```
+
+You only ever type `gls segment`; building-and-exiting vs attaching to a running engine is
+automatic. Once an engine is up, build flags (`-r`, `-s`) are ignored; query flags (`-t`,
+`--compete`, `-O`/`-o`) apply. The engine always releases the GPU (idle timeout, `gls stop`,
+Ctrl-C, or crash).
+
+**Concept competition** (`--compete` / `config.compete`): a Gaussian is kept only if it beats
+the best distractor prompt by `margin` (stops a query bleeding into near-synonyms — e.g.
+`building` leaking onto an empty pool that reads as `water`). Works in both execution modes;
+`bake` competes implicitly via argmax.
+
+**Dual-head fusion** (`config.dual_head`, off by default): SAM3 hands back two maps from one
+pass --> an *instance* head for countable "things" (cars, buildings) and a *semantic* head for
+"stuff" (ground, road, vegetation). The instance head drops anything it isn't sure is an
+object, so amorphous classes can come back empty --> blending the semantic map back in recovers
+them at no extra cost. Still training-free. On indoor/object scenes, `dual_head=True,
+sem_weight=0.5` works best.
+
+---
+
+## Scene quality
+
+Results depend on the reconstruction — the method can only label Gaussians the views observe.
+Check readiness first (geometry only, no SAM3 weights):
+
+```bash
+gls check scene.ply -r satellite # coverage, views/gaussian, verdict: good|fair|poor
+```
+
+```python
+from geolangsplat import assess_scene
+assess_scene(model, recipe="satellite") # same dict the CLI prints
+```
+
+A `poor` verdict usually means: raise the view count, use a closer/denser recipe, switch
+`view_source`, or improve the reconstruction. If a clearly-visible object returns 0 Gaussians,
+it is usually wording — try a synonym (`"oven mitt"` vs `"mitten"`).
+
+---
+
+## Where this fits
+
+GeoLangSplat is a post-processing step for `fvdb-reality-capture`: reconstruct a scene -->
+hand the splat to GeoLangSplat --> get per-Gaussian labels back. There is no per-scene training
+and nothing to wire up beforehand, so it drops in right after reconstruction alongside the
+other post-processing tools (mesh, points, evaluation). The bundled `viewer/` powers the
+optional `gls explore` UI for live prompt tuning and catalog browsing; the stable surface is
+`gls segment` / `gls catalog` and the `segment` / `build_catalog` APIs.
+
+---
+
+## Tests & layout
+
+```bash
+pip install -e ".[test]" && pytest tests/ # 120 tests, CPU-only, SAM 3 / fVDB mocked, no GPU
+```
+
+```
+geolangsplat/
+ config.py # GeoLangSplatConfig + recipes (auto/satellite/.../aerial)
+ autoview.py # geometry -> view plan: capture detect, up-axis RANSAC, dome rings, framing
+ cameras.py # intrinsics, orbit / look-at / inside-out cameras
+ views.py # view generation: synthesized render | dome | real photos (--sfm)
+ sam3.py # SAM3 encode + promptable scoremap
+ lift.py # alpha-weighted back-projection + low-VRAM streaming lift
+ select.py # selection, competition, cleanup, multi-class labels
+ instances.py # split a selection into spatial objects (3D connected components)
+ catalog.py # multi-prompt object catalog: cluster + ID + per-object .ply (notebook db)
+ engine.py # build-once / query-many engine (one-shot streaming + warm modes)
+ api.py # segment() / build_catalog() -> the public surface
+ outputs.py # ply / report writers
+ cli/ # segment|bake|catalog|check|doctor|serve|status|stop|show|render|explore
+ viewer/ # web query UI that backs `gls explore` (optional)
+```
+
+---
+
+## Scope & scale
+
+GeoLangSplat is a **reference implementation**, not a distributed engine: it shows the
+training-free path end-to-end on a single GPU, clearly enough to lift and scale.
+
+- **Single GPU, single process** --> no multi-GPU sharding or multi-node; the build encodes
+ every view's SAM 3 features and lifts them on one device.
+- **VRAM-bound, not out-of-core** --> per-view features + per-Gaussian scores live in device
+ memory; `--low-vram` streaming caps peak to ~one view, but there is no spatial tiling, so a
+ city-scale splat (tens of millions of Gaussians / hundreds of views) will not fit.
+- **Auto-view targets one coherent capture** --> the synthesized orbit/ladder/dome is fit to a
+ single site or block; a large spread-out mosaic under-samples unless you grow the view count
+ (and with it build time + VRAM).
+
+Best fit: a park, a few blocks, an object, or an interior. Tested on **SafetyPark** (oblique
+drone) and the **JAX_\*** WorldView-3 satellite scenes; the object/interior recipes were
+exercised on Mip-NeRF 360 and LERF-OVS captures.
+
+---
+
+## References
+
+**Methods & models**
+
+- **SAM 3** — the promptable 2D segmenter GeoLangSplat runs over views of the splat (Meta; not on PyPI): https://github.com/facebookresearch/sam3
+- **LangSplat / LangSplatV2** — the open-vocabulary language-field lineage this builds on; see the sibling [`langsplatv2/`](../langsplatv2) example (Qin et al., CVPR 2024).
+- **CLIP** — the vision-language backbone behind most open-vocabulary segmentation; GeoLangSplat skips the per-scene CLIP field and prompts SAM 3 directly (Radford et al., 2021): https://github.com/openai/CLIP
+- **SegEarth-OV / SegEarth-OV3** — related training-free open-vocabulary segmentation for remote sensing imagery (Li et al., CVPR 2025): https://github.com/earth-insights/SegEarth-OV-3
+- **fVDB** — the 3D representation, rendering, and alpha-weighted back-projection that lifts 2D scores to per-Gaussian labels: https://github.com/openvdb/fvdb-core
+
+**Datasets**
+
+- **US3D / DFC2019** — WorldView-3 satellite imagery over Jacksonville (the `JAX_*` scenes), from the 2019 IEEE GRSS Data Fusion Contest; imagery courtesy of Maxar/DigitalGlobe (Bosch et al., WACV 2019, arXiv:1811.08739): https://ieee-dataport.org/open-access/data-fusion-contest-2019-dfc2019
+- **Mip-NeRF 360** and **LERF-OVS** — ground/object 360 captures used to exercise the globe / inside-out recipes.
diff --git a/open_vocabulary_segmentation/geolangsplat/completions/gls.bash b/open_vocabulary_segmentation/geolangsplat/completions/gls.bash
new file mode 100644
index 0000000..21a17f5
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/completions/gls.bash
@@ -0,0 +1,59 @@
+# Bash tab-completion for the `gls` CLI.
+#
+# Install (one time):
+# echo "source /path/to/geolangsplat/completions/gls.bash" >> ~/.bashrc
+# source ~/.bashrc
+#
+# Then:
+# gls -> segment bake catalog check doctor serve show status stop render explore
+# gls segment x.ply foo -r sat -> satellite satellite_dense
+# gls segment ... -O -> mask ply_segmented ply_overlay report
+# .ply / paths complete as filenames.
+
+_gls_complete() {
+ local cur prev
+ cur="${COMP_WORDS[COMP_CWORD]}"
+ prev="${COMP_WORDS[COMP_CWORD-1]}"
+
+ local subcommands="segment bake catalog check doctor serve show status stop render explore"
+ local recipes="auto satellite satellite_dense aerial"
+ local outputs="mask ply_segmented ply_overlay report"
+ local view_sources="render images globe"
+ local cache_dtypes="auto amp fp16 bf16"
+ local early_stops="auto on off"
+
+ case "$prev" in
+ -r|--recipe)
+ COMPREPLY=( $(compgen -W "$recipes" -- "$cur") ); return 0 ;;
+ -O|--output)
+ COMPREPLY=( $(compgen -W "$outputs" -- "$cur") ); return 0 ;;
+ --view-source)
+ COMPREPLY=( $(compgen -W "$view_sources" -- "$cur") ); return 0 ;;
+ --cache-dtype)
+ COMPREPLY=( $(compgen -W "$cache_dtypes" -- "$cur") ); return 0 ;;
+ --stream-early-stop)
+ COMPREPLY=( $(compgen -W "$early_stops" -- "$cur") ); return 0 ;;
+ -o|--out-path|-s|--sfm)
+ COMPREPLY=( $(compgen -f -- "$cur") ); return 0 ;;
+ esac
+
+ # first token after `gls` -> the subcommand
+ if [ "$COMP_CWORD" -eq 1 ]; then
+ COMPREPLY=( $(compgen -W "$subcommands" -- "$cur") ); return 0
+ fi
+
+ # flags when the user has started typing one
+ if [[ "$cur" == -* ]]; then
+ COMPREPLY=( $(compgen -W "-r --recipe -O --output -o --out-path -t --select --keep-alive -b --background \
+ --view-source -s --sfm --compete --profile -d --device \
+ --low-vram --no-low-vram --cache-dtype --stream-early-stop --stream-chunk \
+ --view-cap --max-views --vram-budget-gb -u --up --fast-views \
+ --inside-out --peak -h --help" -- "$cur") )
+ return 0
+ fi
+
+ # default: filenames (model .ply, output paths)
+ COMPREPLY=( $(compgen -f -- "$cur") )
+}
+
+complete -o filenames -F _gls_complete gls
diff --git a/open_vocabulary_segmentation/geolangsplat/examples/.gitkeep b/open_vocabulary_segmentation/geolangsplat/examples/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/open_vocabulary_segmentation/geolangsplat/examples/building0_safetypark_cutout.png b/open_vocabulary_segmentation/geolangsplat/examples/building0_safetypark_cutout.png
new file mode 100644
index 0000000..543b2e2
Binary files /dev/null and b/open_vocabulary_segmentation/geolangsplat/examples/building0_safetypark_cutout.png differ
diff --git a/open_vocabulary_segmentation/geolangsplat/examples/building1_safetypark_cutout.png b/open_vocabulary_segmentation/geolangsplat/examples/building1_safetypark_cutout.png
new file mode 100644
index 0000000..d5b8f68
Binary files /dev/null and b/open_vocabulary_segmentation/geolangsplat/examples/building1_safetypark_cutout.png differ
diff --git a/open_vocabulary_segmentation/geolangsplat/examples/catalog_safetypark_topdown.png b/open_vocabulary_segmentation/geolangsplat/examples/catalog_safetypark_topdown.png
new file mode 100644
index 0000000..0638b90
Binary files /dev/null and b/open_vocabulary_segmentation/geolangsplat/examples/catalog_safetypark_topdown.png differ
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/__init__.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/__init__.py
new file mode 100644
index 0000000..4cd435e
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/__init__.py
@@ -0,0 +1,78 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""GeoLangSplat: unified, training-free open-vocabulary 3D segmentation for
+Gaussian splats.
+
+A text prompt is segmented in 2D (SAM3) over a set of views of the splat, then
+lifted to per-Gaussian scores by alpha-weighted back-projection. The same engine
+serves both aerial (oblique drone) and satellite (near-nadir) captures; only the
+recipe presets differ.
+
+Typical use::
+
+ from fvdb import GaussianSplat3d
+ from geolangsplat import segment
+
+ model = GaussianSplat3d.from_ply("scene.ply")
+ result = segment(model, "house", recipe="satellite", output="ply_overlay",
+ out_path="house.ply")
+ print(result.num_selected)
+"""
+from __future__ import annotations
+
+import time as _time
+
+# Single source of truth is pyproject's [project].version; read it from the installed
+# metadata when available (pip install) and fall back to a literal for a bare checkout
+# / zip drop-in (e.g. shipped into a DLI lab image without `pip install`).
+try: # pragma: no cover - trivial
+ from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+ try:
+ __version__ = _pkg_version("fvdb-geolangsplat")
+ except PackageNotFoundError:
+ __version__ = "0.0.1"
+except Exception: # pragma: no cover
+ __version__ = "0.0.1"
+
+# Captured at first import (≈ process start, before torch is imported). Lets the
+# CLI report total wall = python+torch import + CUDA init + engine work, so the
+# fixed per-invocation startup cost (which a warm engine amortizes) is visible.
+_PROCESS_T0 = _time.perf_counter()
+
+from .config import RECIPES, GeoLangSplatConfig, apply_recipe, list_recipes
+from .errors import GeoLangSplatError
+
+__all__ = [
+ "__version__",
+ "GeoLangSplatConfig",
+ "RECIPES",
+ "apply_recipe",
+ "list_recipes",
+ "segment",
+ "assess_scene",
+ "SegmentResult",
+ "build_catalog",
+ "SegmentCatalog",
+ "CatalogObject",
+ "GeoLangSplatError",
+]
+
+# Lazily expose the API (which imports torch/engine) so that just importing the
+# package -- e.g. for the CLI's lightweight paths that only talk to a running
+# daemon (segment-attach, status, stop) -- does NOT pay torch import + CUDA init.
+_LAZY = {"segment", "assess_scene", "SegmentResult", "build_catalog"}
+_LAZY_CATALOG = {"SegmentCatalog", "CatalogObject"}
+
+
+def __getattr__(name: str):
+ if name in _LAZY:
+ from . import api
+
+ return getattr(api, name)
+ if name in _LAZY_CATALOG:
+ from . import catalog
+
+ return getattr(catalog, name)
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/__main__.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/__main__.py
new file mode 100644
index 0000000..56ee992
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/__main__.py
@@ -0,0 +1,8 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Enable `python -m geolangsplat ...` (used to spawn the background daemon)."""
+from .cli import gls
+
+if __name__ == "__main__":
+ gls()
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/api.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/api.py
new file mode 100644
index 0000000..b5fcdf1
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/api.py
@@ -0,0 +1,213 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Public, pure API: a text prompt in, per-Gaussian results out.
+
+``segment`` is the one entry point. With a single prompt it returns a per-Gaussian
+selection; with several prompts it returns a multi-class labelling (argmax with
+confidence/ambiguity gates). Results carry helpers to write the segmented ``.ply``,
+a highlight-overlay ``.ply``, or a per-Gaussian report.
+"""
+from __future__ import annotations
+
+import copy
+import pathlib
+from dataclasses import dataclass, field
+from typing import Literal
+
+import numpy as np
+import torch
+
+from .config import GeoLangSplatConfig, apply_recipe
+from .engine import GeoLangSplatEngine, load_or_build_engine
+from . import outputs as _out
+from .select import assign_labels
+
+OutputMode = Literal["mask", "ply_segmented", "ply_overlay", "report"]
+
+
+@dataclass
+class SegmentResult:
+ """Per-Gaussian segmentation result, in input ``.ply`` order.
+
+ Attributes
+ ----------
+ prompts: the query word(s).
+ scores: ``[N]`` for a single prompt, ``[N, C]`` for multiple.
+ selected: ``[N]`` bool selection mask (for multi-prompt: any labeled Gaussian).
+ label_ids: ``[N]`` long, class index per Gaussian (``-1`` = unlabeled);
+ ``None`` for a single prompt.
+ """
+
+ prompts: list[str]
+ scores: torch.Tensor
+ selected: torch.Tensor
+ label_ids: torch.Tensor | None
+ config: GeoLangSplatConfig
+ stats: dict = field(default_factory=dict)
+ metadata: dict = field(default_factory=dict)
+ _model: object = None
+
+ @property
+ def num_selected(self) -> int:
+ return int(self.selected.sum())
+
+ @property
+ def means_np(self) -> np.ndarray:
+ return self._model.means.detach().cpu().numpy()
+
+ # -- writers ------------------------------------------------------------
+
+ def to_ply_segmented(self, path) -> int:
+ return _out.write_ply_segmented(self._model, self.selected, path, metadata=self.metadata)
+
+ def to_ply_overlay(self, path, color=None, blend=None) -> int:
+ if self.label_ids is not None:
+ return _out.write_ply_labels(self._model, self.label_ids, path, metadata=self.metadata)
+ color = self.config.highlight_color if color is None else color
+ blend = self.config.blend if blend is None else blend
+ return _out.write_ply_overlay(
+ self._model, self.selected, path, color=color, blend=blend, metadata=self.metadata
+ )
+
+ def to_report(self, path) -> None:
+ means = self.means_np
+ if self.label_ids is not None:
+ labels = self.label_ids.detach().cpu().numpy()
+ scores = self.scores.detach().cpu().numpy()
+ else:
+ labels = np.where(self.selected.detach().cpu().numpy(), 0, -1).astype(np.int64)
+ scores = self.scores.detach().cpu().numpy()
+ path = pathlib.Path(path)
+ if path.suffix == ".npz":
+ _out.write_report_npz(means, labels, scores, self.prompts, path)
+ else:
+ _out.write_report_csv(means, labels, scores, self.prompts, path)
+
+
+def build_catalog(
+ model,
+ vocab=None,
+ *,
+ config: GeoLangSplatConfig | None = None,
+ recipe: str | None = None,
+ engine: GeoLangSplatEngine | None = None,
+ scorer=None,
+ select: float | None = None,
+ iou: float | None = None,
+ link_frac: float | None = None,
+ min_size: int | None = None,
+):
+ """Build a browsable, ID'd object catalog for ``model`` over a prompt ``vocab``.
+
+ Runs the vocabulary in one pass and clusters every prompt's segments into
+ distinct physical objects, returning a
+ :class:`~geolangsplat.catalog.SegmentCatalog` (a table + per-object ``.ply``
+ extraction). ``vocab`` defaults to a general scene vocabulary. Reuse a warm
+ ``engine`` for repeat catalogs of the same scene.
+ """
+ cfg = engine.cfg if engine is not None else _resolve_config(config, recipe)
+ eng = engine if engine is not None else load_or_build_engine(model, cfg, scorer=scorer)
+ return eng.catalog(vocab, select=select, iou=iou, link_frac=link_frac, min_size=min_size)
+
+
+def assess_scene(
+ model,
+ *,
+ config: GeoLangSplatConfig | None = None,
+ recipe: str | None = None,
+) -> dict:
+ """Geometry-only readiness check: does this reconstruction look segmentable?
+
+ Builds the views + lift cache (no SAM3 weights required) and returns coverage
+ statistics plus a ``verdict`` of ``"good" | "fair" | "poor"``. Run this before
+ a full segment to catch sparse/low-quality scenes early.
+ """
+ cfg = _resolve_config(config, recipe)
+ eng = GeoLangSplatEngine(model, cfg, build=False)
+ eng.build_geometry()
+ return eng.assess()
+
+
+def _resolve_config(config: GeoLangSplatConfig | None, recipe: str | None) -> GeoLangSplatConfig:
+ cfg = copy.deepcopy(config) if config is not None else GeoLangSplatConfig()
+ if recipe:
+ applied = apply_recipe(cfg, recipe)
+ if applied:
+ print(f"[recipe] {recipe}: filled {applied}", flush=True)
+ return cfg
+
+
+def segment(
+ model,
+ prompts: "str | list[str]",
+ *,
+ cameras=None, # reserved; ground-truth image views are loaded from config.sfm (COLMAP)
+ config: GeoLangSplatConfig | None = None,
+ recipe: str | None = None,
+ output: OutputMode = "mask",
+ out_path=None,
+ engine: GeoLangSplatEngine | None = None,
+ scorer=None,
+) -> SegmentResult:
+ """Segment ``model`` by ``prompts``.
+
+ Parameters
+ ----------
+ model: a ``GaussianSplat3d`` or a path to a ``.ply``.
+ prompts: a single prompt (selection) or several (multi-class labelling).
+ config: a :class:`GeoLangSplatConfig`; defaults are used if omitted.
+ recipe: ``"aerial" | "satellite" | "satellite_dense"`` preset (fills unset fields).
+ output: ``"mask"`` (no file), ``"ply_segmented"``, ``"ply_overlay"`` or ``"report"``.
+ out_path: where to write when ``output != "mask"``.
+ engine: reuse a prebuilt engine (skips the one-time build); keep one warm
+ (``gls serve`` / ``GeoLangSplatEngine``) for fast repeated queries.
+ scorer: inject a SAM3 scorer (e.g. a mock in tests).
+ """
+ single = isinstance(prompts, str)
+ plist = [prompts] if single else list(prompts)
+ if not plist:
+ raise ValueError("prompts must be a non-empty string or list of strings")
+
+ cfg = engine.cfg if engine is not None else _resolve_config(config, recipe)
+ if engine is not None:
+ eng = engine
+ else:
+ eng = load_or_build_engine(model, cfg, scorer=scorer)
+
+ stats: dict = {}
+ if single:
+ scores, selected, dt = eng.query(plist[0])
+ label_ids = None
+ stats["query_seconds"] = dt
+ stats["num_selected"] = int(selected.sum())
+ else:
+ scores = eng.bake_vocab(plist) # [N, C]
+ denom = eng.denom if eng.denom is not None else eng.seen
+ label_ids, lab_stats = assign_labels(scores, denom, cfg.min_weight, cfg.tau, cfg.delta)
+ selected = label_ids >= 0
+ stats.update(lab_stats)
+
+ result = SegmentResult(
+ prompts=plist,
+ scores=scores,
+ selected=selected,
+ label_ids=label_ids,
+ config=cfg,
+ stats=stats,
+ metadata=getattr(eng, "metadata", {}) or {},
+ _model=eng.model,
+ )
+
+ if output != "mask":
+ if out_path is None:
+ raise ValueError(f"output={output!r} requires out_path")
+ if output == "ply_segmented":
+ result.to_ply_segmented(out_path)
+ elif output == "ply_overlay":
+ result.to_ply_overlay(out_path)
+ elif output == "report":
+ result.to_report(out_path)
+ else:
+ raise ValueError(f"unknown output mode {output!r}")
+ return result
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/autoview.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/autoview.py
new file mode 100644
index 0000000..161d01b
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/autoview.py
@@ -0,0 +1,395 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Automatic view-generation config from scene geometry.
+
+The named recipes (``satellite``, ``aerial`` ...) hard-code absolute orbit radii
+that only frame scenes at one particular scale. The ``auto`` recipe instead
+*measures* the splat and derives a render config scaled to that scene, so the
+same command frames a tiny object scene and a city-block satellite tile alike.
+
+Flow: :func:`measure_geometry` probes the point cloud (robustly, ignoring
+floaters), :func:`recommend_view_config` turns those stats into orbit tiers and
+selection defaults, and :func:`apply_auto_view_config` fills them into a config
+(respecting any value the caller set explicitly, exactly like ``apply_recipe``).
+This is deterministic, so ``gls check`` and ``gls segment`` derive the *same*
+config from a scene -- check just shows you what segment will do.
+"""
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+
+from .cameras import up_vector
+from .config import GeoLangSplatConfig, Tier
+
+# --- up-axis (RANSAC ground-plane) estimation -----------------------------
+_UP_MIN_INLIERS = 0.3 # min inlier fraction to trust an estimated ground plane
+_UP_PLANE_TOL = 0.02 # plane inlier band as a fraction of the bbox diagonal
+_UP_ITERS = 300
+_UP_SAMPLE = 20000 # subsample cap for the RANSAC fit (speed)
+
+# --- geometry probing ------------------------------------------------------
+_LO_Q = 0.02
+_HI_Q = 0.98
+
+# --- aerial recommender ----------------------------------------------------
+_OVERVIEW_RADIUS_FRAC = 1.25
+_OBLIQUE_RADIUS_FRAC = 0.55
+_GRID_MIN = 2
+_GRID_MAX = 8
+_OVERVIEW_SHARE = 0.3
+_VRAM_BASE_GB = 6
+_VRAM_PER_VIEW_GB = 0.16
+
+# --- object-centric recommender -------------------------------------------
+_OBJECT_FLATNESS = 0.5 # min(extent)/max(extent) above this => roughly isotropic => object
+_OBJECT_RADIUS_FRAC = 1.1
+_OBJECT_FOV = 55
+_OBJECT_ELEVATIONS = (-30, 0, 30, 60)
+_OBJECT_AZIMUTH_MIN = 8
+_OBJECT_AZIMUTH_MAX = 36
+
+# Dome orbit: elevation rings looking DOWN at the scene core, from low-oblique
+# (side-on, to catch object *flanks*) up to near-nadir. We stop above the horizon
+# (lowest ring well >0 deg): exactly-level and below-horizon angles see through to
+# the unbounded background halo and come out blurry, so we never go under or on-
+# level. The lower rings ride closer to the subject's sides; the upper rings cap
+# coverage from above. The orbit radius is auto-framed from the dense core.
+_GLOBE_ELEVATIONS = (18.0, 32.0, 46.0, 60.0, 74.0, 88.0)
+_GLOBE_AZ_MIN = 3
+
+
+def globe_rings(budget: int, elevations: tuple[float, ...] = _GLOBE_ELEVATIONS):
+ """Distribute ``budget`` views across the dome's elevation rings (azimuth count
+ per ring proportional to ``cos(elevation)``, so lower rings -- which see more of
+ the scene -- get more azimuths and the near-nadir cap isn't over-sampled).
+ Returns ``(elevations, az_counts)``; ``sum(az_counts)`` is approximately
+ ``budget``.
+ """
+ weights = [max(0.25, math.cos(math.radians(e))) for e in elevations]
+ total = sum(weights)
+ az_counts = [max(_GLOBE_AZ_MIN, int(round(budget * w / total))) for w in weights]
+ return list(elevations), az_counts
+
+
+@dataclass
+class GeometryStats:
+ """Robust scene-extent measurements used to derive a view plan."""
+
+ n: int
+ dx: float
+ dy: float
+ dz: float
+ footprint: float
+ aspect: float
+ vertical_ratio: float
+ flatness: float
+ span: float
+
+
+def measure_geometry(means) -> GeometryStats:
+ """Probe scene extents robustly (quantile-trimmed to ignore floaters)."""
+ m = means.detach().float()
+ n = int(m.shape[0])
+ q = torch.tensor([_LO_Q, _HI_Q], device=m.device, dtype=m.dtype)
+ lo_hi = torch.quantile(m, q, dim=0)
+ ext = (lo_hi[1] - lo_hi[0]).clamp_min(1e-6)
+ dx = float(ext[0])
+ dy = float(ext[1])
+ dz = float(ext[2])
+ footprint = max(dx, dy)
+ aspect = footprint / max(min(dx, dy), 1e-6)
+ flatness = min(dx, dy, dz) / max(dx, dy, dz, 1e-6)
+ return GeometryStats(
+ n=n,
+ dx=dx,
+ dy=dy,
+ dz=dz,
+ footprint=footprint,
+ aspect=aspect,
+ vertical_ratio=dz / max(footprint, 1e-6),
+ flatness=flatness,
+ span=float(ext.norm()),
+ )
+
+
+def estimate_up(means, seed: int = 0) -> tuple[np.ndarray, float]:
+ """Estimate the gravity-up axis from geometry via RANSAC ground-plane fit.
+
+ Real captures (rooms, tabletops, terrain, city tiles) contain a dominant flat
+ surface; its normal is gravity. Returns ``(unit_up_vec, confidence)`` where
+ confidence is the plane's inlier fraction. The sign is chosen so the bulk of the
+ scene sits *above* the plane. Deterministic for a given seed.
+ """
+ m = means.detach().float().cpu().numpy()
+ n = m.shape[0]
+ if n < 3:
+ return np.array([0.0, 0.0, 1.0]), 0.0
+ rng = np.random.default_rng(seed)
+ # Drop the unbounded background shell first: in 360/interior captures a sparse
+ # far halo of Gaussians dominates the bbox and corrupts the plane fit (the
+ # ground plane should come from the dense core, not the floaters).
+ lo = np.quantile(m, 0.02, axis=0)
+ hi = np.quantile(m, 0.98, axis=0)
+ core = m[np.all((m >= lo) & (m <= hi), axis=1)]
+ if core.shape[0] >= 3:
+ m = core
+ if m.shape[0] > _UP_SAMPLE:
+ m = m[rng.choice(m.shape[0], _UP_SAMPLE, replace=False)]
+ diag = float(np.linalg.norm(m.max(0) - m.min(0)))
+ if diag <= 0:
+ return np.array([0.0, 0.0, 1.0]), 0.0
+ tol = _UP_PLANE_TOL * diag
+ best_n, best_in = None, 0
+ for _ in range(_UP_ITERS):
+ idx = rng.choice(m.shape[0], 3, replace=False)
+ p0, p1, p2 = m[idx]
+ nrm = np.cross(p1 - p0, p2 - p0)
+ ln = np.linalg.norm(nrm)
+ if ln < 1e-9:
+ continue
+ nrm = nrm / ln
+ d = np.abs((m - p0) @ nrm)
+ inl = int((d < tol).sum())
+ if inl > best_in:
+ best_n, best_in = nrm, inl
+ if best_n is None:
+ return np.array([0.0, 0.0, 1.0]), 0.0
+ conf = best_in / float(m.shape[0])
+ # Orient the normal "up" by where the STRUCTURE rises relative to the ground:
+ # isolate the dominant ground slab (the half-max-width band around the density
+ # peak along the normal), then compare the mass *beyond* it on each side. Above
+ # the ground sit buildings/trees (a lot of mass); below it is near-empty (only
+ # sparse sub-ground floaters) -- so the heavier side is up. Excluding the slab
+ # itself is the crucial part: the ground peak is roughly symmetric and dominates
+ # any thin band right at the mode (after subsampling its sign is just noise),
+ # while skew/mean-position get tipped by heavy below-ground floaters. Mass beyond
+ # the slab is the stable signal (verified +z on JAX_264/JAX_175 across seeds).
+ # A wrong sign puts the whole auto orbit *below* the scene, so cameras look up
+ # through the ground and miss every rooftop (JAX_264: 0 buildings, coverage 0.38).
+ ts = m @ best_n
+ lo_t, hi_t = (float(x) for x in np.quantile(ts, [0.02, 0.98]))
+ if hi_t > lo_t:
+ hist, edges = np.histogram(ts, bins=256, range=(lo_t, hi_t))
+ k = int(hist.argmax())
+ half = hist[k] / 2.0
+ lo_i = k
+ while lo_i > 0 and hist[lo_i] > half:
+ lo_i -= 1
+ hi_i = k
+ while hi_i < len(hist) - 1 and hist[hi_i] > half:
+ hi_i += 1
+ slab_lo, slab_hi = edges[lo_i], edges[hi_i + 1]
+ above = int((ts > slab_hi).sum()) # structure rising off the ground
+ below = int((ts < slab_lo).sum()) # sparse sub-ground floaters
+ if below > above: # heavier side is below the slab -> normal points down
+ best_n = -best_n
+ return best_n.astype(np.float64), float(conf)
+
+
+def resolve_up(up_spec, means) -> np.ndarray:
+ """Resolve a config ``up`` to a concrete unit vector.
+
+ Explicit axis (``"+z"``) or a 3-vector is honoured as-is; ``auto``/unset triggers
+ geometry estimation, falling back to +z when no confident ground plane is found.
+ """
+ if isinstance(up_spec, (list, tuple, np.ndarray)):
+ return up_vector(up_spec)
+ s = str(up_spec).lower().replace(" ", "")
+ if s in ("", "auto", "none"):
+ vec, conf = estimate_up(means)
+ if conf >= _UP_MIN_INLIERS:
+ return vec
+ return np.array([0.0, 0.0, 1.0])
+ return up_vector(s)
+
+
+def detect_capture(stats: GeometryStats) -> str:
+ """Classify the capture as ``"aerial"`` (flat, z-up, top-down) or ``"object"``
+ (roughly isotropic, arbitrary frame, 360 ring) from extent isotropy."""
+ if stats.flatness >= _OBJECT_FLATNESS:
+ return "object"
+ return "aerial"
+
+
+def budget_views(max_views, vram_budget_gb) -> int:
+ """Resolve the effective view budget.
+
+ If ``vram_budget_gb`` is set, convert it to a view count via the rough VRAM
+ model; otherwise use ``max_views`` directly. Clamped to a sane range.
+ """
+ if vram_budget_gb and vram_budget_gb > 0:
+ n = int((vram_budget_gb - _VRAM_BASE_GB) / _VRAM_PER_VIEW_GB)
+ return max(40, min(400, n))
+ return max(8, int(max_views))
+
+
+def _grid_for_budget(budget: int, n_elev: int, num_azimuth: int) -> int:
+ """Largest square grid whose view count (grid^2 * n_elev * azimuths) fits the
+ per-tier budget, clamped to [_GRID_MIN, _GRID_MAX]."""
+ per_center = max(1, n_elev * num_azimuth)
+ g = int(math.isqrt(max(1, budget // per_center)))
+ return max(_GRID_MIN, min(_GRID_MAX, g))
+
+
+def _oblique_angles(vertical_ratio: float):
+ """Pick oblique azimuth count + elevations from how much vertical structure
+ the scene has. Taller structure -> more headings (so every facade is seen) and
+ lower (more side-on) elevations."""
+ if vertical_ratio > 0.1:
+ return (4, (52, 34))
+ if vertical_ratio > 0.04:
+ return (3, (54, 38))
+ return (2, (56, 44))
+
+
+def _tier_views(tier: Tier) -> int:
+ return tier.grid * tier.grid * len(tier.elevations) * tier.num_azimuth
+
+
+def recommend_view_config(stats: GeometryStats, budget: int) -> dict:
+ """Derive an orbit config scaled to ``stats``, sized near ``budget`` total views.
+
+ Dispatches on capture type: a flat z-up aerial slab gets a nadir + oblique
+ top-down orbit; a roughly isotropic object-centric scene (arbitrary frame, 360
+ ring) gets a spherical orbit around the centroid. Returns config fields to fill
+ (``view_source``, ``tiers``, ``select``, ``margin``) plus ``capture``,
+ ``n_views_planned`` and a human-readable ``rationale``.
+ """
+ if detect_capture(stats) == "object":
+ return _recommend_object(stats, budget)
+ return _recommend_aerial(stats, budget)
+
+
+def _recommend_object(stats: GeometryStats, budget: int) -> dict:
+ """Upper-dome orbit around the core for object-centric / indoor captures.
+
+ The eye sweeps full azimuth rings at elevations looking DOWN at the core
+ (oblique -> near-nadir). A bare splat only renders sharply near its original
+ inward/above camera region; horizontal and below-horizon angles see through to
+ the unbounded background and come out blurry, so the dome stays in the reliable
+ region. The radius is auto-framed from the splat's dense core at render time
+ rather than guessed from a fixed multiple of the extent (which over/under-zooms
+ depending on how the scene was scaled).
+ """
+ elevs, az_counts = globe_rings(budget)
+ n_views = sum(az_counts)
+ # Placeholder tier (radius is auto-framed at render; kept only as a fallback if
+ # globe rendering is unavailable for some reason).
+ tier = Tier(
+ radius=round(stats.span * _OBJECT_RADIUS_FRAC, 2),
+ elevations=tuple(float(e) for e in elevs),
+ fov_deg=_OBJECT_FOV,
+ grid=1,
+ num_azimuth=max(az_counts),
+ )
+ rationale = [
+ f"capture: object-centric (flatness {stats.flatness:.2f} >= {_OBJECT_FLATNESS}) -> upper-dome orbit (auto-framed)",
+ f"span {stats.span:.1f} (x={stats.dx:.1f}, y={stats.dy:.1f}, z={stats.dz:.1f})",
+ f"view budget {budget} -> {n_views} views across {len(elevs)} elevation rings "
+ f"({elevs[0]:.0f}..{elevs[-1]:.0f} deg, looking down)",
+ "radius: auto-framed at render (zoom out until the scene shrinks, then one step in)",
+ ]
+ return {
+ "view_source": "globe",
+ "tiers": (tier,),
+ "select": 0.33,
+ "margin": 0.1,
+ "capture": "object",
+ "n_views_planned": n_views,
+ "rationale": rationale,
+ }
+
+
+def _recommend_aerial(stats: GeometryStats, budget: int) -> dict:
+ """Smart nadir + oblique-multi-azimuth orbit scaled to ``stats``, sized
+ to land near ``budget`` total views (the cost knob; VRAM/time/query ~ view count).
+
+ The capture has two tiers, mirroring photogrammetry practice:
+
+ * **overview** -- high and near-nadir, frames the whole footprint for ground/
+ roof coverage. Cheap (one altitude shot already sees the tile), so it gets a
+ small share of the budget.
+ * **oblique** -- moves in and looks across the scene from several compass
+ headings (azimuths) at lower elevations, so vertical faces (building sides,
+ tree canopies, car bodies) are actually observed. This is what disambiguates
+ 3D structure and lifts quality, so it gets the bulk of the budget.
+
+ Returns config fields to fill (``view_source``, ``tiers``, ``select``,
+ ``margin``) plus ``n_views_planned`` and a human-readable ``rationale``.
+ """
+ fp = stats.footprint
+ ov_elev = (80, 60)
+ ov_budget = max(2 * _GRID_MIN * _GRID_MIN, int(round(_OVERVIEW_SHARE * budget)))
+ g_ov = _grid_for_budget(ov_budget, n_elev=len(ov_elev), num_azimuth=1)
+ overview = Tier(
+ radius=round(fp * _OVERVIEW_RADIUS_FRAC, 2), elevations=ov_elev, fov_deg=52, grid=g_ov, num_azimuth=1
+ )
+ ov_views = g_ov * g_ov * len(ov_elev)
+ obl_az, obl_elev = _oblique_angles(stats.vertical_ratio)
+ obl_budget = max(obl_az * len(obl_elev) * _GRID_MIN * _GRID_MIN, budget - ov_views)
+ g_obl = _grid_for_budget(obl_budget, n_elev=len(obl_elev), num_azimuth=obl_az)
+ oblique = Tier(
+ radius=round(fp * _OBLIQUE_RADIUS_FRAC, 2), elevations=obl_elev, fov_deg=42, grid=g_obl, num_azimuth=obl_az
+ )
+ tiers = (overview, oblique)
+ n_views = sum(_tier_views(t) for t in tiers)
+ rationale = [
+ f"footprint {fp:.1f} (x={stats.dx:.1f}, y={stats.dy:.1f}), height {stats.dz:.1f}, aspect {stats.aspect:.2f}, vertical_ratio {stats.vertical_ratio:.2f}",
+ f"view budget {budget} -> {n_views} views",
+ f"overview (nadir): radius {overview.radius:.1f} (={_OVERVIEW_RADIUS_FRAC}x fp), grid {g_ov}, fov 52, elevations {ov_elev}, 1 heading -> {ov_views} views",
+ f"oblique (structure): radius {oblique.radius:.1f} (={_OBLIQUE_RADIUS_FRAC}x fp), grid {g_obl}, fov 42, elevations {obl_elev}, {obl_az} headings -> {n_views - ov_views} views",
+ ]
+ return {
+ "view_source": "render",
+ "tiers": tiers,
+ "select": 0.33,
+ "margin": 0.1,
+ "capture": "aerial",
+ "n_views_planned": n_views,
+ "rationale": rationale,
+ }
+
+
+def apply_auto_view_config(cfg: GeoLangSplatConfig, means) -> dict:
+ """Fill an auto-derived view config into ``cfg`` in place.
+
+ Only fills fields the caller left at their default (so an explicit
+ ``--select`` / ``--view-source`` still wins), mirroring ``apply_recipe``.
+ Returns a report dict (``stats``, ``tiers``, ``n_views_planned``,
+ ``rationale``, ``applied``) for callers/CLIs to display.
+ """
+ stats = measure_geometry(means)
+ budget = budget_views(cfg.max_views, cfg.vram_budget_gb)
+ rec = recommend_view_config(stats, budget=budget)
+ default = GeoLangSplatConfig()
+ applied: list[str] = []
+ for k in ("view_source", "tiers", "select", "margin"):
+ if getattr(cfg, k) == getattr(default, k):
+ setattr(cfg, k, rec[k])
+ applied.append(k)
+ if str(cfg.up).lower() in ("", "auto", "none"):
+ vec, conf = estimate_up(means)
+ if conf >= _UP_MIN_INLIERS:
+ cfg.up = vec
+ up_note = (
+ f"up: estimated from ground plane " f"{tuple(round(float(x), 2) for x in vec)} (inlier {conf:.0%})"
+ )
+ else:
+ cfg.up = np.array([0.0, 0.0, 1.0])
+ up_note = f"up: no confident ground plane (inlier {conf:.0%}) -> default +z"
+ applied.append("up")
+ rec["rationale"] = list(rec["rationale"]) + [up_note]
+ return {
+ "stats": stats,
+ "tiers": cfg.tiers,
+ "capture": rec["capture"],
+ "up": cfg.up,
+ "n_views_planned": rec["n_views_planned"],
+ "rationale": rec["rationale"],
+ "applied": applied,
+ }
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cameras.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cameras.py
new file mode 100644
index 0000000..951183c
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cameras.py
@@ -0,0 +1,174 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Camera helpers: pinhole intrinsics, orbit-camera generation, and projection.
+
+Conventions
+-----------
+* World frame is ENU-style with +Z up (the satellite/aerial splats are stored
+ this way: x=East, y=North, z=Up).
+* ``elev`` is the elevation angle above the horizontal plane in degrees
+ (90 = straight down / nadir, 0 = on the horizon).
+* ``azimuth`` is measured in the world XY plane in degrees, 0 along +X (East),
+ increasing toward +Y (North).
+* A returned ``w2c`` is an OpenCV world->camera extrinsic (camera +X right,
+ +Y down, +Z forward), which is what ``project`` and
+ ``GaussianSplat3d.render_*`` expect.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+_WORLD_UP = np.array([0.0, 0.0, 1.0])
+_ALT_UP = np.array([0.0, 1.0, 0.0])
+
+# Named up-axis specs -> unit vectors.
+_AXES = {
+ "+x": (1.0, 0.0, 0.0),
+ "-x": (-1.0, 0.0, 0.0),
+ "+y": (0.0, 1.0, 0.0),
+ "-y": (0.0, -1.0, 0.0),
+ "+z": (0.0, 0.0, 1.0),
+ "-z": (0.0, 0.0, -1.0),
+}
+
+
+def up_vector(up=None) -> np.ndarray:
+ """Resolve an up-axis spec (``"+z"`` ... ``"-x"``) to a unit vector.
+
+ Geospatial splats are ``+z`` up; object/COLMAP plys carry no convention, so the
+ caller can pick the axis that renders the scene upright (SAM3 needs upright
+ images, and the viewer needs the right up to be navigable). A 3-vector is
+ accepted and normalized as-is; ``""``/``"auto"``/``"none"`` map to ``+z``.
+ """
+ if isinstance(up, (list, tuple, np.ndarray)):
+ v = np.asarray(up, dtype=np.float64).reshape(3)
+ return v / max(np.linalg.norm(v), 1e-9)
+ s = str(up).lower().replace(" ", "")
+ if s in ("", "auto", "none"):
+ return np.array([0.0, 0.0, 1.0])
+ v = _AXES.get(s)
+ if v is None:
+ raise ValueError(f"unknown up axis {up!r}; use one of {sorted(_AXES)} or 'auto'")
+ return np.array(v, dtype=np.float64)
+
+
+def _perp_basis(up_hat: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+ """Two orthonormal vectors spanning the plane perpendicular to ``up_hat``."""
+ ref = _ALT_UP if abs(float(np.dot(up_hat, _WORLD_UP))) > 0.9 else _WORLD_UP
+ e1 = np.cross(ref, up_hat)
+ e1 /= max(np.linalg.norm(e1), 1e-9)
+ e2 = np.cross(up_hat, e1)
+ e2 /= max(np.linalg.norm(e2), 1e-9)
+ return e1, e2
+
+
+def intrinsics(fov_deg: float, width: int, height: int) -> np.ndarray:
+ """Pinhole intrinsics (3x3) with square pixels and a centered principal point."""
+ f = 0.5 * width / np.tan(0.5 * np.deg2rad(fov_deg))
+ cx = width / 2.0
+ cy = height / 2.0
+ return np.array([[f, 0, cx], [0, f, cy], [0, 0, 1]], dtype=np.float64)
+
+
+def _look_at_w2c(eye: np.ndarray, target: np.ndarray, up=None) -> np.ndarray:
+ """OpenCV world->camera matrix for a camera at ``eye`` looking at ``target``.
+
+ ``up`` is the world-space up direction used to roll the camera (defaults to +z).
+ """
+ up = _WORLD_UP if up is None else np.asarray(up, dtype=np.float64).reshape(3)
+ z_c = target - eye
+ n = np.linalg.norm(z_c)
+ z_c = np.array([0.0, 0.0, -1.0]) if n < 1e-9 else z_c / n
+ if abs(float(np.dot(z_c, up))) > 0.999:
+ # view direction parallel to up: fall back to an axis that isn't.
+ up = _ALT_UP if abs(float(np.dot(z_c, _ALT_UP))) <= 0.999 else _WORLD_UP
+ x_c = np.cross(z_c, up)
+ x_c /= np.linalg.norm(x_c)
+ y_c = np.cross(z_c, x_c)
+ R = np.stack([x_c, y_c, z_c], axis=0)
+ t = -R @ eye
+ w2c = np.eye(4, dtype=np.float64)
+ w2c[:3, :3] = R
+ w2c[:3, 3] = t
+ return w2c
+
+
+def orbit_cameras(
+ center,
+ elev: float,
+ radius: float,
+ num_azimuth: int = 1,
+ azimuth_offset_deg: float = 0.0,
+ up=None,
+) -> list[tuple[np.ndarray, np.ndarray]]:
+ """Generate ``num_azimuth`` cameras orbiting ``center`` at a fixed elevation.
+
+ ``elev`` is measured from the plane perpendicular to ``up`` (90 = along +up,
+ looking back down it; 0 = on that plane). ``up`` defaults to +z. Returns a list
+ (length ``num_azimuth``) of ``(w2c, c2w)`` 4x4 float64 arrays.
+ """
+ center = np.asarray(center, dtype=np.float64).reshape(3)
+ up_hat = up_vector("+z" if up is None else up)
+ e1, e2 = _perp_basis(up_hat)
+ e = np.deg2rad(elev)
+ cos_e, sin_e = np.cos(e), np.sin(e)
+ out = []
+ n = max(1, num_azimuth)
+ for i in range(n):
+ az = azimuth_offset_deg + 360.0 * i / n
+ a = np.deg2rad(az)
+ offset = radius * (cos_e * (np.cos(a) * e1 + np.sin(a) * e2) + sin_e * up_hat)
+ eye = center + offset
+ w2c = _look_at_w2c(eye, center, up=up_hat)
+ c2w = np.linalg.inv(w2c)
+ out.append((w2c, c2w))
+ return out
+
+
+def inside_out_cameras(
+ center,
+ elevations,
+ num_azimuth: int,
+ up=None,
+) -> list[tuple[np.ndarray, np.ndarray]]:
+ """Cameras placed AT ``center`` looking OUTWARD (for interior / unbounded scenes).
+
+ The outside-in orbit fails on a room: from outside you see the backs of walls.
+ Here the eye sits at the scene core and sweeps a full azimuth ring at a few
+ pitches, so the surrounding content (walls, counters, furniture) is actually
+ framed. Returns ``(w2c, c2w)`` pairs. ``up`` defaults to +z.
+ """
+ center = np.asarray(center, dtype=np.float64).reshape(3)
+ up_hat = up_vector("+z" if up is None else up)
+ e1, e2 = _perp_basis(up_hat)
+ n = max(1, int(num_azimuth))
+ out = []
+ for elev in elevations:
+ e = np.deg2rad(elev)
+ cos_e, sin_e = np.cos(e), np.sin(e)
+ for i in range(n):
+ a = np.deg2rad(360.0 * i / n)
+ direction = cos_e * (np.cos(a) * e1 + np.sin(a) * e2) + sin_e * up_hat
+ w2c = _look_at_w2c(center, center + direction, up=up_hat)
+ c2w = np.linalg.inv(w2c)
+ out.append((w2c, c2w))
+ return out
+
+
+def project(means, w2c, f: float, cx: float, cy: float):
+ """Project world-space means into a view (OpenCV: x right, y down, z fwd).
+
+ Returns ``(u, v, z)`` pixel coordinates and camera-space depth. ``means`` is
+ ``[N, 3]`` and ``w2c`` is ``[4, 4]``; both are ``torch.Tensor`` on the same
+ device.
+ """
+ import torch
+
+ n = means.shape[0]
+ homog = torch.cat([means, torch.ones(n, 1, device=means.device)], dim=1)
+ pc = (w2c @ homog.T).T
+ x, y, z = pc[:, 0], pc[:, 1], pc[:, 2]
+ u = f * x / z + cx
+ v = f * y / z + cy
+ return u, v, z
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/catalog.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/catalog.py
new file mode 100644
index 0000000..70bd12b
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/catalog.py
@@ -0,0 +1,583 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Multi-prompt segment catalog: cluster every query's segments into a browsable,
+ID'd table of objects you can pull out as ``.ply`` -- the notebook entry point.
+
+A single ``segment`` answers "where is X". A *catalog* runs a whole vocabulary in
+one pass and turns the result into a small object database. The flow per build:
+
+ score the vocab (one SAM3 pass) --> threshold each prompt + split it into
+ spatial objects (3D connected components) --> merge objects that several
+ prompts land on (e.g. ``car`` and ``vehicle``) by 3D overlap, give each a
+ stable id --> expose a table (pandas ``DataFrame``) + per-object ``.ply``.
+
+Everything here is training-free and runs on CPU once the per-prompt scores
+exist; only ``.ply`` extraction / rendering touches the splat tensors.
+
+ cat = engine.catalog(["building", "car", "tree", "road"])
+ cat.table # browse: id, label, n_gaussians, centroid, size
+ cat.show() # top-down render with ids drawn on each object
+ cat.extract(3, "obj3.ply") # pull one object out as a .ply
+ cat.export_all("scene_catalog/") # a folder of per-object .ply + catalog.csv
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+import re
+from dataclasses import dataclass, field
+
+import numpy as np
+import torch
+
+from . import outputs as _out
+from .instances import connected_components
+from .select import select_query, spatial_cleanup
+
+# A general starting vocabulary. Override it with the prompts that fit your scene
+# (outdoor/aerial defaults shown); object/interior scenes want furniture words, etc.
+DEFAULT_CATALOG_VOCAB: tuple[str, ...] = (
+ "building",
+ "house",
+ "car",
+ "road",
+ "tree",
+ "grass",
+ "water",
+ "sidewalk",
+)
+
+# Table column order (kept stable so a cudf/pandas swap is a one-liner).
+COLUMNS: tuple[str, ...] = (
+ "id",
+ "label",
+ "n_gaussians",
+ "score",
+ "cx",
+ "cy",
+ "cz",
+ "dx",
+ "dy",
+ "dz",
+ "prompts",
+)
+
+
+@dataclass
+class CatalogObject:
+ """One physical object in the catalog (a merged group of segment instances)."""
+
+ id: int
+ label: str # best (highest-scoring) prompt for this object
+ prompts: list[str] # every prompt whose segment overlapped this object
+ n_gaussians: int
+ score: float
+ centroid: np.ndarray # (3,)
+ bbox_min: np.ndarray # (3,)
+ bbox_max: np.ndarray # (3,)
+
+ @property
+ def size(self) -> np.ndarray:
+ """Axis-aligned bounding-box extent ``(dx, dy, dz)``."""
+ return self.bbox_max - self.bbox_min
+
+ def row(self) -> dict:
+ """Flat dict for one table row (the unit a DataFrame is built from)."""
+ c, s = self.centroid, self.size
+ return {
+ "id": self.id,
+ "label": self.label,
+ "n_gaussians": self.n_gaussians,
+ "score": round(float(self.score), 4),
+ "cx": round(float(c[0]), 4),
+ "cy": round(float(c[1]), 4),
+ "cz": round(float(c[2]), 4),
+ "dx": round(float(s[0]), 4),
+ "dy": round(float(s[1]), 4),
+ "dz": round(float(s[2]), 4),
+ "prompts": ", ".join(self.prompts),
+ }
+
+
+@dataclass
+class SegmentCatalog:
+ """A browsable, ID'd table of objects, ready to inspect / extract in a notebook.
+
+ ``labels`` is a ``[N]`` long tensor in input ``.ply`` order: ``-1`` for an
+ unassigned Gaussian, else the owning object's :attr:`CatalogObject.id`.
+ """
+
+ objects: list[CatalogObject]
+ labels: torch.Tensor # [N] long, -1 = unassigned, else object id (CPU)
+ config: object
+ _model: object = None
+ metadata: dict = field(default_factory=dict)
+ _engine: object = None # warm engine, if any -- lets browse() re-query in place
+
+ # -- container sugar ----------------------------------------------------
+
+ def __len__(self) -> int:
+ return len(self.objects)
+
+ def __getitem__(self, obj_id: int) -> CatalogObject:
+ return self._by_id(obj_id)
+
+ def __iter__(self):
+ return iter(self.objects)
+
+ def __repr__(self) -> str:
+ head = f"SegmentCatalog: {len(self.objects)} objects from {len(self._vocab())} prompts"
+ rows = [f" #{o.id}: {o.label!r} {o.n_gaussians:,} gaussians score={o.score:.3f}" for o in self.objects[:20]]
+ if len(self.objects) > 20:
+ rows.append(f" ... (+{len(self.objects) - 20} more)")
+ return "\n".join([head, *rows])
+
+ def _repr_html_(self):
+ """Rich table when a catalog is the last expression in a notebook cell."""
+ try:
+ import pandas as pd # noqa: PLC0415 - optional, notebook-only
+
+ head = f"SegmentCatalog : {len(self.objects)} objects from {len(self._vocab())} prompts"
+ return head + pd.DataFrame(self.rows(), columns=list(COLUMNS)).to_html(index=False)
+ except ModuleNotFoundError:
+ return None # Jupyter falls back to __repr__
+
+ def _vocab(self) -> list[str]:
+ seen: list[str] = []
+ for o in self.objects:
+ for p in o.prompts:
+ if p not in seen:
+ seen.append(p)
+ return seen
+
+ # -- table --------------------------------------------------------------
+
+ def rows(self) -> list[dict]:
+ """The catalog as a list of flat row dicts (backend-agnostic)."""
+ return [o.row() for o in self.objects]
+
+ @property
+ def table(self):
+ """The catalog as a DataFrame for notebook browsing.
+
+ Uses pandas (swap to cudf for a GPU frame -- same columns). Falls back to
+ the plain :meth:`rows` list if no DataFrame library is installed.
+ """
+ rows = self.rows()
+ try:
+ import pandas as pd # noqa: PLC0415 - optional, notebook-only
+
+ return pd.DataFrame(rows, columns=list(COLUMNS))
+ except ModuleNotFoundError:
+ return rows
+
+ # -- masks / extraction -------------------------------------------------
+
+ def mask(self, obj_id: int) -> torch.Tensor:
+ """Boolean ``[N]`` mask selecting only object ``obj_id`` (model device)."""
+ self._by_id(obj_id)
+ m = self.labels == obj_id
+ dev = getattr(getattr(self._model, "means", None), "device", None)
+ return m.to(dev) if dev is not None else m
+
+ @property
+ def all_mask(self) -> torch.Tensor:
+ """Boolean ``[N]`` mask of every catalogued Gaussian."""
+ m = self.labels >= 0
+ dev = getattr(getattr(self._model, "means", None), "device", None)
+ return m.to(dev) if dev is not None else m
+
+ def extract(self, obj_id: int, path) -> int:
+ """Write only object ``obj_id`` to ``path`` as a ``.ply``. Returns its count."""
+ return _out.write_ply_segmented(self._model, self.mask(obj_id), path, metadata=self.metadata)
+
+ def export_all(self, out_dir, *, labeled_ply: bool = True) -> pathlib.Path:
+ """Write the whole catalog to a folder -- the "segment database" to browse
+ or hand to Omniverse:
+
+ * ``catalog.csv`` -- the table (one row per object),
+ * ``objects/_.ply`` -- each object as its own splat,
+ * ``catalog_labeled.ply`` -- the full splat recoloured by object (optional).
+ """
+ out = pathlib.Path(out_dir)
+ (out / "objects").mkdir(parents=True, exist_ok=True)
+ (out / "catalog.csv").write_text(_to_csv(self.rows()))
+ for o in self.objects:
+ self.extract(o.id, out / "objects" / f"{o.id:03d}_{_slug(o.label)}.ply")
+ if labeled_ply:
+ _out.write_ply_labels(
+ self._model,
+ self.labels.to(self._model.means.device),
+ out / "catalog_labeled.ply",
+ metadata=self.metadata,
+ )
+ print(f"[catalog] wrote {len(self.objects)} objects -> {out}", flush=True)
+ return out
+
+ # -- persistence --------------------------------------------------------
+
+ def save(self, out_dir) -> pathlib.Path:
+ """Persist the catalog (table + per-Gaussian object labels) so it can be
+ reloaded later with :meth:`load` and the same source ``.ply``."""
+ out = pathlib.Path(out_dir)
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "catalog.csv").write_text(_to_csv(self.rows()))
+ np.savez_compressed(out / "labels.npz", labels=self.labels.cpu().numpy())
+ meta = {"objects": self.rows()}
+ (out / "catalog.json").write_text(json.dumps(meta, indent=2))
+ print(f"[catalog] saved -> {out}", flush=True)
+ return out
+
+ @classmethod
+ def load(cls, in_dir, model, config=None, *, metadata: dict | None = None) -> "SegmentCatalog":
+ """Reload a catalog saved by :meth:`save`, bound to ``model`` for extraction."""
+ in_dir = pathlib.Path(in_dir)
+ labels = torch.from_numpy(np.load(in_dir / "labels.npz")["labels"]).long()
+ meta = json.loads((in_dir / "catalog.json").read_text())
+ objects = [_object_from_row(r) for r in meta["objects"]]
+ return cls(objects=objects, labels=labels, config=config, _model=model, metadata=metadata or {})
+
+ # -- visualization ------------------------------------------------------
+
+ def show(self, *, size: int = 900, device=None, path=None):
+ """Inline top-down render with each object's id drawn on it (returns a
+ ``PIL.Image``; pass ``path`` to also save the PNG)."""
+ from .instances import _annotate, _render_topdown
+
+ pil, uv = _render_topdown(self._model, self.config, self.objects, size=size, device=device)
+ labels = [(int(o.id), uv[i]) for i, o in enumerate(self.objects) if uv[i] is not None]
+ out = _annotate(pil, labels)
+ if path is not None:
+ out.save(path)
+ print(f"[catalog] wrote {path}", flush=True)
+ return out
+
+ def show_one(self, obj_id: int, *, size: int = 900, device=None, path=None):
+ """Top-down render with only object ``obj_id`` highlighted (rest dimmed)."""
+ from .instances import _annotate, _render_topdown
+
+ self._by_id(obj_id)
+ i = next(k for k, o in enumerate(self.objects) if o.id == obj_id)
+ pil, uv = _render_topdown(
+ self._model, self.config, self.objects, size=size, device=device, highlight=self.mask(obj_id)
+ )
+ out = _annotate(pil, [(obj_id, uv[i])] if uv[i] is not None else [])
+ if path is not None:
+ out.save(path)
+ print(f"[catalog] wrote {path}", flush=True)
+ return out
+
+ def preview(self, ids=None, *, size: int = 700, device=None):
+ """Top-down render with object ``ids`` highlighted (all dimmed if empty).
+
+ ``ids`` may be a single id or an iterable; returns a ``PIL.Image``.
+ """
+ from .instances import _annotate, _render_topdown
+
+ ids = [] if ids is None else ([ids] if isinstance(ids, int) else [int(i) for i in ids])
+ highlight = self._ids_mask(ids).to(self._model.means.device) if ids else None
+ pil, uv = _render_topdown(self._model, self.config, self.objects, size=size, device=device, highlight=highlight)
+ keep = set(ids) if ids else {o.id for o in self.objects}
+ labels = [(o.id, uv[i]) for i, o in enumerate(self.objects) if uv[i] is not None and o.id in keep]
+ return _annotate(pil, labels)
+
+ def browse(self, *, size: int = 600, out_dir: str = "picked", device=None):
+ """Interactive notebook picker -- the no-id-juggling way to grab segments.
+
+ Renders a live preview beside a multi-select list of every object: click
+ objects to highlight them in the render, then hit **Export selected** to
+ write their ``.ply`` files. If the catalog has a warm engine attached
+ (``engine.catalog`` / ``build_catalog``), a query box re-runs the vocabulary
+ in place. Requires ``ipywidgets`` in a Jupyter kernel.
+ """
+ try:
+ import ipywidgets as widgets # noqa: PLC0415 - optional, notebook-only
+ except ModuleNotFoundError as exc: # pragma: no cover - needs a notebook
+ raise RuntimeError(
+ "browse() needs ipywidgets (pip install ipywidgets); use .table / .extract otherwise"
+ ) from exc
+ import io # noqa: PLC0415
+
+ from IPython.display import display # noqa: PLC0415 - notebook-only
+
+ def _png(ids) -> bytes:
+ buf = io.BytesIO()
+ self.preview(ids, size=size, device=device).save(buf, format="PNG")
+ return buf.getvalue()
+
+ def _options():
+ return [(f"#{o.id} {o.label} ({o.n_gaussians:,} gaussians)", o.id) for o in self.objects]
+
+ picker = widgets.SelectMultiple(
+ options=_options(), rows=min(16, max(4, len(self.objects))), layout=widgets.Layout(width="340px")
+ )
+ img = widgets.Image(value=_png([]), format="png")
+ outdir = widgets.Text(value=out_dir, description="out dir:", layout=widgets.Layout(width="280px"))
+ export = widgets.Button(description="Export selected", button_style="primary", icon="download")
+ status = widgets.Output()
+
+ def _on_select(_change):
+ img.value = _png(list(picker.value))
+
+ picker.observe(_on_select, names="value")
+
+ def _on_export(_btn):
+ with status:
+ status.clear_output()
+ ids = list(picker.value) or [o.id for o in self.objects]
+ dest = pathlib.Path(outdir.value)
+ (dest / "objects").mkdir(parents=True, exist_ok=True)
+ for i in ids:
+ obj = self._by_id(i)
+ path = dest / "objects" / f"{i:03d}_{_slug(obj.label)}.ply"
+ self.extract(i, path)
+ print(f" #{i} {obj.label} -> {path}")
+ print(f"exported {len(ids)} object(s) to {dest}/")
+
+ export.on_click(_on_export)
+ controls = [picker, widgets.HBox([outdir, export]), status]
+
+ if getattr(self, "_engine", None) is not None:
+ query = widgets.Text(
+ placeholder="type a vocab and Search (comma/space separated)", layout=widgets.Layout(width="280px")
+ )
+ search = widgets.Button(description="Search", icon="search")
+
+ def _on_search(_btn):
+ with status:
+ status.clear_output()
+ words = [w for w in re.split(r"[,\s]+", query.value) if w]
+ if not words:
+ print("type a vocab first")
+ return
+ print(f"re-querying {words} ...")
+ fresh = self._engine.catalog(words)
+ self.objects, self.labels, self.metadata = fresh.objects, fresh.labels, fresh.metadata
+ picker.options = _options()
+ picker.value = ()
+ img.value = _png([])
+ print(f"{len(self.objects)} objects")
+
+ search.on_click(_on_search)
+ controls = [widgets.HBox([query, search]), *controls]
+
+ ui = widgets.HBox([widgets.VBox(controls), img])
+ display(ui)
+ return ui
+
+ # -- internals ----------------------------------------------------------
+
+ def _ids_mask(self, ids) -> torch.Tensor:
+ """Boolean ``[N]`` mask (CPU) selecting the union of object ``ids``."""
+ m = torch.zeros_like(self.labels, dtype=torch.bool)
+ for i in ids:
+ m |= self.labels == int(i)
+ return m
+
+ def _by_id(self, obj_id: int) -> CatalogObject:
+ for o in self.objects:
+ if o.id == obj_id:
+ return o
+ valid = f"0..{len(self.objects) - 1}" if self.objects else "(empty catalog)"
+ raise KeyError(f"no object with id {obj_id}; valid ids: {valid}")
+
+
+# --- clustering core (pure, CPU-testable) ----------------------------------
+
+
+def _pair_intersections(covers: list[torch.Tensor], n_obj: int) -> dict[tuple[int, int], int]:
+ """Gaussian-count of the intersection of every pair of objects from *different*
+ prompts. ``covers[c]`` is ``[N]`` long: the global object id covering each
+ Gaussian for prompt ``c`` (``-1`` = none). Objects from the same prompt are
+ disjoint (connected components), so only cross-prompt pairs can intersect.
+ """
+ inter: dict[tuple[int, int], int] = {}
+ c = len(covers)
+ for a in range(c):
+ ca = covers[a]
+ for b in range(a + 1, c):
+ cb = covers[b]
+ both = (ca >= 0) & (cb >= 0)
+ if not bool(both.any()):
+ continue
+ key = ca[both].long() * n_obj + cb[both].long()
+ uk, cnt = torch.unique(key, return_counts=True)
+ for k_, cnt_ in zip(uk.tolist(), cnt.tolist()):
+ i, j = divmod(int(k_), n_obj)
+ pair = (i, j) if i < j else (j, i)
+ inter[pair] = inter.get(pair, 0) + int(cnt_)
+ return inter
+
+
+def _union_find(n: int, edges) -> list[int]:
+ parent = list(range(n))
+
+ def find(a: int) -> int:
+ while parent[a] != a:
+ parent[a] = parent[parent[a]]
+ a = parent[a]
+ return a
+
+ for a, b in edges:
+ ra, rb = find(a), find(b)
+ if ra != rb:
+ parent[ra] = rb
+ return [find(i) for i in range(n)]
+
+
+# --- assembly --------------------------------------------------------------
+
+
+def catalog_from_scores(
+ model,
+ vocab,
+ cfg,
+ scores: torch.Tensor,
+ *,
+ seen: torch.Tensor,
+ denom: torch.Tensor | None = None,
+ select: float | None = None,
+ iou: float | None = None,
+ link_frac: float | None = None,
+ min_size: int | None = None,
+ metadata: dict | None = None,
+) -> SegmentCatalog:
+ """Assemble a :class:`SegmentCatalog` from per-prompt scores ``[N, C]``.
+
+ For each prompt: threshold (reusing :func:`select_query`) -> spatial cleanup ->
+ connected components. Then merge objects across prompts whose 3D IoU clears
+ ``iou`` (``cfg.cat_iou``) and assign ids largest-first.
+ """
+ vocab = list(vocab)
+ iou_thr = cfg.cat_iou if iou is None else iou
+ means_cpu = model.means.detach().float().cpu()
+ span = _robust_span(means_cpu) # ignore far floaters so the voxel size tracks the real scene
+ scores_cpu = scores.detach().float().cpu()
+ if scores_cpu.ndim == 1:
+ scores_cpu = scores_cpu.unsqueeze(1)
+ seen_cpu = seen.detach().cpu() if seen is not None else torch.ones(means_cpu.shape[0])
+ denom_cpu = denom.detach().cpu() if denom is not None else None
+ n = means_cpu.shape[0]
+
+ # 1. per-prompt selections -> spatial instances ("raw" objects, one per (prompt, component)).
+ raw_masks: list[torch.Tensor] = [] # bool [N] (CPU)
+ raw_label: list[str] = []
+ raw_score: list[float] = []
+ covers: list[torch.Tensor] = [] # per prompt: [N] long global-raw-id or -1
+ for c, prompt in enumerate(vocab):
+ sel = select_query(scores_cpu[:, c], seen_cpu, cfg, query=prompt, select=select, denom=denom_cpu, compete=False)
+ sel = spatial_cleanup(means_cpu, sel, cfg)
+ comp_ids, infos = connected_components(means_cpu, sel, cfg, link_frac=link_frac, min_size=min_size, span=span)
+ cover = torch.full((n,), -1, dtype=torch.long)
+ for info in infos:
+ local = info["idx"]
+ m = comp_ids == local
+ gid = len(raw_masks)
+ cover[m] = gid
+ raw_masks.append(m)
+ raw_label.append(prompt)
+ raw_score.append(float(scores_cpu[:, c][m].mean()) if bool(m.any()) else 0.0)
+ covers.append(cover)
+
+ if not raw_masks:
+ return SegmentCatalog([], torch.full((n,), -1, dtype=torch.long), cfg, _model=model, metadata=metadata or {})
+
+ # 2. merge raw objects that overlap across prompts (same physical thing).
+ sizes = [int(m.sum()) for m in raw_masks]
+ inter = _pair_intersections(covers, len(raw_masks))
+ edges = []
+ for (i, j), nij in inter.items():
+ union = sizes[i] + sizes[j] - nij
+ if union > 0 and (nij / union) >= iou_thr:
+ edges.append((i, j))
+ roots = _union_find(len(raw_masks), edges)
+
+ groups: dict[int, list[int]] = {}
+ for k, r in enumerate(roots):
+ groups.setdefault(r, []).append(k)
+
+ # 3. build one CatalogObject per group; assign ids largest-first; paint labels.
+ built: list[tuple[torch.Tensor, str, list[str], float]] = []
+ for members in groups.values():
+ gmask = raw_masks[members[0]].clone()
+ for k in members[1:]:
+ gmask |= raw_masks[k]
+ best = max(members, key=lambda k: raw_score[k])
+ prompts = sorted({raw_label[k] for k in members})
+ built.append((gmask, raw_label[best], prompts, raw_score[best]))
+
+ built.sort(key=lambda t: int(t[0].sum()), reverse=True)
+ labels = torch.full((n,), -1, dtype=torch.long)
+ objects: list[CatalogObject] = []
+ for obj_id, (gmask, label, prompts, score) in enumerate(built):
+ labels[gmask] = obj_id # later (smaller) objects can only overwrite empty slots below
+ pts = means_cpu[gmask]
+ objects.append(
+ CatalogObject(
+ id=obj_id,
+ label=label,
+ prompts=prompts,
+ n_gaussians=int(gmask.sum()),
+ score=score,
+ centroid=pts.mean(dim=0).numpy(),
+ bbox_min=pts.min(dim=0).values.numpy(),
+ bbox_max=pts.max(dim=0).values.numpy(),
+ )
+ )
+ # paint largest-first so overlapping smaller objects don't steal a bigger one's Gaussians
+ labels.fill_(-1)
+ for obj_id, (gmask, *_rest) in enumerate(built):
+ labels[gmask & (labels < 0)] = obj_id
+
+ return SegmentCatalog(objects=objects, labels=labels, config=cfg, _model=model, metadata=metadata or {})
+
+
+# --- small helpers ---------------------------------------------------------
+
+
+def _robust_span(means_cpu: torch.Tensor) -> float:
+ """Scene extent from a 1-99% quantile box (per axis), ignoring far floaters.
+
+ The full min/max range is dominated by stray Gaussians on large captures, which
+ inflates the connected-components voxel and merges separate objects; the
+ quantile box tracks the real scene. Subsampled for speed on huge splats.
+ """
+ mc = means_cpu
+ if mc.shape[0] > 500_000:
+ mc = mc[torch.randperm(mc.shape[0])[:500_000]]
+ lo = torch.quantile(mc, 0.01, dim=0)
+ hi = torch.quantile(mc, 0.99, dim=0)
+ return float((hi - lo).max())
+
+
+def _slug(text: str) -> str:
+ s = re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_")
+ return s or "object"
+
+
+def _to_csv(rows: list[dict]) -> str:
+ header = ",".join(COLUMNS)
+ lines = [header]
+ for r in rows:
+ vals = []
+ for col in COLUMNS:
+ v = r.get(col, "")
+ v = f'"{v}"' if isinstance(v, str) and ("," in v) else v
+ vals.append(str(v))
+ lines.append(",".join(vals))
+ return "\n".join(lines) + "\n"
+
+
+def _object_from_row(r: dict) -> CatalogObject:
+ prompts = [p.strip() for p in str(r.get("prompts", "")).split(",") if p.strip()]
+ return CatalogObject(
+ id=int(r["id"]),
+ label=str(r["label"]),
+ prompts=prompts,
+ n_gaussians=int(r["n_gaussians"]),
+ score=float(r["score"]),
+ centroid=np.array([r["cx"], r["cy"], r["cz"]], dtype=np.float32),
+ bbox_min=np.array([r["cx"] - r["dx"] / 2, r["cy"] - r["dy"] / 2, r["cz"] - r["dz"] / 2], dtype=np.float32),
+ bbox_max=np.array([r["cx"] + r["dx"] / 2, r["cy"] + r["dy"] / 2, r["cz"] + r["dz"] / 2], dtype=np.float32),
+ )
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/__init__.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/__init__.py
new file mode 100644
index 0000000..421523c
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/__init__.py
@@ -0,0 +1,64 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""GeoLangSplat command-line interface.
+
+``gls`` dispatches subcommands, mirroring ``frgs``. The commands subclass the
+shared ``BaseCommand`` so they can be registered directly inside ``frgs`` later
+(e.g. as ``frgs segment "house"``).
+"""
+from __future__ import annotations
+
+import sys
+import time
+
+import tyro
+
+from .. import _PROCESS_T0
+from ..errors import GeoLangSplatError
+from ._bake import Bake
+from ._catalog import Catalog
+from ._check import Check
+from ._common import BaseCommand
+from ._doctor import Doctor
+from ._explore import Explore
+from ._render import Render
+from ._segment import Segment
+from ._serve import Serve
+from ._show import Show
+from ._status import Status
+from ._stop import Stop
+
+__all__ = [
+ "gls",
+ "Segment",
+ "Bake",
+ "Catalog",
+ "Check",
+ "Doctor",
+ "Serve",
+ "Show",
+ "Explore",
+ "Render",
+ "Status",
+ "Stop",
+ "BaseCommand",
+]
+
+
+def gls() -> None:
+ cmd: BaseCommand = tyro.cli(
+ Segment | Bake | Catalog | Check | Doctor | Serve | Show | Explore | Render | Status | Stop
+ )
+ try:
+ cmd.execute()
+ except GeoLangSplatError as e: # user-actionable: print the message, no traceback
+ print(f"[gls] {e}", file=sys.stderr)
+ raise SystemExit(1)
+ finally:
+ if getattr(cmd, "profile", False):
+ print(
+ f"[gls] total wall {time.perf_counter() - _PROCESS_T0:.1f}s "
+ "(includes python/torch import + CUDA init)",
+ flush=True,
+ )
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_bake.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_bake.py
new file mode 100644
index 0000000..52423cc
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_bake.py
@@ -0,0 +1,113 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls bake` - fixed-vocabulary multi-class labelling of every Gaussian."""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass, field
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName
+from ._common import BaseCommand, build_config, read_vocab_file, resolve_recipe
+
+
+@dataclass
+class Bake(BaseCommand):
+ """
+ Assign every Gaussian one label from a fixed vocabulary (argmax with
+ confidence/ambiguity gates), and write a per-Gaussian report (+ recoloured ply).
+
+ The vocabulary can be given inline (--vocab) or from a text file
+ (--vocab-file, one word per line, '#' for comments).
+
+ Example usage:
+
+ gls bake model.ply --sfm /path/scene --recipe satellite \\
+ --vocab house tree grass sand water road -o labels/
+
+ gls bake model.ply --sfm /path/scene --vocab-file classes.txt -o labels/
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # The fixed label vocabulary (ignored if --vocab-file is given).
+ vocab: list[str] = field(default_factory=lambda: ["house", "tree", "grass", "sand", "water", "road"])
+
+ # Read the vocabulary from a .txt file instead (one word per line, '#' comments).
+ vocab_file: Optional[pathlib.Path] = None
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # Capture-type preset: auto (default) | satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # View source override: cameras | render | images.
+ view_source: Optional[str] = None
+
+ # Cap the synthesized/subsampled view count.
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Output directory (labels.csv, labels.npz, legend.json, labels.ply).
+ out: Annotated[pathlib.Path, arg(aliases=["-o"])] = pathlib.Path("labels")
+
+ # Confidence floor for a label.
+ tau: Optional[float] = None
+
+ # Ambiguity margin (top1 - top2) below which a Gaussian is left unlabeled.
+ delta: Optional[float] = None
+
+ # Also write a label-recoloured .ply.
+ viz_ply: bool = True
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ # Low-VRAM streaming: one bounded pass scores the whole vocab per view then evicts,
+ # so peak VRAM is bounded to ~one view. ON by default; --no-low-vram for the full
+ # all-views cache (higher VRAM, only worth it when re-baking many vocabularies).
+ low_vram: bool = True
+
+ def execute(self) -> None:
+ from ..api import segment # lazy: keeps `gls` startup torch-free
+
+ cfg = build_config(
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ tau=self.tau,
+ delta=self.delta,
+ device=self.device,
+ low_vram=self.low_vram,
+ )
+ vocab = read_vocab_file(self.vocab_file) if self.vocab_file else list(self.vocab)
+ if not vocab:
+ raise ValueError("empty vocabulary")
+ print(f"[gls bake] vocab ({len(vocab)}): {vocab}", flush=True)
+ result = segment(
+ self.model_path,
+ vocab,
+ config=cfg,
+ recipe=resolve_recipe(self.recipe, self.view_source),
+ )
+ out = pathlib.Path(self.out)
+ out.mkdir(parents=True, exist_ok=True)
+ result.to_report(out / "labels.csv")
+ result.to_report(out / "labels.npz")
+ if self.viz_ply:
+ result.to_ply_overlay(out / "labels.ply")
+ print(f"[gls bake] {result.stats}", flush=True)
+ print(f"[gls bake] wrote report + ply -> {out}/", flush=True)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_catalog.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_catalog.py
new file mode 100644
index 0000000..f793fa8
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_catalog.py
@@ -0,0 +1,111 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls catalog` - run a vocabulary and build a browsable object catalog.
+
+One pass over a prompt list, clustered into distinct physical objects and written
+to a folder you can browse (``catalog.csv``) and pull single objects from
+(``objects/_.ply``) -- e.g. for import into a downstream tool. The
+notebook surface is :func:`geolangsplat.build_catalog` / ``engine.catalog``.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass, field
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName
+from ._common import BaseCommand, build_config, read_vocab_file, resolve_recipe
+
+
+@dataclass
+class Catalog(BaseCommand):
+ """
+ Build an ID'd object catalog for a splat over a prompt vocabulary.
+
+ Each prompt is segmented and split into spatial objects; objects hit by several
+ prompts are merged. The result is a table plus one ``.ply`` per object.
+
+ Examples:
+
+ gls catalog model.ply --vocab building car tree road -o scene_catalog/
+ gls catalog model.ply --vocab-file classes.txt -o scene_catalog/
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # The prompt vocabulary to catalog (ignored if --vocab-file is given).
+ vocab: list[str] = field(
+ default_factory=lambda: ["building", "house", "car", "road", "tree", "grass", "water", "sidewalk"]
+ )
+
+ # Read the vocabulary from a .txt file instead (one word per line, '#' comments).
+ vocab_file: Optional[pathlib.Path] = None
+
+ # Output directory (catalog.csv, objects/_.ply, catalog_labeled.ply).
+ out: Annotated[pathlib.Path, arg(aliases=["-o"])] = pathlib.Path("catalog")
+
+ # Capture-type preset: auto (default) | satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # View source: render | globe | images (real photos via --sfm).
+ view_source: Optional[str] = None
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # Cap the synthesized/subsampled view count.
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Per-prompt selection score threshold (defaults to the config's `select`).
+ select: Annotated[Optional[float], arg(aliases=["-t"])] = None
+
+ # Merge objects from different prompts when their 3D IoU is >= this.
+ iou: Optional[float] = None
+
+ # Also write the full splat recoloured by object id.
+ labeled_ply: bool = True
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ # Low-VRAM streaming build (see `gls segment`). --no-low-vram for the full cache.
+ low_vram: bool = True
+
+ def execute(self) -> None:
+ from ..api import build_catalog # lazy: keeps `gls` startup torch-free
+
+ cfg = build_config(
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ device=self.device,
+ low_vram=self.low_vram,
+ )
+ vocab = read_vocab_file(self.vocab_file) if self.vocab_file else list(self.vocab)
+ if not vocab:
+ raise ValueError("empty vocabulary")
+ print(f"[gls catalog] vocab ({len(vocab)}): {vocab}", flush=True)
+ cat = build_catalog(
+ self.model_path,
+ vocab,
+ config=cfg,
+ recipe=resolve_recipe(self.recipe, self.view_source),
+ select=self.select,
+ iou=self.iou,
+ )
+ cat.export_all(self.out, labeled_ply=self.labeled_ply)
+ print(f"[gls catalog] {len(cat)} objects:", flush=True)
+ print(cat, flush=True)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_check.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_check.py
new file mode 100644
index 0000000..8920adf
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_check.py
@@ -0,0 +1,93 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls check` - scene-readiness smoke test (no SAM3 weights required)."""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName
+from ._common import BaseCommand, build_config, resolve_recipe
+
+
+@dataclass
+class Check(BaseCommand):
+ """
+ Report whether a reconstruction is dense enough for segmentation to work.
+
+ Builds the views + per-Gaussian lift cache and prints coverage statistics and
+ a verdict (good / fair / poor). This needs a GPU to render, but does NOT need
+ SAM3 weights -- run it before a full segment to catch weak scenes early.
+
+ Example usage:
+
+ gls check model.ply --sfm /path/scene --recipe satellite
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # Capture-type preset: auto (default) | satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # View source override: cameras | render | images.
+ view_source: Optional[str] = None
+
+ # Number of views to sample / render (view_source=images).
+ n_views: Optional[int] = None
+
+ # Cap the synthesized/subsampled view count.
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Inside-out: place the eye at the scene core looking outward (interior scenes).
+ inside_out: bool = False
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ def execute(self) -> None:
+ from ..api import assess_scene # lazy: keeps `gls` startup torch-free
+
+ cfg = build_config(
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ n_views=self.n_views,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ inside_out=(True if self.inside_out else None),
+ device=self.device,
+ )
+ report = assess_scene(self.model_path, config=cfg, recipe=resolve_recipe(self.recipe, self.view_source))
+ print("\n=== scene readiness ===")
+ for k in (
+ "gaussians",
+ "views",
+ "capture",
+ "coverage",
+ "mean_views_per_observed_gaussian",
+ "well_observed_frac",
+ ):
+ if k in report and report[k] is not None:
+ v = report[k]
+ print(f" {k:<34} {v:.3f}" if isinstance(v, float) else f" {k:<34} {v}")
+ rationale = report.get("auto_rationale")
+ if rationale:
+ print("\n recommended views (auto):")
+ for line in rationale:
+ print(f" - {line}")
+ print(f"\n verdict: {report['verdict'].upper()} - {report['note']}\n")
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_common.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_common.py
new file mode 100644
index 0000000..3061698
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_common.py
@@ -0,0 +1,70 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Shared CLI helpers."""
+from __future__ import annotations
+
+import pathlib
+import sys
+
+from ..config import GeoLangSplatConfig
+
+# BaseCommand: subclass fvdb-reality-capture's so these commands can drop straight
+# into `frgs` -- but ONLY when frc is already imported (i.e. we're running inside
+# frgs, where torch/fvdb are loaded anyway). For the standalone `gls` CLI we use a
+# local stand-in with the identical contract, so importing the CLI stays torch-free
+# and the lightweight paths (daemon-attach segment, status, stop) start instantly.
+BaseCommand = None
+if "fvdb_reality_capture" in sys.modules: # pragma: no cover - frc-present path
+ try:
+ from fvdb_reality_capture.cli import BaseCommand
+ except Exception:
+ BaseCommand = None
+
+if BaseCommand is None:
+ from abc import ABC, abstractmethod
+
+ class BaseCommand(ABC): # type: ignore[no-redef]
+ """Local stand-in matching fvdb_reality_capture.cli.BaseCommand."""
+
+ @abstractmethod
+ def execute(self) -> None: ...
+
+
+def build_config(**overrides) -> GeoLangSplatConfig:
+ """Build a config, applying only the overrides the user actually provided
+ (``None`` values are ignored so recipe presets can still fill them)."""
+ cfg = GeoLangSplatConfig()
+ for k, v in overrides.items():
+ if v is None:
+ continue
+ if not hasattr(cfg, k):
+ raise KeyError(f"unknown config field {k!r}")
+ setattr(cfg, k, v)
+ return cfg
+
+
+def resolve_recipe(recipe, view_source=None):
+ """Default the recipe when the user did not pick one.
+
+ Falls back to ``"aerial"`` when the user asked for ground-truth photos
+ (``--view-source images``), since that path wants the curated photo recipe;
+ otherwise ``"auto"`` (geometry-driven views), so every command frames a scene
+ sensibly out of the box.
+ """
+ if recipe:
+ return recipe
+ if view_source == "images":
+ return "aerial"
+ return "auto"
+
+
+def read_vocab_file(path) -> list[str]:
+ """Read a vocabulary from a text file: one word/phrase per line, '#' comments."""
+ text = pathlib.Path(path).read_text()
+ words: list[str] = []
+ for line in text.splitlines():
+ line = line.split("#", 1)[0].strip()
+ if line:
+ words.append(line)
+ return words
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_doctor.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_doctor.py
new file mode 100644
index 0000000..8162e76
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_doctor.py
@@ -0,0 +1,105 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls doctor` - one-command environment preflight.
+
+Validates everything `gls segment`/`bake` needs on a fresh machine -- GPU, fvdb,
+SAM3, the SAM3 checkpoint, bf16 support -- and prints actionable hints for whatever
+is missing, instead of failing deep inside a build. (SAM3 weights + a working CUDA
+stack are the usual pain when moving to a new box.)
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import pathlib
+from dataclasses import dataclass
+
+from ._common import BaseCommand
+
+_OK = "[ ok ]"
+_FAIL = "[FAIL]"
+_WARN = "[warn]"
+
+
+@dataclass
+class Doctor(BaseCommand):
+ """Check that this machine can actually run GeoLangSplat segmentation."""
+
+ def execute(self) -> None:
+ from .. import __version__
+
+ hard_ok = True
+ print(f"[doctor] GeoLangSplat v{__version__} environment check\n")
+
+ # --- torch + CUDA ---------------------------------------------------
+ try:
+ import torch
+
+ print(f"{_OK} torch {torch.__version__}")
+ if torch.cuda.is_available():
+ n = torch.cuda.device_count()
+ name = torch.cuda.get_device_name(0)
+ cap = ".".join(str(x) for x in torch.cuda.get_device_capability(0))
+ print(f"{_OK} CUDA available: {n} device(s); cuda:0 = {name} (sm_{cap})")
+ bf16 = torch.cuda.is_bf16_supported()
+ mark = _OK if bf16 else _WARN
+ print(
+ f"{mark} bf16 {'supported' if bf16 else 'unsupported -> SAM3 falls back to fp16 (slower/less stable)'}"
+ )
+ else:
+ hard_ok = False
+ print(f"{_FAIL} CUDA not available -- segmentation needs a GPU")
+ print(
+ " check drivers / `nvidia-smi` / CUDA_VISIBLE_DEVICES "
+ "(geometry-only `gls check` still works on CPU)"
+ )
+ except Exception as e:
+ hard_ok = False
+ print(f"{_FAIL} torch import failed: {e}")
+
+ # --- fvdb -----------------------------------------------------------
+ if importlib.util.find_spec("fvdb") is not None:
+ print(f"{_OK} fvdb importable")
+ else:
+ hard_ok = False
+ print(f"{_FAIL} fvdb not importable -- activate the env that has the fvdb build")
+
+ # --- fvdb-reality-capture (frgs integration / checkpoint IO) --------
+ if importlib.util.find_spec("fvdb_reality_capture") is not None:
+ print(f"{_OK} fvdb_reality_capture importable (frgs IO + .pt/.pth checkpoints)")
+ else:
+ print(
+ f"{_WARN} fvdb_reality_capture not importable -- .ply still works; "
+ ".pt/.pth checkpoint loading + frgs drop-in unavailable"
+ )
+
+ # --- SAM3 -----------------------------------------------------------
+ if importlib.util.find_spec("sam3") is not None:
+ print(f"{_OK} sam3 importable")
+ else:
+ hard_ok = False
+ print(
+ f"{_FAIL} sam3 not importable -- install SAM3 from source and put it on PYTHONPATH "
+ "(needed for segment/bake; not for check)"
+ )
+
+ # --- SAM3 checkpoint ------------------------------------------------
+ ckpt = os.environ.get("GEOLANGSPLAT_SAM_CKPT", "").strip()
+ if not ckpt:
+ hard_ok = False
+ print(f"{_FAIL} GEOLANGSPLAT_SAM_CKPT not set")
+ print(" export GEOLANGSPLAT_SAM_CKPT=/path/to/sam3.1_multiplex.pt " "(or pass --sam-ckpt)")
+ elif not pathlib.Path(ckpt).exists():
+ hard_ok = False
+ print(f"{_FAIL} GEOLANGSPLAT_SAM_CKPT points to a missing file: {ckpt}")
+ else:
+ size_gb = pathlib.Path(ckpt).stat().st_size / 1e9
+ print(f"{_OK} SAM3 checkpoint: {ckpt} ({size_gb:.1f} GB)")
+
+ print()
+ if hard_ok:
+ print("[doctor] all required checks passed -- `gls segment` should run here.")
+ else:
+ print("[doctor] one or more required checks FAILED -- fix the [FAIL] lines above.")
+ raise SystemExit(1)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_explore.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_explore.py
new file mode 100644
index 0000000..7de9991
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_explore.py
@@ -0,0 +1,114 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls explore` - interactive web catalog browser + fvdb viewer.
+
+An optional visual companion to `gls catalog`: launches the fvdb.viz viewer plus a small web
+UI for the one-class-at-a-time catalog flow. Type a class ("building", "car"); every instance
+lights up in its own colour; click one in the list and the viewer swaps to a cutout of
+just that object; Back returns to the highlighted view. Tune the confidence / split /
+min-size knobs live, and Export all to dump per-object ``.ply``. Handy for eyeballing and
+curation -- the stable surface is `gls segment` / `gls catalog` and the `segment` &
+`build_catalog` APIs.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName
+from ._common import BaseCommand, build_config, resolve_recipe
+
+
+@dataclass
+class Explore(BaseCommand):
+ """
+ Launch the interactive viewer + web query UI for live catalog browsing.
+
+ Example:
+
+ gls explore scene.ply --recipe satellite
+ # then open the web UI (default :8090) and the 3D viewer (default :8080)
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # UI backend: catalog (GLS instance catalog) | refimg (REF_IMG open-vocab heatmap
+ # with live gate sliders). refimg builds a teacher bake + reference-photo prototypes
+ # at startup and imports the dev-tree field builder (see --backend-root).
+ backend: Annotated[str, arg(aliases=["-b"])] = "catalog"
+
+ # Root dir holding query_field.py / render_queries.py / distill_field.py (for
+ # --backend refimg). Defaults to $GLS_QUERY_ROOT or the current working directory.
+ backend_root: Optional[str] = None
+
+ # Capture-type preset: auto (default) | satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # View source: render (synthetic orbit/ladder) | globe (dome) | images (real photos via --sfm).
+ view_source: Optional[str] = None
+
+ # Cap the synthesized/subsampled view count.
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Inside-out: place the eye at the scene core looking outward (interior scenes).
+ inside_out: bool = False
+
+ # Blend per-Gaussian peak score into the mean (0..1) for sparse-class recall.
+ peak: Optional[float] = None
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ # Viewer (fvdb.viz) port.
+ viewer_port: int = 8080
+
+ # Web query-UI port.
+ web_port: int = 8090
+
+ # Vulkan device id for the viewer.
+ vk_device_id: int = 0
+
+ def execute(self) -> None:
+ import os
+
+ from ..viewer import run_viewer
+
+ if not self.model_path.exists():
+ print(f"[explore] no such file: {self.model_path}", flush=True)
+ return
+
+ if self.backend_root:
+ os.environ["GLS_QUERY_ROOT"] = self.backend_root
+
+ cfg = build_config(
+ recipe=resolve_recipe(self.recipe, self.view_source),
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ inside_out=(True if self.inside_out else None),
+ peak=self.peak,
+ device=self.device,
+ viewer_port=self.viewer_port,
+ web_port=self.web_port,
+ vk_device_id=self.vk_device_id,
+ )
+ # recipe already folded into cfg (recipe-first); pass recipe=None so run_viewer
+ # does not re-apply it and clobber the explicit --view-source override.
+ run_viewer(str(self.model_path), config=cfg, recipe=None, backend=self.backend)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_render.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_render.py
new file mode 100644
index 0000000..061488e
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_render.py
@@ -0,0 +1,202 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls render` - render a splat .ply to a montage PNG (viewer-free preview).
+
+Renders any `GaussianSplat3d` .ply from a ring of orbit views and tiles them into
+one image. Handy for confirming a `segment`/`bake` overlay result over SSH or while
+travelling, where the interactive `fvdb.viz` viewer isn't reachable. The up axis is
+estimated from the scene's ground plane (same as the rest of the pipeline), so the
+montage comes out upright on object/COLMAP scenes too.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ._common import BaseCommand
+
+
+@dataclass
+class Render(BaseCommand):
+ """
+ Render a splat .ply to a single montage PNG (no interactive viewer needed).
+
+ Example:
+
+ gls segment scene.ply "table" -O ply_overlay -o table.ply
+ gls render table.ply -o table.png # eyeball the result as an image
+ """
+
+ # Path to the splat .ply (or .pt/.pth checkpoint) to render.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # Output PNG path (default: _views.png).
+ out: Annotated[Optional[pathlib.Path], arg(aliases=["-o"])] = None
+
+ # Number of views around the orbit.
+ n_views: Annotated[int, arg(aliases=["-n"])] = 8
+
+ # Elevation(s) above the horizon plane, in degrees (comma-separated for rings).
+ elevation: str = "15,45"
+
+ # Up axis: "auto" (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[str, arg(aliases=["-u"])] = "auto"
+
+ # Orbit radius as a fraction of the scene's bounding-box diagonal (globe off).
+ radius_frac: float = 1.1
+
+ # Globe zoom: <1 pulls the camera in, >1 backs it out (default frames the core).
+ zoom: Annotated[float, arg(aliases=["-z"])] = 1.0
+
+ # Globe (default): auto-frame the distance from the scene's dense core and sweep
+ # an upper dome of elevation rings looking DOWN (where bare-splat renders stay
+ # sharp). Turn off (--no-globe) for a fixed --radius-frac / --elevation orbit.
+ globe: bool = True
+
+ # Inside-out: place the eye at the scene core and look outward (interior scenes).
+ inside_out: bool = False
+
+ # COLMAP scene dir (with sparse/ + images/). When given, render from a sample of
+ # the scene's ACTUAL camera poses instead of a synthesized orbit -- crisp and
+ # upright by construction (this is the production / FRC path). Overrides --globe.
+ sfm: Annotated[Optional[pathlib.Path], arg(aliases=["-s"])] = None
+
+ # Per-tile render size (pixels).
+ size: int = 512
+
+ # Camera field of view (degrees).
+ fov: float = 55.0
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ def execute(self) -> None:
+ import math
+ import os
+
+ import numpy as np
+ import torch
+ from PIL import Image
+
+ from ..autoview import measure_geometry, resolve_up
+ from ..cameras import inside_out_cameras, intrinsics, orbit_cameras
+ from ..engine import _load_model, resolve_device
+ from ..views import auto_frame_radius, core_extent
+
+ if not self.model_path.exists():
+ print(f"[render] no such file: {self.model_path}", flush=True)
+ return
+
+ # Default outputs to scratch, never the (small) home dir: honour
+ # GEOLANGSPLAT_OUT_DIR if set, else write next to the source .ply.
+ if self.out is not None:
+ out = self.out
+ else:
+ base = os.environ.get("GEOLANGSPLAT_OUT_DIR")
+ name = self.model_path.stem + "_views.png"
+ out = (pathlib.Path(base) / name) if base else self.model_path.with_name(name)
+
+ dev = resolve_device(self.device)
+ model, _meta = _load_model(self.model_path, dev)
+ means = model.means.detach()
+
+ stats = measure_geometry(means)
+ up = resolve_up(self.up, means)
+ radius = float(stats.span) * self.radius_frac
+ try:
+ elevs = [float(e) for e in str(self.elevation).split(",") if e.strip()]
+ except ValueError:
+ elevs = [15.0, 45.0]
+
+ # Build the camera list. Three modes, in priority order:
+ # inside-out : eye at core looking out (interior scenes)
+ # globe : auto-framed radius + upper dome rings looking down
+ # orbit : fixed --radius-frac at the user's --elevation rings
+ grid_cols = None # when set, montage lays out rows=elevation, cols=azimuth
+ if self.sfm is not None:
+ # Render from the scene's real COLMAP poses (sampled evenly across the
+ # trajectory). No view synthesis, no up-axis guessing -- this is what the
+ # splat actually looks like from where it was captured.
+ from ..views import _colmap_c2w, _find_colmap, _read_colmap_images
+
+ _cam, img_path, _img_dir = _find_colmap(self.sfm)
+ recs = sorted(_read_colmap_images(img_path), key=lambda d: d["name"])
+ n = max(1, self.n_views * len(elevs))
+ idx = np.linspace(0, len(recs) - 1, num=min(n, len(recs))).round().astype(int)
+ fov = self.fov
+ cams = [np.linalg.inv(_colmap_c2w(recs[i])) for i in idx]
+ elevs = [] # not a ring layout
+ elif self.inside_out:
+ center = means.float().median(0).values.cpu().numpy().astype(np.float64)
+ fov = max(self.fov, 80.0)
+ cams = [w2c for w2c, _ in inside_out_cameras(center, elevs, self.n_views, up=up)]
+ elif self.globe:
+ from ..views import dome_radius_scale
+
+ center, _r = core_extent(model.means)
+ fov = self.fov
+ radius = auto_frame_radius(model, center, up, fov=fov, zoom=self.zoom)
+ elevs = [18.0, 38.0, 58.0, 78.0] # dome: low-oblique (sides) -> near-nadir, looking down
+ grid_cols = self.n_views
+ e_lo, e_hi = min(elevs), max(elevs)
+ cams = []
+ for elev in elevs:
+ # Pull low/oblique rings closer (matches the segment dome path) so
+ # blocked side-on subjects fill more of the frame.
+ ring_radius = radius * dome_radius_scale(elev, e_lo, e_hi, 0.72)
+ for ai in range(self.n_views):
+ az = 360.0 * ai / self.n_views
+ cams.append(
+ orbit_cameras(center, elev, ring_radius, num_azimuth=1, azimuth_offset_deg=az, up=up)[0][0]
+ )
+ else:
+ center = (
+ ((means.float().min(0).values + means.float().max(0).values) / 2.0).cpu().numpy().astype(np.float64)
+ )
+ fov = self.fov
+ cams = []
+ for elev in elevs:
+ for ai in range(self.n_views):
+ az = 360.0 * ai / self.n_views
+ cams.append(orbit_cameras(center, elev, radius, num_azimuth=1, azimuth_offset_deg=az, up=up)[0][0])
+
+ K_np = intrinsics(fov, self.size, self.size)
+ K = torch.from_numpy(K_np).float().unsqueeze(0).to(dev)
+ tiles: list[Image.Image] = []
+ with torch.no_grad():
+ for w2c_np in cams:
+ w2c = torch.from_numpy(w2c_np).float().to(dev)
+ img, _a = model.render_images_and_depths(
+ world_to_camera_matrices=w2c.unsqueeze(0),
+ projection_matrices=K,
+ image_width=self.size,
+ image_height=self.size,
+ near=0.01,
+ far=1e12,
+ )
+ rgb = (img[0, ..., :3].clamp(0, 1).cpu().numpy() * 255).astype(np.uint8)
+ tiles.append(Image.fromarray(rgb))
+
+ # Lay out rows=elevation, cols=azimuth for the ring modes (clean pattern);
+ # otherwise a square grid.
+ cols = grid_cols or math.ceil(math.sqrt(len(tiles)))
+ rows = math.ceil(len(tiles) / cols)
+ montage = Image.new("RGB", (cols * self.size, rows * self.size), (10, 12, 16))
+ for i, t in enumerate(tiles):
+ montage.paste(t, ((i % cols) * self.size, (i // cols) * self.size))
+ out.parent.mkdir(parents=True, exist_ok=True)
+ montage.save(out)
+ if self.sfm is not None:
+ print(f"[render] {len(tiles)} real-camera views (from {self.sfm}) -> {out}", flush=True)
+ else:
+ mode = "inside-out" if self.inside_out else ("globe" if self.globe else "orbit")
+ print(
+ f"[render] {len(tiles)} {mode} views ({len(elevs)} elev x {self.n_views} az) "
+ f"-> {out} (up={tuple(round(float(x), 2) for x in up)})",
+ flush=True,
+ )
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_segment.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_segment.py
new file mode 100644
index 0000000..3061075
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_segment.py
@@ -0,0 +1,195 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls segment` - segment a splat by a single text prompt.
+
+The one command you always use to query. It does the right thing automatically:
+
+* if a warm engine is already serving this model (you ran ``gls serve``), it
+ attaches and answers instantly;
+* otherwise it runs one-shot -- build, query, write, exit -- leaving nothing
+ behind.
+
+So there are two ways to use GeoLangSplat: one-shot ``gls segment`` (above), or
+``gls serve`` to build an engine once, ``gls segment`` to query it many times,
+``gls stop`` to free it. Fine-grained tuning beyond ``--select`` lives in the
+Python API.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Annotated, Literal, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName
+from ._common import BaseCommand, build_config, resolve_recipe
+
+
+@dataclass
+class Segment(BaseCommand):
+ """
+ Open-vocabulary 3D segmentation of a Gaussian splat from a text prompt.
+
+ SAM3 segments the prompt in 2D over views of the splat; the masks are lifted
+ to per-Gaussian scores by alpha-weighted back-projection. Aerial and satellite
+ captures use the same path -- just pick a recipe.
+
+ Examples:
+
+ # one-shot highlight overlay (synthesized views, no photos needed)
+ gls segment model.ply "house" --recipe satellite -O ply_overlay -o house.ply
+
+ # engine mode: build once with `gls serve`, then query instantly
+ gls serve model.ply --recipe satellite -b
+ gls segment model.ply "house" -O ply_overlay -o house.ply # attaches, instant
+ gls segment model.ply "road" -O ply_overlay -o road.ply # attaches, instant
+ gls stop model.ply
+
+ # use the scene's real photos when it ships a COLMAP reconstruction
+ gls segment model.ply "tree" --view-source images --sfm /path/scene -O report -o tree.csv
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # Text prompt to segment (e.g. "house", "road", "tree").
+ prompt: tyro.conf.Positional[str]
+
+ # Capture-type preset: auto (default) | satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # Output mode: mask (no file) | ply_segmented | ply_overlay | report.
+ output: Annotated[Literal["mask", "ply_segmented", "ply_overlay", "report"], arg(aliases=["-O"])] = "ply_overlay"
+
+ # Where to write the output (.ply for ply_*, .csv/.npz for report).
+ out_path: Annotated[pathlib.Path, arg(aliases=["-o"])] = pathlib.Path("segmented.ply")
+
+ # Selection score threshold -- the main knob. Higher = stricter/fewer gaussians.
+ select: Annotated[Optional[float], arg(aliases=["-t"])] = None
+
+ # Blend per-Gaussian peak score into the mean (0..1) to recover diluted, sparsely
+ # observed classes (trees, grass) without broadly lowering the threshold.
+ peak: Optional[float] = None
+
+ # View source: render (synthetic orbit/ladder) |
+ # globe (auto-framed dome for objects/interiors) | images (real photos via --sfm).
+ view_source: Optional[Literal["render", "globe", "images"]] = None
+
+ # Cap the synthesized/subsampled view count (build VRAM & query latency ~ views).
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Inside-out: place the eye at the scene core looking outward (interior scenes).
+ inside_out: bool = False
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # Suppress near-synonyms by competing against the recipe's distractor set.
+ compete: bool = False
+
+ # Print the total wall time (incl. python/torch import + CUDA init) at the end.
+ profile: bool = False
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ # Resident embedding-cache dtype (VRAM lever): auto|amp|fp16|bf16. "amp" matches
+ # the autocast dtype and halves the cache when SAM3's features are fp32.
+ cache_dtype: str = "auto"
+
+ # Low-VRAM streaming: encode + score + project + evict one view at a time, so peak
+ # VRAM is bounded to ~one view (vs. caching every view). ON by default for one-shot;
+ # it re-encodes per query, so for fast repeated queries use `gls serve` (warm cache).
+ # Pass --no-low-vram for the full-quality, higher-VRAM one-shot build.
+ low_vram: bool = True
+
+ # Streaming early-stop: auto|on|off. "auto" stops once enough diverse views agree
+ # for a COMPACT subject (object/dome), but streams every view for aerial/satellite
+ # (the class is spread across the scene, so stopping early loses recall). Force
+ # "off" to always stream all views (max recall), "on" to always early-stop.
+ stream_early_stop: Literal["auto", "on", "off"] = "auto"
+
+ # Views encoded (batched) then evicted per streaming step. Higher = faster encode,
+ # more transient VRAM; lower = tighter VRAM. Only affects --low-vram.
+ stream_chunk: Optional[int] = None
+
+ def execute(self) -> None:
+ from ..ipc import daemon_alive, default_socket
+
+ if not self.model_path.exists():
+ print(f"[gls] no such file: {self.model_path}", flush=True)
+ return
+
+ sock = default_socket(self.model_path)
+
+ # If `gls serve` already built an engine for this model, attach and answer
+ # instantly. The engine's build settings win, so this path ignores the
+ # build-affecting flags (-r/-s); only query flags (-t/--compete/-O/-o)
+ # apply. Otherwise run one-shot in-process and exit, leaving nothing behind.
+ if daemon_alive(sock):
+ self._via_daemon(sock)
+ else:
+ self._one_shot()
+
+ def _via_daemon(self, sock: str) -> None:
+ from ..ipc import request
+
+ req = {
+ "prompt": self.prompt,
+ "output": self.output,
+ "out_path": (str(self.out_path) if self.output != "mask" else None),
+ "select": self.select,
+ "compete": (True if self.compete else None),
+ }
+ resp = request(sock, req)
+ if not resp.get("ok"):
+ print(f"[gls] error: {resp.get('error') or 'unknown'}", flush=True)
+ return
+ t = resp.get("t")
+ ts = f"{t:.2f}s" if isinstance(t, (int, float)) else "?"
+ print(f'[gls] "{resp["prompt"]}" -> {resp["n"]:,} / {resp["N"]:,} gaussians ({ts}, warm)', flush=True)
+ if resp.get("path"):
+ print(f"[gls] wrote {resp['output']} -> {resp['path']}", flush=True)
+
+ def _one_shot(self) -> None:
+ from ..api import segment # lazy: keeps the daemon-attach path torch-free
+
+ cfg = build_config(
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ select=self.select,
+ peak=self.peak,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ inside_out=(True if self.inside_out else None),
+ compete=(True if self.compete else None),
+ device=self.device,
+ cache_dtype=(self.cache_dtype if self.cache_dtype != "auto" else None),
+ low_vram=self.low_vram,
+ stream_early_stop=(self.stream_early_stop if self.stream_early_stop != "auto" else None),
+ stream_chunk=self.stream_chunk,
+ )
+ result = segment(
+ self.model_path,
+ self.prompt,
+ config=cfg,
+ recipe=resolve_recipe(self.recipe, self.view_source),
+ output=self.output,
+ out_path=self.out_path if self.output != "mask" else None,
+ )
+ print(
+ f'[gls] "{self.prompt}" -> {result.num_selected:,} / {result.scores.shape[0]:,} gaussians',
+ flush=True,
+ )
+ if self.output != "mask":
+ print(f"[gls] wrote {self.output} -> {self.out_path}", flush=True)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_serve.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_serve.py
new file mode 100644
index 0000000..a0ecd5f
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_serve.py
@@ -0,0 +1,260 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls serve` - build a warm engine once, then query it with `gls segment`.
+
+Builds the scene cache (load + views + SAM3 embeddings) ONCE and listens on a
+Unix socket; query it afterwards with plain ``gls segment`` (which auto-detects
+the warm engine and answers instantly). This is the "engine" way to use
+GeoLangSplat: ``gls serve`` -> ``gls segment`` (many times) -> ``gls stop``. The
+engine auto-stops after ``--keep-alive`` idle minutes so it never lingers.
+"""
+from __future__ import annotations
+
+import atexit
+import os
+import pathlib
+import signal
+import socket
+import time
+from dataclasses import dataclass
+from typing import Annotated, Optional
+
+import tyro
+from tyro.conf import arg
+
+from ..config import RecipeName, apply_recipe
+from ._common import resolve_recipe
+from ..ipc import (
+ DEFAULT_KEEP_ALIVE_MIN,
+ default_socket,
+ handle_request,
+ recv_obj,
+ remove_runtime_files,
+ send_obj,
+ write_pidfile,
+)
+from ._common import BaseCommand, build_config
+
+
+@dataclass
+class Serve(BaseCommand):
+ """
+ Start a warm engine daemon, then query it with `gls segment`.
+
+ Example:
+
+ gls serve model.ply --recipe satellite -b # build once, detach
+ gls segment model.ply "building" -O ply_overlay -o b.ply # auto-uses it
+ gls stop model.ply
+ """
+
+ # Path to the input Gaussian splat .ply.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # Capture-type preset: satellite | satellite_dense | aerial.
+ recipe: Annotated[Optional[RecipeName], arg(aliases=["-r"])] = None
+
+ # View source: render (synthetic orbit/ladder) | globe (dome) | images (real photos).
+ view_source: Optional[str] = None
+
+ # Cap the synthesized/subsampled view count (build VRAM & query latency ~ views).
+ max_views: Optional[int] = None
+
+ # If >0, trim the view plan to fit this VRAM budget (GB).
+ vram_budget_gb: Optional[float] = None
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[Optional[str], arg(aliases=["-u"])] = None
+
+ # Path to the SfM/COLMAP scene (required for --view-source images).
+ sfm: Annotated[str, arg(aliases=["-s"])] = ""
+
+ # Default score threshold for this session (per-query --select still wins).
+ select: Annotated[Optional[float], arg(aliases=["-t"])] = None
+
+ # Default concept competition for this session.
+ compete: bool = False
+
+ # Fast queries: aggregate only this many angularly-spread views per query (0 = all
+ # views, full quality). The grounding decode is the per-query cost, so k of N views
+ # is ~N/k faster -- a session-wide interactive speed lever (e.g. for DGX Spark).
+ fast_views: Optional[int] = None
+
+ # Launch detached in the background and return the terminal (instead of blocking).
+ background: Annotated[bool, arg(aliases=["-b"])] = False
+
+ # Auto-stop after this many idle minutes (sliding; reset only by real queries,
+ # not status checks). 0 or negative = stay up until `gls stop`. The one knob
+ # for the engine's lifetime -- a forgotten daemon frees the GPU on its own.
+ keep_alive: float = DEFAULT_KEEP_ALIVE_MIN
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ # Advanced: socket the daemon binds (auto-derived from the model path; the
+ # warm-spawn path passes this through, so it must remain a real CLI option).
+ socket_path: Optional[str] = None
+
+ def execute(self) -> None:
+ from ..engine import load_or_build_engine
+ from ..ipc import daemon_alive, spawn_daemon, wait_ready
+
+ if not self.model_path.exists():
+ print(f"[serve] no such file: {self.model_path}", flush=True)
+ return
+
+ sock_path = self.socket_path or default_socket(self.model_path)
+ recipe = resolve_recipe(self.recipe, self.view_source)
+
+ if self.background:
+ if daemon_alive(sock_path):
+ print(f"[serve] already running for {self.model_path}", flush=True)
+ return
+ print("[serve] launching in background (streaming build below)...", flush=True)
+ proc, log = spawn_daemon(
+ self.model_path,
+ recipe=recipe,
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ select=self.select,
+ compete=self.compete,
+ fast_views=self.fast_views,
+ device=self.device,
+ socket_path=sock_path,
+ keep_alive=self.keep_alive,
+ )
+ if wait_ready(sock_path, proc=proc, log_path=log):
+ print(
+ f"[serve] ready.\n"
+ f'[serve] query it: gls segment {self.model_path} "" -O ply_overlay -o out.ply\n'
+ f"[serve] stop it: gls stop {self.model_path}",
+ flush=True,
+ )
+ else:
+ print(f"[serve] engine failed to start (see above / {log})", flush=True)
+ return
+
+ cfg = build_config(
+ sfm=self.sfm or None,
+ view_source=self.view_source,
+ max_views=self.max_views,
+ vram_budget_gb=self.vram_budget_gb,
+ up=self.up,
+ select=self.select,
+ compete=(True if self.compete else None),
+ fast_views=self.fast_views,
+ device=self.device,
+ )
+ apply_recipe(cfg, recipe)
+ t_start = time.time()
+ engine = load_or_build_engine(self.model_path, cfg) # one-time build
+
+ if os.path.exists(sock_path):
+ os.unlink(sock_path)
+ srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ srv.bind(sock_path)
+ srv.listen(8)
+ write_pidfile(sock_path)
+
+ # Guaranteed cleanup: remove socket + pidfile however we exit (normal
+ # shutdown, idle, signal, or crash). atexit covers the normal/crash paths;
+ # the signal handlers convert SIGTERM/SIGINT/SIGHUP (what `gls stop`, a
+ # bare `kill`, or a closing terminal send) into a clean break of the loop.
+ def _cleanup() -> None:
+ try:
+ srv.close()
+ finally:
+ remove_runtime_files(sock_path)
+
+ atexit.register(_cleanup)
+
+ class _Stop(Exception):
+ pass
+
+ def _on_signal(signum, _frame):
+ raise _Stop(signal.Signals(signum).name)
+
+ for _sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
+ try:
+ signal.signal(_sig, _on_signal)
+ except (ValueError, OSError):
+ pass
+
+ keep_seconds = self.keep_alive * 60 if self.keep_alive and self.keep_alive > 0 else None
+ idle_note = f"auto-stops after {self.keep_alive:g} min idle" if keep_seconds else "stays up until `gls stop`"
+ last_activity = time.time() # bumped ONLY by real queries, so polling status can't keep it alive
+ print(
+ f"[serve] engine warm ({engine.N:,} gaussians), listening ({idle_note}).\n"
+ f'[serve] query it: gls segment {self.model_path} "" -O ply_overlay -o out.ply\n'
+ f"[serve] check it: gls status\n"
+ f"[serve] stop it: Ctrl-C (or: gls stop {self.model_path})",
+ flush=True,
+ )
+ try:
+ while True:
+ if keep_seconds is not None:
+ remaining = last_activity + keep_seconds - time.time()
+ if remaining <= 0:
+ print(f"[serve] idle for {self.keep_alive:g} min -> shutting down", flush=True)
+ break
+ srv.settimeout(min(remaining, 30.0)) # wake periodically to re-check the deadline
+ else:
+ srv.settimeout(None)
+ try:
+ conn, _ = srv.accept()
+ except socket.timeout:
+ continue # not idle yet; loop re-checks the real deadline
+ try:
+ req = recv_obj(conn)
+ cmd = req.get("cmd") or req.get("prompt")
+ if cmd in (":shutdown", "shutdown"):
+ send_obj(conn, {"ok": True, "msg": "shutting down"})
+ print("[serve] shutdown requested", flush=True)
+ break
+ if cmd in (":ping", "ping"):
+ send_obj(conn, {"ok": True, "N": engine.N})
+ continue # liveness check: does NOT reset the idle timer
+ if cmd in (":status", "status"):
+ now = time.time()
+ idle_for = now - last_activity
+ send_obj(
+ conn,
+ {
+ "ok": True,
+ "pid": os.getpid(),
+ "model": str(self.model_path),
+ "N": engine.N,
+ "recipe": self.recipe,
+ "uptime_s": now - t_start,
+ "idle_s": idle_for,
+ "idle_remaining_s": (keep_seconds - idle_for) if keep_seconds else None,
+ "vram": engine.vram_report(),
+ },
+ )
+ continue # status check: does NOT reset the idle timer
+ last_activity = time.time()
+ resp = handle_request(engine, req)
+ send_obj(conn, resp)
+ if resp.get("ok"):
+ t = resp.get("t")
+ ts = f"{t:.2f}s" if isinstance(t, (int, float)) else "?"
+ print(f'[serve] "{resp["prompt"]}" -> {resp["n"]:,}/{resp["N"]:,} ({ts})', flush=True)
+ else:
+ print(f"[serve] error: {resp.get('error')}", flush=True)
+ except Exception as e: # keep the daemon alive on a bad request
+ try:
+ send_obj(conn, {"ok": False, "error": str(e)})
+ except Exception:
+ pass
+ print(f"[serve] request failed: {e}", flush=True)
+ finally:
+ conn.close()
+ except (_Stop, KeyboardInterrupt) as e:
+ reason = e.args[0] if getattr(e, "args", None) else "interrupt"
+ print(f"\n[serve] {reason} -> stopping", flush=True)
+ finally:
+ _cleanup()
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_show.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_show.py
new file mode 100644
index 0000000..c9235a6
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_show.py
@@ -0,0 +1,94 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls show` - open a splat .ply in the fvdb viewer to eyeball a result.
+
+Renders any `GaussianSplat3d` .ply (or checkpoint) in `fvdb.viz`, the same stack
+as `frgs show`. Our `segment`/`bake` outputs are standard splats, so the overlay
+tint / recolouring shows up directly -- handy for confirming a query worked.
+For LIVE prompt-by-prompt exploration use `geolangsplat.viewer.run_viewer` instead.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Annotated
+
+import tyro
+from tyro.conf import arg
+
+from ._common import BaseCommand
+
+
+@dataclass
+class Show(BaseCommand):
+ """
+ View a splat .ply (e.g. a `segment` overlay output) in the fvdb viewer.
+
+ Example:
+
+ gls segment scene.ply "house" -r satellite -O ply_overlay -o house.ply
+ gls show house.ply # confirm the tinted result in 3D
+ """
+
+ # Path to the splat .ply (or .pt/.pth checkpoint) to display.
+ model_path: tyro.conf.Positional[pathlib.Path]
+
+ # Viewer port.
+ viewer_port: Annotated[int, arg(aliases=["-p"])] = 8080
+
+ # IP to bind the viewer on.
+ ip: str = "0.0.0.0"
+
+ # Vulkan device id for the viewer.
+ vk_device_id: int = 0
+
+ # Scene up axis: auto (estimate from ground plane) or +z,-z,+y,-y,+x,-x.
+ up: Annotated[str, arg(aliases=["-u"])] = "auto"
+
+ # Device.
+ device: Annotated[str, arg(aliases=["-d"])] = "cuda"
+
+ def execute(self) -> None:
+ import time
+
+ import numpy as np
+
+ from ..autoview import resolve_up
+ from ..cameras import orbit_cameras
+ from ..engine import _load_model, resolve_device
+
+ if not self.model_path.exists():
+ print(f"[show] no such file: {self.model_path}", flush=True)
+ return
+
+ import fvdb.viz as viz
+
+ dev = resolve_device(self.device)
+ model, _meta = _load_model(self.model_path, dev)
+ viz.init(ip_address=self.ip, port=self.viewer_port, vk_device_id=self.vk_device_id)
+ scene = viz.get_scene("GeoLangSplat Viewer")
+ scene.add_gaussian_splat_3d("splat", model)
+ # Orient + frame the camera (object/COLMAP plys aren't z-up, and the default
+ # near clip is too far to zoom into small scenes).
+ try:
+ up = resolve_up(self.up, model.means)
+ m = model.means.detach().float().cpu().numpy()
+ lo = np.quantile(m, 0.02, axis=0)
+ hi = np.quantile(m, 0.98, axis=0)
+ center = (lo + hi) / 2
+ radius = max(float(np.linalg.norm(hi - lo)) / 2, 0.001)
+ scene.camera_up_direction = up
+ scene.camera_near = max(radius * 0.002, 0.0001)
+ scene.camera_far = radius * 100
+ c2w = orbit_cameras(center, 18, radius * 2.2, num_azimuth=1, azimuth_offset_deg=35, up=up)[0][1]
+ scene.set_camera_lookat(c2w[:3, 3], center, up)
+ except Exception as e:
+ print(f"[show] could not frame scene (up={self.up!r}): {e}", flush=True)
+ if hasattr(viz, "show"):
+ viz.show()
+ print(f"[show] viewer at http://:{self.viewer_port} (Ctrl-C to stop)", flush=True)
+ try:
+ time.sleep(10**9)
+ except KeyboardInterrupt:
+ print("\n[show] bye", flush=True)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_status.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_status.py
new file mode 100644
index 0000000..8e70a8c
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_status.py
@@ -0,0 +1,60 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls status` - show any warm engines running on this machine.
+
+Like ``docker ps`` / ``ollama ps``: a quick answer to "is anything still loaded,
+and how do I stop it?" so a background engine is never a mystery. One-shot
+``gls segment`` keeps no state, so if nothing shows here, nothing is running.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..ipc import cleanup_stale, list_sockets, request_status
+from ._common import BaseCommand
+
+
+def _fmt_secs(s) -> str:
+ if s is None:
+ return "-"
+ s = int(s)
+ if s < 60:
+ return f"{s}s"
+ return f"{s // 60}m{s % 60:02d}s"
+
+
+@dataclass
+class Status(BaseCommand):
+ """List running warm engines (and how to stop them)."""
+
+ def execute(self) -> None:
+ running = []
+ for sock in list_sockets():
+ st = request_status(sock)
+ if st:
+ running.append((sock, st))
+ else:
+ cleanup_stale(sock) # drop the socket file of a dead daemon
+
+ if not running:
+ print("[status] no warm engines running.")
+ print("[status] start one: gls serve -b")
+ return
+
+ print(f"[status] {len(running)} warm engine(s) running:\n")
+ for _sock, st in running:
+ model = st.get("model", "?")
+ print(f" {model}")
+ print(
+ f" pid {st.get('pid')} | {st.get('N', 0):,} gaussians" f" | recipe {st.get('recipe') or '-'}"
+ )
+ print(
+ f" up {_fmt_secs(st.get('uptime_s'))}"
+ f" | idle {_fmt_secs(st.get('idle_s'))}"
+ f" | auto-stop in {_fmt_secs(st.get('idle_remaining_s'))}"
+ )
+ if st.get("vram"):
+ print(f" {st['vram']}")
+ print(f" stop: gls stop {model}")
+ print("\n[status] stop everything: gls stop")
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_stop.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_stop.py
new file mode 100644
index 0000000..5bb9ab3
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_stop.py
@@ -0,0 +1,49 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""`gls stop` - shut down a warm engine started by `gls serve`.
+
+With a model path, stops that engine. With no arguments, stops every warm engine
+on this machine (handy when you forgot what's running -- see `gls status`).
+Shutdown is guaranteed: polite IPC first, then SIGTERM/SIGKILL, then the socket
+and pidfile are cleaned up either way.
+"""
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Optional
+
+import tyro
+
+from ..ipc import default_socket, kill_daemon, list_sockets
+from ._common import BaseCommand
+
+
+@dataclass
+class Stop(BaseCommand):
+ """Stop a warm background engine, or all of them if no model is given."""
+
+ # Splat the daemon was started with (omit to stop ALL running engines).
+ model_path: tyro.conf.Positional[Optional[pathlib.Path]] = None
+
+ # Explicit Unix socket path (overrides the per-model default).
+ socket_path: Optional[str] = None
+
+ def execute(self) -> None:
+ if self.model_path is None and not self.socket_path:
+ self._stop_all()
+ return
+ sock = self.socket_path or default_socket(self.model_path)
+ # kill_daemon guarantees the process is gone (IPC -> SIGTERM -> SIGKILL)
+ # and removes the socket + pidfile either way.
+ status = kill_daemon(sock)
+ print(f"[stop] {status} ({self.model_path or sock})", flush=True)
+
+ def _stop_all(self) -> None:
+ stopped = 0
+ for sock in list_sockets():
+ status = kill_daemon(sock)
+ if status in ("stopped", "killed"):
+ stopped += 1
+ print(f"[stop] stopped {stopped} engine(s)" if stopped else "[stop] no engines were running", flush=True)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/config.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/config.py
new file mode 100644
index 0000000..a03cdc2
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/config.py
@@ -0,0 +1,306 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Unified configuration and capture-type recipes for GeoLangSplat.
+
+A single :class:`GeoLangSplatConfig` drives both aerial and satellite captures.
+:data:`RECIPES` are named presets that fill sensible defaults for a capture
+type; :func:`apply_recipe` only fills fields the caller left at their default, so
+an explicitly set value always wins over the preset.
+"""
+from __future__ import annotations
+
+import difflib
+import os
+from dataclasses import dataclass, field, fields
+from typing import Literal
+
+from .errors import GeoLangSplatError
+
+# Canonical recipe names. Used as a CLI choice type so `tyro` validates the value
+# (and can tab-complete it) instead of failing deep inside apply_recipe.
+RecipeName = Literal["auto", "satellite", "satellite_dense", "aerial"]
+
+# Generic scene vocabulary used as the default competition set (when competition
+# is enabled without an explicit distractor list).
+DEFAULT_DISTRACTORS: tuple[str, ...] = (
+ "building",
+ "ground",
+ "road",
+ "grass",
+ "tree",
+ "water",
+ "car",
+ "sidewalk",
+)
+
+
+@dataclass(frozen=True)
+class Tier:
+ """One level of the synthesized camera ladder (used when ``view_source='render'``).
+
+ A tier renders a grid of orbit cameras at a given distance (``radius``), set
+ of ``elevations``, and field of view (``fov_deg``; lower = optical zoom-in,
+ which enlarges small objects). Tiers stack to form a multi-scale view set.
+ """
+
+ radius: float
+ elevations: tuple[float, ...]
+ fov_deg: float = 50.0
+ grid: int = 5
+ num_azimuth: int = 1
+
+
+@dataclass
+class GeoLangSplatConfig:
+ """All knobs for the unified open-vocabulary segmentation pipeline.
+
+ Grouped by stage. The most commonly tuned knobs are ``view_source``,
+ ``select``, ``min_weight`` and ``distractors``; prefer a recipe and override
+ only what you need.
+ """
+
+ # --- scene / IO --------------------------------------------------------
+ ply: str = ""
+ sfm: str = ""
+
+ # --- view generation ---------------------------------------------------
+ # When True, derive the view set from scene geometry (footprint, height,
+ # capture type) instead of a fixed tier ladder. The "auto" recipe sets this.
+ auto_views: bool = False
+ # "render" = synthesize an orbit/ladder from scene geometry;
+ # "globe" = synthesized object/interior close dome;
+ # "images" = use the scene's ground-truth SfM photos (COLMAP).
+ view_source: str = "render"
+ # Inside-out: anchor the eye at the scene core and look OUTWARD (interior /
+ # unbounded room captures, where an outside-in orbit only sees wall backs).
+ inside_out: bool = False
+ inside_out_fov: float = 80.0 # wide FOV for inside-out sweeps
+ # Cap on the synthesized/subsampled view count (build VRAM & query latency ~
+ # view count). 0 = no cap beyond the recipe/auto plan.
+ max_views: int = 200
+ vram_budget_gb: float = 0.0 # if >0, trim the view plan to fit this VRAM budget
+ # If >0, single-prompt queries decode only this many angularly-spread views
+ # (no competition) for a faster interactive answer; 0 uses the full view set.
+ fast_views: int = 0
+ # Low-VRAM streaming lift: encode + score + project + EVICT one view at a time
+ # instead of caching every view's embedding. Peak VRAM is bounded to ~one view
+ # regardless of count, at the cost of no warm reuse (re-encodes per call) -- it
+ # backs the one-shot path while `gls serve` keeps the fast all-views cache.
+ low_vram: bool = False
+ stream_chunk: int = 8 # views encoded (batched) then evicted per streaming step
+ # Early-stop only helps for a COMPACT subject (object/dome capture): a few
+ # angularly-diverse views see the whole thing. On aerial/satellite the class is
+ # spread across the scene, so stopping early guts recall -- there we stream every
+ # view (bounded VRAM, full recall). "auto" = on only for object/globe captures.
+ stream_early_stop: str = "auto" # "auto" | "on" | "off"
+ view_cap: int = 0 # hard ceiling on streamed views (0 = all candidate views)
+ agree_k: int = 12 # early-stop: views with a hit that must agree (single prompt)
+ min_azimuth_spread: float = 120.0 # ...spanning at least this much azimuth (diff sides)
+ converge_frac: float = 0.02 # ...with the selection changing < this fraction to stop
+ n_views: int = 90 # how many ground-truth images to subsample (view_source="images")
+ orbit_radius_frac: float = 1.2 # camera distance as a fraction of scene span
+ lift_res: int = 640 # resolution at which contributing-Gaussian weights are rendered
+ size: int = 640 # render size for synthesized views (view_source="render")
+ grid_frac: float = 0.9 # footprint fraction covered by the render grid
+ zoom: float = 1.0 # global multiplier on every tier radius
+ # Dome (globe) view tuning. dome_low_zoom pulls the LOW/oblique rings closer
+ # (radius scale at the lowest ring; 1.0 at near-nadir) so side-on subjects that
+ # were "covered/blocked" fill more pixels. reject_blur enables the per-view
+ # quality gate on the rendered dome: a view is rejected when too little of its
+ # center is covered by the splat (view_min_coverage -> see-through/empty), when a
+ # near foreground blob blocks the subject (view_max_occlusion), or when it is far
+ # blurrier than the set median (blur_rel). Rejected views are resampled once from
+ # a higher, closer angle (to clear floor clutter) and kept only if they pass.
+ dome_low_zoom: float = 0.72
+ reject_blur: bool = True
+ blur_rel: float = 0.40
+ view_min_coverage: float = 0.22 # min central alpha coverage to keep a dome view
+ view_max_occlusion: float = 0.45 # max fraction of the center blocked by a near floater
+ # Scene up axis: "auto" estimates it from the dominant ground plane (RANSAC);
+ # otherwise one of +z,-z,+y,-y,+x,-x. Drives view generation and viewer framing.
+ up: str = "auto"
+ tiers: tuple[Tier, ...] = (Tier(radius=120.0, elevations=(80.0, 60.0), fov_deg=50.0, grid=5),)
+
+ # --- lift / scoring ----------------------------------------------------
+ # "alpha" = alpha-weighted back-projection (continuous, recommended);
+ # "band" = depth-band footprint voting (legacy, render view source).
+ lift: str = "alpha"
+ top_k: int = 8 # top-k contributing Gaussians kept per pixel (alpha lift)
+ peak: float = 0.0 # blend weight of per-Gaussian peak score into the mean (0 = mean only)
+ min_weight: float = 0.03 # min accumulated render weight for a Gaussian to count (floater cull)
+ depth_band: float = 0.0025 # absolute two-sided depth band (band lift)
+ depth_tol: float = 0.05 # multiplicative depth tolerance for visibility (render cache)
+ foot: int = 0 # max-pool radius for footprint mask sampling (band lift)
+ view_thresh: float = 0.35 # per-view score floor when counting consensus hits (band lift)
+ min_views: int = 3 # min number of views that must hit a Gaussian (band lift consensus)
+ strong_select: float = 0.99 # score above which a single strong view is enough (band lift)
+ # Multi-view consensus gate (alpha lift): require a Gaussian to be supported by
+ # enough views whose per-view score clears consensus_thr before it can be selected.
+ consensus: bool = False
+ consensus_thr: float = 0.3 # per-view score floor that counts as support
+ consensus_frac: float = 0.0 # required supporting views as a fraction of the views that saw it
+ consensus_min: int = 1 # absolute minimum number of supporting views
+
+ # --- selection / competition ------------------------------------------
+ select: float = 0.30 # score threshold to select a Gaussian for the query
+ margin: float = 0.08 # query must beat the best distractor by this margin
+ # "fixed" = keep candidates with qscore >= select (absolute floor);
+ # "relative" = keep candidates with qscore >= select_rel * max(qscore) -- robust
+ # to prompts whose grounding scores are globally weak (the main recall killer).
+ select_mode: str = "fixed"
+ select_rel: float = 0.5 # relative-mode threshold as a fraction of the peak candidate score
+ min_keep: int = 0 # fixed-mode non-empty guard: keep top-k candidates if the threshold selects none
+ # Concept competition is OFF by default for single queries ("show me X" should
+ # just work); turn it on to suppress near-synonyms. The fixed-vocab bake does
+ # its own implicit competition (argmax across the vocabulary), independent of this.
+ compete: bool = False
+ # Generic scene vocabulary used as the competition set when competition is on
+ # and no distractors are supplied (recipes override with curated sets).
+ distractors: tuple[str, ...] = DEFAULT_DISTRACTORS
+
+ # --- spatial cleanup ---------------------------------------------------
+ clean3d: bool = True # drop isolated selected Gaussians via voxel cull
+ voxel_frac: float = 0.01 # voxel size as a fraction of scene span
+ min_pts: int = 4 # min selected Gaussians per voxel to survive
+
+ # --- score smoothing (training-free voxel regularization) -------------
+ # Blend each Gaussian's score with its voxel-neighborhood mean *before*
+ # thresholding: fills object interiors (recall) and pulls isolated false
+ # positives down toward their empty neighborhood (precision).
+ smooth: bool = False
+ smooth_beta: float = 0.5 # blend weight toward the neighborhood mean (0 = off)
+ smooth_vox_frac: float = 0.02 # smoothing voxel size as a fraction of scene span
+ smooth_weighted: bool = True # weight the neighborhood mean by render contribution
+
+ # --- SAM3 dual-head fusion (training-free instance + semantic) --------
+ # Blend SAM3's presence-gated instance head with its dense prompt-conditioned
+ # semantic head in one decode. Recovers amorphous/"stuff" classes the instance
+ # head hard-filters out (the dominant recall failure) at no extra forward cost.
+ dual_head: bool = False
+ sem_weight: float = 0.5 # blend weight of the semantic head (0 = instance only)
+ sem_mode: str = "mean" # "mean" = (1-w)*inst + w*sem ; "max" = max((1-w)*inst, w*sem)
+
+ # --- instances / catalog (connected-component object split) -----------
+ inst_link_frac: float = 0.02 # voxel/link size as a fraction of scene span
+ inst_min_size: int = 25 # drop instances smaller than this many Gaussians
+ cat_iou: float = 0.5 # merge objects from different prompts when their 3D IoU >= this
+
+ # --- multi-class labelling (bake) -------------------------------------
+ tau: float = 0.15 # absolute confidence floor for a label
+ delta: float = 0.02 # top1-top2 margin below which a Gaussian is left unlabeled
+
+ # --- display -----------------------------------------------------------
+ highlight_color: tuple[float, float, float] = (1.0, 0.95, 0.1)
+ blend: float = 0.75 # how strongly to tint selected Gaussians in overlays
+
+ # --- SAM3 --------------------------------------------------------------
+ # Path to the SAM3 checkpoint. Defaults to the GEOLANGSPLAT_SAM_CKPT env var
+ # (set it once), or pass --sam-ckpt / config.sam_ckpt explicitly.
+ sam_ckpt: str = field(default_factory=lambda: os.environ.get("GEOLANGSPLAT_SAM_CKPT", ""))
+ sam_res: int = 1008
+ sam_conf: float = 0.20
+ amp: str = "bf16" # autocast precision: "bf16" or "fp16"
+ batch_encode: bool = True
+ batch_size: int = 8
+ # Resident embedding-cache dtype. "auto" keeps SAM3's native output dtype; "amp"
+ # casts the cached per-view features to the autocast dtype (safe -- matches what
+ # scoring runs in, and halves the cache when the native output is fp32);
+ # "fp16"/"bf16" force a specific half precision. This is our training-free
+ # "embedding compression": the cache is the dominant resident VRAM consumer.
+ cache_dtype: str = "auto"
+
+ # --- runtime / viewer ports (used by `gls explore`) -------------------
+ device: str = "cuda"
+ viewer_port: int = 8080
+ web_port: int = 8090
+ vk_device_id: int = 0
+
+
+# Capture-type presets. Each maps field name -> preset value; apply_recipe fills
+# only fields still at their default. Keep these general; fine-tune per scene.
+RECIPES: "dict[str, dict]" = {
+ # Geometry-driven default. Derives the whole view plan (radii, elevations,
+ # azimuths, capture type) from the scene itself -- works across satellite,
+ # aerial and object scales without hand-tuned tiers.
+ "auto": {
+ "view_source": "render",
+ "auto_views": True,
+ },
+ # Near-nadir satellite / high-altitude captures. Synthesizes a top-down orbit
+ # at altitude (radius 200, grid 7 -> ~49 views/elev) -- the configuration
+ # validated on the JAX scenes. Works without source photos.
+ "satellite": {
+ "view_source": "render",
+ "tiers": (Tier(radius=200.0, elevations=(80.0, 65.0, 50.0), fov_deg=50.0, grid=7),),
+ "select": 0.35,
+ "margin": 0.10,
+ "distractors": ("road", "grass", "tree", "water"),
+ },
+ # Satellite with extra coverage for small objects: the altitude tier plus a
+ # closer, zoomed-in multi-azimuth tier (lower fov enlarges small structures).
+ "satellite_dense": {
+ "view_source": "render",
+ "tiers": (
+ Tier(radius=200.0, elevations=(80.0, 65.0, 50.0), fov_deg=50.0, grid=7),
+ Tier(radius=140.0, elevations=(70.0, 55.0), fov_deg=42.0, grid=6, num_azimuth=2),
+ ),
+ "select": 0.33,
+ "margin": 0.10,
+ "distractors": ("road", "grass", "tree", "water"),
+ },
+ # Oblique drone / aerial photogrammetry (e.g. SafetyPark): segment the scene's
+ # REAL source photos -- SAM3 segments them most cleanly -- and back-project.
+ # This is the validated "ground-truth-image" configuration; it requires --sfm.
+ "aerial": {
+ "view_source": "images",
+ "n_views": 90,
+ "select": 0.30,
+ "margin": 0.08,
+ "distractors": (
+ "building",
+ "ground",
+ "road",
+ "grass",
+ "tree",
+ "water",
+ "car",
+ "bus",
+ "truck",
+ "person",
+ ),
+ },
+}
+
+
+def list_recipes() -> list[str]:
+ """Return the available recipe names."""
+ return sorted(RECIPES)
+
+
+def apply_recipe(cfg: GeoLangSplatConfig, name: str | None) -> list[str]:
+ """Fill recipe presets into ``cfg`` in place, but only for fields the caller
+ left at their default. An explicitly set value always wins over the preset.
+
+ Returns the list of field names actually overwritten by the preset.
+ """
+ if not name:
+ return []
+ if name not in RECIPES:
+ hint = difflib.get_close_matches(name, list_recipes(), n=1)
+ suggest = f" (did you mean {hint[0]!r}?)" if hint else ""
+ raise GeoLangSplatError(f"unknown recipe {name!r}{suggest}; choices: {list_recipes()}")
+ default = GeoLangSplatConfig()
+ applied: list[str] = []
+ for k, v in RECIPES[name].items():
+ if not hasattr(cfg, k):
+ raise KeyError(f"recipe {name!r} sets unknown field {k!r}")
+ if getattr(cfg, k) == getattr(default, k):
+ setattr(cfg, k, v)
+ applied.append(k)
+ return applied
+
+
+def config_field_names() -> list[str]:
+ """All field names of :class:`GeoLangSplatConfig` (handy for CLIs/tests)."""
+ return [f.name for f in fields(GeoLangSplatConfig)]
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/engine.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/engine.py
new file mode 100644
index 0000000..afeab1e
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/engine.py
@@ -0,0 +1,612 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""The unified GeoLangSplat engine.
+
+It loads a splat, builds the SAM3 image-embedding cache + the per-Gaussian lift
+cache once, precomputes distractor scores, and then answers text queries cheaply
+(text encode + grounding decode + scatter). One engine serves aerial and
+satellite captures; only the recipe presets differ.
+"""
+from __future__ import annotations
+
+import pathlib
+import threading
+import time
+
+import torch
+
+from . import autoview as _autoview
+from . import lift as _lift
+from .profile import Stopwatch
+from .errors import GeoLangSplatError
+from .sam3 import Sam3Scorer
+from .select import select_query, smooth_scores, spatial_cleanup
+from .views import generate_views
+
+
+_PERF_FLAGS_SET = False
+
+
+def enable_perf_flags() -> None:
+ """Idempotent inference perf flags (CUDA only).
+
+ Turns on TF32 matmul/cuDNN and cuDNN autotuning. These speed up SAM3's conv
+ backbone and the fvdb rasterization with only minor numeric drift -- harmless
+ for thresholded masks / argmax labels. Set once per process.
+ """
+ global _PERF_FLAGS_SET
+ if _PERF_FLAGS_SET:
+ return
+ try:
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.backends.cudnn.allow_tf32 = True
+ torch.backends.cudnn.benchmark = True
+ except Exception:
+ pass
+ _PERF_FLAGS_SET = True
+
+
+def resolve_device(requested) -> torch.device:
+ """Resolve and validate the compute device.
+
+ GeoLangSplat needs CUDA: both fvdb rasterization (used to render views and the
+ per-Gaussian alpha weights) and SAM3 inference are GPU paths. If CUDA was
+ requested but is unavailable, fail early with a clear message instead of a deep
+ ``cudaGetDeviceCount`` traceback. An explicit non-cuda device is honored (so
+ CPU-only unit tests and tooling work), but real segmentation will require a GPU.
+ """
+ want = str(requested)
+ if want.startswith("cuda") and not torch.cuda.is_available():
+ raise GeoLangSplatError(
+ "GeoLangSplat needs a CUDA GPU (fvdb rasterization + SAM3), but no CUDA "
+ "device is available.\n"
+ " - On a GPU box: check drivers / `nvidia-smi` and CUDA_VISIBLE_DEVICES.\n"
+ " - Geometry-only `gls check` and the unit tests can run with `--device cpu`, "
+ "but text segmentation cannot."
+ )
+ dev = torch.device(want)
+ if dev.type == "cuda":
+ enable_perf_flags()
+ return dev
+
+
+def _load_model(model_or_path, device) -> tuple[object, dict]:
+ """Load a splat and its metadata. Returns ``(model, metadata)``.
+
+ Prefers ``fvdb_reality_capture``'s loader so ``.ply`` AND training checkpoints
+ (``.pt``/``.pth``) work and metadata round-trips; falls back to a plain
+ ``GaussianSplat3d.from_ply`` when frc is not importable.
+ """
+ if not (isinstance(model_or_path, (str, bytes)) or hasattr(model_or_path, "__fspath__")):
+ return model_or_path, {} # already a GaussianSplat3d
+
+ path = pathlib.Path(str(model_or_path))
+ if not path.exists():
+ raise GeoLangSplatError(f"no such model file: {path}")
+
+ try: # reuse frc's IO so `frgs segment` behaves identically (ply + checkpoints)
+ from fvdb_reality_capture.cli.frgs._common import load_splats_from_file
+
+ return load_splats_from_file(path, device)
+ except GeoLangSplatError:
+ raise
+ except Exception:
+ from fvdb import GaussianSplat3d
+
+ gs = GaussianSplat3d.from_ply(str(path), device=device)
+ if isinstance(gs, tuple):
+ model = gs[0]
+ meta = gs[1] if len(gs) > 1 and isinstance(gs[1], dict) else {}
+ return model, meta
+ return gs, {}
+
+
+class GeoLangSplatEngine:
+ """Build-once, query-many open-vocabulary segmentation over one splat."""
+
+ def __init__(self, model_or_path, cfg, scorer=None, build: bool = True):
+ self.cfg = cfg
+ self.device = resolve_device(cfg.device)
+ self.sw = Stopwatch(self.device)
+ is_path = isinstance(model_or_path, (str, bytes)) or hasattr(model_or_path, "__fspath__")
+ self.source_path = pathlib.Path(str(model_or_path)) if is_path else None
+ with self.sw.span("load_splat"):
+ self.model, self.metadata = _load_model(model_or_path, self.device)
+ self.means = self.model.means.detach()
+ self.N = self.means.shape[0]
+ self.lock = threading.Lock()
+ # Scorer is created lazily (only when SAM3 is actually needed), so the
+ # geometry-only readiness check does not require SAM3 weights.
+ self.scorer = scorer
+ self.views: list = []
+ self.cache = None
+ self.states = None
+ self.seen = None
+ self.denom = None
+ self.dist_names: list[str] = []
+ self.dist_scores: torch.Tensor | None = None
+ self.auto_report: dict | None = None
+ if build:
+ self.build()
+
+ # -- one-time build -----------------------------------------------------
+
+ def build(self) -> None:
+ """Full build: scene geometry/lift cache + SAM3 embeddings + distractors.
+
+ This is the one-time cost (load splat, load SAM3, render + encode views).
+ Every query afterwards reuses this cache and is fast, so to benefit you must
+ keep the engine alive (a session, the viewer, or reusing it via the API).
+ """
+ if getattr(self.cfg, "low_vram", False):
+ return self.build_streaming()
+ print("[build] one-time setup (load + views + SAM3 embeddings); queries after this reuse the cache", flush=True)
+ t0 = time.time()
+ if self.device.type == "cuda":
+ torch.cuda.reset_peak_memory_stats(self.device)
+ self.build_geometry()
+ self.build_scorer()
+ print(
+ f"[ready] cached {len(self.views)} views, {self.N:,} gaussians in {time.time() - t0:.1f}s",
+ flush=True,
+ )
+ print(self.sw.report("build"), flush=True)
+ print(self.vram_report(), flush=True)
+
+ def build_streaming(self) -> None:
+ """Low-VRAM build: render views + load SAM3, but DON'T encode/cache embeddings.
+
+ The per-view embeddings (the dominant resident VRAM) are produced and evicted
+ inside each query (:func:`lift.stream_scores`), so peak VRAM is bounded to
+ ~one view. There is no warm cache: each query re-encodes. This is the one-shot
+ backend; use ``gls serve`` for the fast all-views cache.
+ """
+ print("[build] low-VRAM mode", flush=True)
+ t0 = time.time()
+ if self.device.type == "cuda":
+ torch.cuda.reset_peak_memory_stats(self.device)
+ self._resolve_auto_views()
+ with self.sw.span("generate_views"):
+ self.views = generate_views(self.model, self.cfg, self.device, self.metadata)
+ if self.scorer is None:
+ with self.sw.span("sam3_load"):
+ self.scorer = Sam3Scorer(
+ self.cfg.sam_ckpt,
+ sam_res=self.cfg.sam_res,
+ sam_conf=self.cfg.sam_conf,
+ amp=self.cfg.amp,
+ device=self.cfg.device,
+ dual_head=self.cfg.dual_head,
+ sem_weight=self.cfg.sem_weight,
+ sem_mode=self.cfg.sem_mode,
+ )
+ self.dist_names = list(self.cfg.distractors)
+ self.dist_scores = None
+ print(f"[ready] {self.N:,} gaussians, {len(self.views)} views in {time.time() - t0:.1f}s", flush=True)
+ print(self.sw.report("build"), flush=True)
+ print(self.vram_report(), flush=True)
+
+ def vram_report(self) -> str:
+ """One-line peak-VRAM summary (or a note when running on CPU)."""
+ if self.device.type != "cuda":
+ return "[vram] cpu device - no GPU memory in use"
+ peak = torch.cuda.max_memory_allocated(self.device) / 1e9
+ resv = torch.cuda.max_memory_reserved(self.device) / 1e9
+ return f"[vram] peak {peak:.2f} GB ({resv:.2f} GB reserved)"
+
+ def _resolve_auto_views(self) -> None:
+ """Derive scene-scaled synthesized view tiers from geometry (auto recipe), once.
+
+ Idempotent: safe to call from build_streaming and again from build_geometry.
+ No-op for the ground-truth-image path or an explicit recipe.
+ """
+ cfg = self.cfg
+ if not getattr(cfg, "auto_views", False) or self.auto_report is not None:
+ return
+ if cfg.view_source == "images":
+ return
+ self.auto_report = _autoview.apply_auto_view_config(cfg, self.means)
+
+ def build_geometry(self) -> None:
+ """Generate views and the per-Gaussian lift cache. No SAM3 weights needed.
+
+ This is enough for the scene-readiness check (:meth:`assess`).
+ """
+ cfg = self.cfg
+ self._resolve_auto_views()
+ with self.sw.span("generate_views"):
+ self.views = generate_views(self.model, cfg, self.device, self.metadata)
+ if cfg.lift == "alpha":
+ with self.sw.span("alpha_lift"):
+ self.cache = _lift.build_alpha_cache(self.model, self.views, cfg, self.device)
+ self.denom = self.cache.denom
+ self.seen = self.cache.seen
+ elif cfg.lift == "band":
+ with self.sw.span("band_lift"):
+ self.cache = _lift.build_band_cache(self.model, self.views, cfg, self.device)
+ self.seen = self.cache.seen
+ else:
+ raise ValueError(f"unknown lift {cfg.lift!r} (expected 'alpha' or 'band')")
+
+ def build_scorer(self) -> None:
+ """Create SAM3 (if needed), cache per-view embeddings, precompute distractors."""
+ cfg = self.cfg
+ if not self.views:
+ self.build_geometry()
+ if self.scorer is None:
+ with self.sw.span("sam3_load"):
+ self.scorer = Sam3Scorer(
+ cfg.sam_ckpt,
+ sam_res=cfg.sam_res,
+ sam_conf=cfg.sam_conf,
+ amp=cfg.amp,
+ device=cfg.device,
+ dual_head=cfg.dual_head,
+ sem_weight=cfg.sem_weight,
+ sem_mode=cfg.sem_mode,
+ )
+ pils = [v.pil for v in self.views]
+ sizes = {(v.height, v.width) for v in self.views}
+ h, w = self.views[0].height, self.views[0].width
+ batch = cfg.batch_encode and len(sizes) == 1
+ te = time.time()
+ with self.sw.span("sam3_encode"):
+ self.states = self.scorer.encode(pils, h, w, batch_encode=batch, batch_size=cfg.batch_size, progress=True)
+ self._compress_embeddings()
+ print(f"[engine] {len(self.states)} embeddings cached in {time.time() - te:.1f}s", flush=True)
+ self._build_distractors()
+
+ def _compress_embeddings(self) -> None:
+ """Optionally shrink the resident per-view embedding cache (the dominant VRAM
+ consumer) by casting its float tensors to a half dtype. Off by default."""
+ want = str(getattr(self.cfg, "cache_dtype", "auto")).lower()
+ if want in ("", "auto", "native") or not self.states:
+ return
+ if want == "amp":
+ target = getattr(self.scorer, "amp_dtype", torch.bfloat16)
+ elif want == "fp16":
+ target = torch.float16
+ elif want == "bf16":
+ target = torch.bfloat16
+ else:
+ raise GeoLangSplatError(f"unknown cache_dtype {want!r}; use auto|amp|fp16|bf16")
+
+ def _cast(obj):
+ if torch.is_tensor(obj):
+ return obj.to(target) if obj.is_floating_point() and obj.dtype != target else obj
+ if isinstance(obj, dict):
+ return {k: _cast(v) for k, v in obj.items()}
+ if isinstance(obj, (list, tuple)):
+ return type(obj)(_cast(v) for v in obj)
+ return obj
+
+ self.states = _cast(self.states)
+ print(f"[engine] embedding cache cast to {str(target).replace('torch.', '')}", flush=True)
+
+ # -- scene readiness ----------------------------------------------------
+
+ def assess(self) -> dict:
+ """Geometry-only verdict on whether the scene is segmentable.
+
+ Reports, without SAM3:
+ * ``coverage`` - fraction of Gaussians any view sees as a top-k
+ contributor. This is naturally bounded below 1.0 on dense splats
+ (interior/occluded Gaussians never surface), so it is informational,
+ not the verdict driver.
+ * ``mean_views_per_observed_gaussian`` - how many distinct views see a
+ typical observed Gaussian (multi-view redundancy).
+ * ``well_observed_frac`` - fraction of Gaussians with enough accumulated
+ render weight (``>= min_weight``) to be scored reliably. This is the
+ quantity that actually gates segmentation quality, so the verdict is
+ driven by it.
+ """
+ if self.seen is None:
+ self.build_geometry()
+ seen = self.seen
+ observed = seen > 0
+ cov = float(observed.float().mean())
+ mean_views = float(seen[observed].float().mean()) if bool(observed.any()) else 0.0
+ report = {
+ "gaussians": int(self.N),
+ "views": int(len(self.views)),
+ "coverage": cov,
+ "mean_views_per_observed_gaussian": mean_views,
+ }
+ if self.auto_report is not None:
+ report["capture"] = self.auto_report.get("capture")
+ report["auto_views_planned"] = self.auto_report.get("n_views_planned")
+ report["auto_rationale"] = self.auto_report.get("rationale")
+ well = cov
+ if self.denom is not None:
+ well = float((self.denom >= self.cfg.min_weight).float().mean())
+ report["well_observed_frac"] = well
+
+ # Verdict is driven by the well-observed fraction. Coverage and multi-view
+ # redundancy (mean_views) are reported but NOT gated on: the tiled render
+ # recipe sees each Gaussian from only ~1-2 views by design, and dense
+ # splats cap coverage well below 1.0 - yet both still segment fine. These
+ # thresholds are anchored on known-good scenes; for the strongest signal,
+ # follow up with an actual query (a few well_observed_frac of ~0.4 is healthy).
+ if well >= 0.35:
+ verdict, note = "good", "enough well-observed gaussians; segmentation should work well"
+ elif well >= 0.15:
+ verdict, note = (
+ "fair",
+ "usable; thin in places - prefer real photos (--view-source images) or a denser recipe",
+ )
+ else:
+ verdict, note = (
+ "poor",
+ "few well-observed gaussians; use real photos, a denser recipe, or improve the reconstruction",
+ )
+ report["verdict"] = verdict
+ report["note"] = note
+ return report
+
+ def _build_distractors(self) -> None:
+ # Competition is opt-in; only precompute the distractor scores if it is
+ # enabled, so a plain "show me X" query does not pay for them.
+ self.dist_names = list(self.cfg.distractors)
+ self.dist_scores = None
+ if self.cfg.compete:
+ self._ensure_distractors()
+
+ def _ensure_distractors(self) -> None:
+ """Compute distractor scores on demand (cached). No-op if already done."""
+ if self.dist_scores is not None or not self.dist_names:
+ return
+ t0 = time.time()
+ cols = [self._score(n) for n in self.dist_names]
+ self.dist_scores = torch.stack(cols, dim=1) # [N, D]
+ print(f"[engine] distractors {self.dist_names} in {time.time() - t0:.1f}s", flush=True)
+
+ # -- scoring ------------------------------------------------------------
+
+ def _score(self, prompt: str) -> torch.Tensor:
+ """Per-Gaussian scalar score for ``prompt`` (lift-agnostic)."""
+ if self.cfg.lift == "alpha":
+ return _lift.alpha_qscore(self.scorer, self.states, self.cache, prompt, self.cfg)
+ smax, _hits = _lift.aggregate_band(self.scorer, self.states, self.cache, prompt, self.cfg)
+ return smax
+
+ def score(self, prompt: str) -> torch.Tensor:
+ """Public: per-Gaussian score for a single prompt."""
+ return self._score(prompt)
+
+ # -- query --------------------------------------------------------------
+
+ def _subset_indices(self, k: int) -> list[int]:
+ """``k`` view indices spread evenly across the (structured) view list, so the
+ subset spans angles rather than clustering. Used by the fast query path."""
+ n = len(self.views)
+ k = min(int(k), n)
+ if k <= 0 or k >= n:
+ return list(range(n))
+ return sorted(set(torch.linspace(0, n - 1, k).round().long().tolist()))
+
+ @torch.no_grad()
+ def query(self, prompt: str, *, select=None, margin=None, compete=None, fast_views=None):
+ """Return ``(scores [N], selected [N] bool, dt_seconds)`` for a prompt."""
+ cfg = self.cfg
+ if getattr(cfg, "low_vram", False):
+ return self._query_streaming(prompt, select=select, margin=margin, compete=compete)
+ use_compete = cfg.compete if compete is None else compete
+ # Fast path: decode only a spread subset of views (no competition support).
+ fv = cfg.fast_views if fast_views is None else fast_views
+ if cfg.lift == "alpha" and fv and not use_compete:
+ with self.lock:
+ t0 = time.time()
+ idx = self._subset_indices(fv)
+ qscore, denom, seen = _lift.aggregate_alpha_views(
+ self.scorer, self.states, self.cache, prompt, cfg, idx
+ )
+ if getattr(cfg, "smooth", False):
+ qscore = smooth_scores(self.means, qscore, denom, cfg)
+ sel = select_query(
+ qscore,
+ seen,
+ cfg,
+ query=prompt,
+ select=select,
+ margin=margin,
+ compete=False,
+ denom=denom,
+ )
+ sel = spatial_cleanup(self.means, sel, cfg)
+ return qscore, sel, time.time() - t0
+ if use_compete:
+ self._ensure_distractors()
+ with self.lock:
+ t0 = time.time()
+ if cfg.lift == "alpha":
+ if getattr(cfg, "consensus", False):
+ qscore, support = _lift.alpha_qscore_support(self.scorer, self.states, self.cache, prompt, cfg)
+ else:
+ qscore = _lift.alpha_qscore(self.scorer, self.states, self.cache, prompt, cfg)
+ support = None
+ if getattr(cfg, "smooth", False):
+ qscore = smooth_scores(self.means, qscore, self.denom, cfg)
+ sel = select_query(
+ qscore,
+ self.seen,
+ cfg,
+ query=prompt,
+ select=select,
+ margin=margin,
+ compete=compete,
+ denom=self.denom,
+ dist_scores=self.dist_scores,
+ dist_names=self.dist_names,
+ support=support,
+ )
+ else:
+ qscore, hits = _lift.aggregate_band(self.scorer, self.states, self.cache, prompt, cfg)
+ sel_v = cfg.select if select is None else select
+ sel = (qscore >= sel_v) & ((hits >= cfg.min_views) | (qscore >= cfg.strong_select)) & (self.seen > 0)
+ use_compete = cfg.compete if compete is None else compete
+ if use_compete and self.dist_scores is not None and self.dist_names:
+ from .select import competitor_idx
+
+ keep = competitor_idx(prompt, self.dist_names)
+ if keep:
+ mrg = cfg.margin if margin is None else margin
+ dmax = self.dist_scores[:, keep].max(dim=1).values
+ sel = sel & (qscore >= dmax + mrg)
+ sel = spatial_cleanup(self.means, sel, cfg)
+ return qscore, sel, time.time() - t0
+
+ def _stream_early_stop(self) -> bool:
+ """Whether the streaming lift may early-stop for this scene.
+
+ Early-stop assumes a *compact* subject: a few angularly-diverse views see the
+ whole thing, so once they agree the rest are redundant. That holds for an
+ object/dome capture but NOT for aerial/satellite, where the queried class
+ (buildings, roads, trees) is spread across the whole footprint -- stopping
+ early there truncates recall (measured: IoU collapses to ~0.05-0.27). So
+ "auto" enables it only for object/globe captures; aerial/sat stream every
+ view (still VRAM-bounded, full recall).
+ """
+ mode = str(getattr(self.cfg, "stream_early_stop", "auto")).lower()
+ if mode == "on":
+ return True
+ if mode == "off":
+ return False
+ if getattr(self.cfg, "view_source", "") == "globe":
+ return True
+ return bool(self.auto_report) and self.auto_report.get("capture") == "object"
+
+ @torch.no_grad()
+ def _query_streaming(self, prompt: str, *, select=None, margin=None, compete=None):
+ """Single-prompt query via the low-VRAM streaming lift (see build_streaming).
+
+ Concept competition is supported here too: when enabled, the queried prompt
+ and the distractor set are scored together in the SAME streaming pass -- the
+ expensive per-view SAM3 encode is shared across all prompts, only the cheap
+ text-vs-features scoring multiplies -- then the query must beat the best
+ distractor by ``margin``. Selection is then identical to the warm path.
+ Competition scores every view (no early-stop), which is also what spread-out
+ aerial/satellite classes want. Sets denom/seen from the stream so
+ selection/labelling behave identically to the cached path.
+ """
+ cfg = self.cfg
+ use_compete = cfg.compete if compete is None else compete
+ dist_names = list(cfg.distractors) if use_compete else []
+ prompts = [prompt] + dist_names
+ # Competition needs every view; only a lone prompt may early-stop (and
+ # stream_scores only early-stops when it is scoring a single prompt).
+ early = self._stream_early_stop() and not use_compete
+ with self.lock:
+ t0 = time.time()
+ scores, denom, seen, stats = _lift.stream_scores(
+ self.model,
+ self.scorer,
+ self.views,
+ cfg,
+ prompts,
+ self.device,
+ want_peak=(cfg.peak > 0),
+ early_stop=early,
+ )
+ self.denom, self.seen = denom, seen
+ qscore = scores[0]
+ if getattr(cfg, "smooth", False):
+ qscore = smooth_scores(self.means, qscore, denom, cfg)
+ dist_scores = scores[1:].t().contiguous() if (use_compete and len(prompts) > 1) else None
+ sel = select_query(
+ qscore,
+ seen,
+ cfg,
+ query=prompt,
+ select=select,
+ margin=margin,
+ compete=use_compete,
+ denom=denom,
+ dist_scores=dist_scores,
+ dist_names=dist_names,
+ )
+ sel = spatial_cleanup(self.means, sel, cfg)
+ if stats["early_stopped"]:
+ note = " (early-stopped)"
+ elif use_compete:
+ note = f" (+{len(dist_names)} distractors)"
+ else:
+ note = ""
+ print(f"[stream] {stats['views_used']}/{stats['views_total']} views{note}", flush=True)
+ return qscore, sel, time.time() - t0
+
+ # -- fixed-vocabulary bake ---------------------------------------------
+
+ @torch.no_grad()
+ def bake_vocab(self, vocab) -> torch.Tensor:
+ """Per-Gaussian scores for a fixed vocabulary: returns ``[N, C]``.
+
+ Only supported for the alpha lift (continuous scores comparable across
+ words). These become instant lookups for multi-class labelling.
+ """
+ if self.cfg.lift != "alpha":
+ raise NotImplementedError("bake_vocab requires the alpha lift")
+ t0 = time.time()
+ if getattr(self.cfg, "low_vram", False):
+ # One bounded streaming pass scores the whole vocab per view (no early-stop
+ # for multi-class -- every class needs full coverage), then evicts.
+ scores, denom, seen, _stats = _lift.stream_scores(
+ self.model,
+ self.scorer,
+ self.views,
+ self.cfg,
+ list(vocab),
+ self.device,
+ want_peak=(self.cfg.peak > 0),
+ early_stop=False,
+ )
+ self.denom, self.seen = denom, seen
+ out = scores.transpose(0, 1).contiguous() # [C, N] -> [N, C]
+ else:
+ cols = [_lift.alpha_qscore(self.scorer, self.states, self.cache, w, self.cfg) for w in vocab]
+ out = torch.stack(cols, dim=1)
+ print(f"[engine] baked {len(vocab)} words in {time.time() - t0:.1f}s", flush=True)
+ return out
+
+ # -- segment catalog ----------------------------------------------------
+
+ @torch.no_grad()
+ def catalog(self, vocab=None, *, select=None, iou=None, link_frac=None, min_size=None):
+ """Build a :class:`~geolangsplat.catalog.SegmentCatalog` over ``vocab``.
+
+ Scores the whole vocabulary in one pass (:meth:`bake_vocab`), then clusters
+ each prompt's selection into spatial objects and merges duplicates across
+ prompts. ``vocab`` defaults to
+ :data:`~geolangsplat.catalog.DEFAULT_CATALOG_VOCAB`.
+ """
+ from .catalog import DEFAULT_CATALOG_VOCAB, catalog_from_scores
+
+ vocab = list(vocab) if vocab else list(DEFAULT_CATALOG_VOCAB)
+ scores = self.bake_vocab(vocab)
+ cat = catalog_from_scores(
+ self.model,
+ vocab,
+ self.cfg,
+ scores,
+ seen=self.seen,
+ denom=self.denom,
+ select=select,
+ iou=iou,
+ link_frac=link_frac,
+ min_size=min_size,
+ metadata=getattr(self, "metadata", {}) or {},
+ )
+ cat._engine = self # keep warm so SegmentCatalog.browse() can re-query in place
+ return cat
+
+
+def load_or_build_engine(model_or_path, cfg, *, scorer=None) -> GeoLangSplatEngine:
+ """Build an engine for ``model_or_path`` under ``cfg`` and return it ready to query.
+
+ Dispatches on ``cfg.low_vram``: the streaming one-shot backend (default) or the
+ full all-views build. Keep the returned engine alive (a session, the viewer, or
+ ``gls serve``) to amortize the one-time build across queries.
+ """
+ engine = GeoLangSplatEngine(model_or_path, cfg, scorer=scorer, build=False)
+ engine.build()
+ return engine
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/errors.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/errors.py
new file mode 100644
index 0000000..8697efa
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/errors.py
@@ -0,0 +1,14 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Typed errors so the CLI can print a clean, actionable message (no traceback)
+for the failures users actually hit: no GPU, missing SAM3 weights, bad inputs."""
+from __future__ import annotations
+
+
+class GeoLangSplatError(RuntimeError):
+ """A user-actionable error (bad device, missing weights, bad input).
+
+ The ``gls`` CLI catches this and prints only the message; everything else
+ propagates as a normal traceback for debugging.
+ """
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/instances.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/instances.py
new file mode 100644
index 0000000..61f22e4
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/instances.py
@@ -0,0 +1,404 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Split a semantic selection into individual instances and pick one.
+
+A text query ("house") selects every Gaussian of that class -- often many
+separate buildings. This module turns that one mask into a list of distinct
+instances via **3D connected components** (voxelize the selection, union
+spatially-adjacent voxels), ranks them largest-first, and exposes a notebook-
+friendly result:
+
+ res = engine.query_instances("house")
+ res.show() # inline top-down PNG with 0,1,2... drawn on each house
+ res.extract(0, "h.ply") # write just the largest house
+
+The connected-components core (:func:`connected_components`) is pure tensor/numpy
+and runs on CPU; only the rendering helpers touch the GPU/fvdb.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import torch
+
+
+# --- connected components (pure, CPU-testable) -----------------------------
+
+# 13 "forward" neighbor offsets for 26-connectivity. Union is symmetric, so
+# visiting half the neighborhood links every adjacent voxel pair exactly once.
+_OFFSETS_26 = (
+ (1, 0, 0),
+ (0, 1, 0),
+ (0, 0, 1),
+ (1, 1, 0),
+ (1, -1, 0),
+ (1, 0, 1),
+ (1, 0, -1),
+ (0, 1, 1),
+ (0, 1, -1),
+ (1, 1, 1),
+ (1, 1, -1),
+ (1, -1, 1),
+ (1, -1, -1),
+)
+
+
+def _union_find(n: int, edges) -> list[int]:
+ """Connected-component roots for ``n`` nodes given an iterable of (a, b) edges."""
+ parent = list(range(n))
+
+ def find(a: int) -> int:
+ while parent[a] != a:
+ parent[a] = parent[parent[a]]
+ a = parent[a]
+ return a
+
+ for a, b in edges:
+ ra, rb = find(a), find(b)
+ if ra != rb:
+ parent[ra] = rb
+ return [find(i) for i in range(n)]
+
+
+def connected_components(
+ means: torch.Tensor,
+ sel: torch.Tensor,
+ cfg,
+ *,
+ link_frac: float | None = None,
+ min_size: int | None = None,
+ span: float | None = None,
+) -> tuple[torch.Tensor, list[dict]]:
+ """Split selected Gaussians into spatial instances via voxel connected components.
+
+ Voxelize the selected points at ``link_frac * scene_span`` and union voxels
+ that are 26-adjacent, so a contiguous blob of Gaussians (one building) becomes
+ one component while spatially separated blobs split apart. Components smaller
+ than ``min_size`` Gaussians are dropped (speckle).
+
+ ``span`` (the scene extent the voxel size is a fraction of) defaults to the
+ full min/max range; pass a robust value (e.g. a 1-99% quantile span) when the
+ scene has far-away floaters that would otherwise inflate the voxel and merge
+ separate objects.
+
+ Returns ``(comp_ids, infos)``:
+
+ * ``comp_ids`` -- ``[N]`` long, in input order. ``-1`` for unselected or
+ dropped Gaussians, else the instance index (``0`` = largest).
+ * ``infos`` -- per-instance dicts (largest first) with ``idx``, ``size``,
+ ``centroid`` (3,), ``bbox_min`` (3,), ``bbox_max`` (3,).
+ """
+ link_frac = cfg.inst_link_frac if link_frac is None else link_frac
+ min_size = cfg.inst_min_size if min_size is None else min_size
+
+ N = sel.shape[0]
+ comp_ids = torch.full((N,), -1, dtype=torch.long)
+ sel_idx = sel.nonzero(as_tuple=True)[0]
+ if sel_idx.numel() == 0:
+ return comp_ids, []
+
+ pts_t = means[sel_idx].detach().float().cpu()
+ pts = pts_t.numpy()
+ if span is None:
+ mn = means.detach().float().min(dim=0).values
+ mx = means.detach().float().max(dim=0).values
+ span = float((mx - mn).max().cpu())
+ vox = max(link_frac * span, 1e-9)
+
+ keys = np.floor(pts / vox).astype(np.int64) # [M, 3]
+ uniq, inv = np.unique(keys, axis=0, return_inverse=True) # uniq [V, 3], inv [M]
+ inv = inv.reshape(-1)
+ V = uniq.shape[0]
+
+ lut = {(int(r[0]), int(r[1]), int(r[2])): i for i, r in enumerate(uniq)}
+ edges = []
+ for i in range(V):
+ x, y, z = int(uniq[i, 0]), int(uniq[i, 1]), int(uniq[i, 2])
+ for dx, dy, dz in _OFFSETS_26:
+ j = lut.get((x + dx, y + dy, z + dz))
+ if j is not None:
+ edges.append((i, j))
+ roots = np.asarray(_union_find(V, edges), dtype=np.int64)
+
+ point_root = roots[inv] # [M] component root per selected point
+ uniq_roots, sizes = np.unique(point_root, return_counts=True)
+ order = np.argsort(-sizes) # largest first
+
+ infos: list[dict] = []
+ new_id = 0
+ for r in uniq_roots[order]:
+ size = int((point_root == r).sum())
+ if size < min_size:
+ continue
+ member = sel_idx[torch.from_numpy(point_root == r)]
+ cpts = means[member].detach().float()
+ infos.append(
+ {
+ "idx": new_id,
+ "size": size,
+ "centroid": cpts.mean(dim=0).cpu().numpy(),
+ "bbox_min": cpts.min(dim=0).values.cpu().numpy(),
+ "bbox_max": cpts.max(dim=0).values.cpu().numpy(),
+ }
+ )
+ comp_ids[member] = new_id
+ new_id += 1
+ return comp_ids, infos
+
+
+# --- notebook-facing result ------------------------------------------------
+
+
+@dataclass
+class Instance:
+ """One spatial instance from a semantic selection (largest first = idx 0)."""
+
+ idx: int
+ size: int
+ centroid: np.ndarray
+ bbox_min: np.ndarray
+ bbox_max: np.ndarray
+
+
+@dataclass
+class InstanceResult:
+ """Instances of one prompt, ready to inspect/extract from a notebook.
+
+ Hold a reference to the splat ``model`` so :meth:`show` can render a numbered
+ top-down and :meth:`extract` can write a single instance's ``.ply``.
+ """
+
+ prompt: str
+ instances: list[Instance]
+ comp_ids: torch.Tensor # [N] long, -1 = none, else instance idx
+ config: object
+ _model: object = None
+ metadata: dict = field(default_factory=dict)
+
+ # -- container sugar ----------------------------------------------------
+
+ def __len__(self) -> int:
+ return len(self.instances)
+
+ def __getitem__(self, k: int) -> Instance:
+ return self.instances[k]
+
+ def __repr__(self) -> str:
+ head = f"InstanceResult({self.prompt!r}: {len(self.instances)} instances)"
+ rows = [
+ f" #{ins.idx}: {ins.size:,} gaussians centroid=("
+ f"{ins.centroid[0]:.2f}, {ins.centroid[1]:.2f}, {ins.centroid[2]:.2f})"
+ for ins in self.instances[:20]
+ ]
+ if len(self.instances) > 20:
+ rows.append(f" ... (+{len(self.instances) - 20} more)")
+ return "\n".join([head, *rows])
+
+ # -- masks / extraction -------------------------------------------------
+
+ def mask(self, k: int) -> torch.Tensor:
+ """Boolean ``[N]`` mask selecting only instance ``k``."""
+ self._check(k)
+ return self.comp_ids == k
+
+ @property
+ def all_mask(self) -> torch.Tensor:
+ """Boolean ``[N]`` mask of every kept instance (the whole class)."""
+ return self.comp_ids >= 0
+
+ def extract(self, k: int, path) -> int:
+ """Write only instance ``k`` to ``path`` as a ``.ply``. Returns its count."""
+ from . import outputs as _out
+
+ self._check(k)
+ return _out.write_ply_segmented(self._model, self.mask(k), path, metadata=self.metadata)
+
+ def _check(self, k: int) -> None:
+ if not (0 <= k < len(self.instances)):
+ raise IndexError(
+ f"instance {k} out of range; this query found {len(self.instances)} "
+ f"(valid 0..{len(self.instances) - 1})"
+ )
+
+ # -- visualization ------------------------------------------------------
+
+ def show(self, *, size: int = 900, device=None, path=None):
+ """Inline top-down render with each instance's index drawn on it.
+
+ Returns a ``PIL.Image`` (Jupyter displays it inline). Pass ``path`` to also
+ save the PNG. The number drawn on each instance is exactly the ``k`` you
+ pass to :meth:`extract`.
+ """
+ pil, uv = _render_topdown(self._model, self.config, self.instances, size=size, device=device)
+ labels = [(int(ins.idx), uv[i]) for i, ins in enumerate(self.instances) if uv[i] is not None]
+ out = _annotate(pil, labels)
+ if path is not None:
+ out.save(path)
+ print(f"[instances] wrote {path}", flush=True)
+ return out
+
+ def show_one(self, k: int, *, size: int = 900, device=None, path=None):
+ """Top-down render with only instance ``k`` highlighted (rest dimmed)."""
+ self._check(k)
+ pil, uv = _render_topdown(
+ self._model, self.config, self.instances, size=size, device=device, highlight=self.mask(k)
+ )
+ out = _annotate(pil, [(k, uv[k])] if uv[k] is not None else [])
+ if path is not None:
+ out.save(path)
+ print(f"[instances] wrote {path}", flush=True)
+ return out
+
+
+# --- rendering helpers (GPU / fvdb) ----------------------------------------
+
+
+def _topdown_camera(means: torch.Tensor, cfg, size: int, fov: float = 60.0):
+ """A nadir camera framing the scene footprint. Returns ``(w2c_np, K_np)``."""
+ from .autoview import resolve_up
+ from .cameras import _perp_basis, intrinsics, orbit_cameras, up_vector
+
+ m = means.detach().float().cpu().numpy()
+ center = 0.5 * (m.min(axis=0) + m.max(axis=0))
+ up = resolve_up(getattr(cfg, "up", "auto"), means)
+ up_hat = up_vector(up)
+ e1, e2 = _perp_basis(up_hat)
+ rel = m - center
+ a, b, c = rel @ e1, rel @ e2, rel @ up_hat
+ half = 0.5 * max(float(a.max() - a.min()), float(b.max() - b.min())) * 1.1
+ half = max(half, 1e-6)
+ height = half / np.tan(np.radians(fov) / 2.0) + max(0.0, float(c.max()))
+ w2c_np = orbit_cameras(center, 90.0, height, num_azimuth=1, up=up_hat)[0][0]
+ K_np = intrinsics(fov, size, size)
+ return w2c_np, K_np
+
+
+@torch.no_grad()
+def _render_topdown(model, cfg, instances, *, size: int, device=None, highlight=None, fov: float = 60.0):
+ """Render the splat top-down; project instance centroids to pixels.
+
+ Returns ``(pil, uv)`` where ``uv[i]`` is the ``(u, v)`` pixel of instance ``i``'s
+ centroid (or ``None`` if it falls outside / behind the camera). When
+ ``highlight`` (a bool mask) is given, those Gaussians are tinted and the rest
+ dimmed before rendering.
+ """
+ from PIL import Image
+
+ from .cameras import project
+
+ means = model.means.detach()
+ dev = means.device if device is None else torch.device(device)
+ w2c_np, K_np = _topdown_camera(means, cfg, size, fov=fov)
+ w2c = torch.from_numpy(w2c_np).float().to(dev)
+ K = torch.from_numpy(K_np).float().to(dev)
+
+ disp = _highlight_model(model, highlight) if highlight is not None else model
+ img, _a = disp.render_images_and_depths(
+ world_to_camera_matrices=w2c.unsqueeze(0),
+ projection_matrices=K.unsqueeze(0),
+ image_width=size,
+ image_height=size,
+ near=0.01,
+ far=1e12,
+ )
+ rgb = (img[0, ..., :3].clamp(0, 1).cpu().numpy() * 255).astype(np.uint8)
+ pil = Image.fromarray(rgb)
+
+ uv: list = []
+ if instances:
+ cents = torch.from_numpy(np.stack([ins.centroid for ins in instances])).float().to(dev)
+ u, v, z = project(cents, w2c, float(K[0, 0]), float(K[0, 2]), float(K[1, 2]))
+ for i in range(len(instances)):
+ zi = float(z[i])
+ ui, vi = float(u[i]), float(v[i])
+ if zi > 0 and 0 <= ui < size and 0 <= vi < size:
+ uv.append((ui, vi))
+ else:
+ uv.append(None)
+ return pil, uv
+
+
+def _highlight_model(model, mask: torch.Tensor):
+ """A copy of the splat with ``mask`` Gaussians tinted bright and the rest dimmed."""
+ from fvdb import GaussianSplat3d
+
+ from .outputs import _recolor_sh0
+
+ sh0 = model.sh0.detach().clone()
+ shN = model.shN.detach().clone()
+ other = ~mask
+ if bool(other.any()): # dim the rest toward grey so the pick pops
+ sh0 = _recolor_sh0(sh0, other, (0.18, 0.18, 0.20), 0.7)
+ shN[other] = shN[other] * 0.3
+ if bool(mask.any()):
+ sh0 = _recolor_sh0(sh0, mask, (1.0, 0.95, 0.1), 0.85)
+ shN[mask] = shN[mask] * 0.15
+ return GaussianSplat3d.from_tensors(
+ means=model.means.detach(),
+ quats=model.quats.detach(),
+ log_scales=model.log_scales.detach(),
+ logit_opacities=model.logit_opacities.detach(),
+ sh0=sh0,
+ shN=shN,
+ )
+
+
+def _annotate(pil, labels):
+ """Draw each ``(index, (u, v))`` label as a numbered marker on a copy of ``pil``."""
+ from PIL import ImageDraw, ImageFont
+
+ img = pil.convert("RGB").copy()
+ draw = ImageDraw.Draw(img)
+ r = max(10, img.size[0] // 60)
+ try:
+ font = ImageFont.truetype("DejaVuSans-Bold.ttf", int(r * 1.6))
+ except Exception:
+ font = ImageFont.load_default()
+ for idx, uv in labels:
+ if uv is None:
+ continue
+ u, v = uv
+ draw.ellipse([u - r, v - r, u + r, v + r], fill=(255, 235, 40), outline=(20, 20, 20), width=2)
+ txt = str(idx)
+ try: # center the glyph in the marker (textbbox carries the baseline offset)
+ x0, y0, x1, y1 = draw.textbbox((0, 0), txt, font=font)
+ tx, ty = u - (x1 - x0) / 2 - x0, v - (y1 - y0) / 2 - y0
+ except Exception:
+ tw, th = draw.textsize(txt, font=font)
+ tx, ty = u - tw / 2, v - th / 2
+ draw.text((tx, ty), txt, fill=(10, 10, 10), font=font)
+ return img
+
+
+def build_instances(
+ model,
+ selected: torch.Tensor,
+ cfg,
+ prompt: str,
+ *,
+ link_frac: float | None = None,
+ min_size: int | None = None,
+ metadata: dict | None = None,
+) -> InstanceResult:
+ """Connected-components a selection into an :class:`InstanceResult`."""
+ comp_ids, infos = connected_components(model.means.detach(), selected, cfg, link_frac=link_frac, min_size=min_size)
+ instances = [
+ Instance(
+ idx=d["idx"],
+ size=d["size"],
+ centroid=d["centroid"],
+ bbox_min=d["bbox_min"],
+ bbox_max=d["bbox_max"],
+ )
+ for d in infos
+ ]
+ return InstanceResult(
+ prompt=prompt,
+ instances=instances,
+ comp_ids=comp_ids,
+ config=cfg,
+ _model=model,
+ metadata=metadata or {},
+ )
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/ipc.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/ipc.py
new file mode 100644
index 0000000..0d904a7
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/ipc.py
@@ -0,0 +1,335 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Tiny line-delimited JSON IPC over a Unix domain socket.
+
+Backs the warm-engine daemon: ``gls serve`` holds the caches in one process, and
+``gls segment`` auto-detects that process and queries it. This is *only* an
+optimization -- the daemon is a thin wrapper around the same stateless
+:func:`geolangsplat.api.segment`, so nothing about the public API depends on it.
+"""
+from __future__ import annotations
+
+import glob
+import hashlib
+import json
+import os
+import pathlib
+import socket
+import time
+
+SOCKET_GLOB = "/tmp/gls-*.sock"
+
+# Single lifecycle knob's default: a warm engine auto-stops after this many idle
+# minutes so a forgotten daemon never pins the GPU. Sliding window; reset only by
+# real queries. 0/negative = pin until explicitly stopped.
+DEFAULT_KEEP_ALIVE_MIN = 15.0
+
+
+def default_socket(model_path) -> str:
+ """A stable per-model socket path under /tmp, so client and server agree
+ without the user having to pass an explicit ``--socket``."""
+ key = str(pathlib.Path(model_path).expanduser().resolve())
+ h = hashlib.sha1(key.encode()).hexdigest()[:10]
+ return f"/tmp/gls-{h}.sock"
+
+
+def list_sockets() -> list[str]:
+ """All candidate GeoLangSplat daemon sockets on this machine."""
+ return sorted(glob.glob(SOCKET_GLOB))
+
+
+# -- pidfile: lets `gls stop` hard-kill a wedged daemon and lets cleanup detect a
+# dead one even when its stale socket file lingers. Written next to the socket. --
+
+
+def pidfile_for(socket_path: str) -> str:
+ return socket_path + ".pid"
+
+
+def write_pidfile(socket_path: str, pid: int | None = None) -> None:
+ try:
+ with open(pidfile_for(socket_path), "w") as f:
+ f.write(str(pid if pid is not None else os.getpid()))
+ except OSError:
+ pass
+
+
+def read_pid(socket_path: str) -> int | None:
+ try:
+ with open(pidfile_for(socket_path)) as f:
+ return int(f.read().strip())
+ except (OSError, ValueError):
+ return None
+
+
+def pid_alive(pid: int) -> bool:
+ """True if a process with ``pid`` exists (signal 0 probes without killing)."""
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ return True # exists, owned by someone else
+ except OSError:
+ return False
+ return True
+
+
+def remove_runtime_files(socket_path: str) -> None:
+ """Remove the socket + pidfile for a daemon (idempotent)."""
+ for p in (socket_path, pidfile_for(socket_path)):
+ try:
+ os.unlink(p)
+ except OSError:
+ pass
+
+
+def cleanup_stale(socket_path: str) -> bool:
+ """Remove the socket + pidfile of a daemon that is gone. Returns True if a
+ stale file was removed. Leaves a live daemon untouched."""
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ s.settimeout(1.0)
+ try:
+ s.connect(socket_path)
+ return False # someone is listening -> not stale
+ except ConnectionRefusedError:
+ remove_runtime_files(socket_path) # bound earlier, nobody home now
+ return True
+ except OSError:
+ # missing socket or connect timeout: trust the pidfile to decide.
+ pid = read_pid(socket_path)
+ if pid is not None and not pid_alive(pid):
+ remove_runtime_files(socket_path)
+ return True
+ return False
+ finally:
+ s.close()
+
+
+def kill_daemon(socket_path: str, grace: float = 5.0) -> str:
+ """Guaranteed-best-effort shutdown: ask politely over IPC, then SIGTERM, then
+ SIGKILL, always cleaning up the socket + pidfile. Returns a short status word
+ ('not running' | 'stopped' | 'killed' | 'stop attempted')."""
+ import signal as _signal
+
+ pid = read_pid(socket_path)
+ alive = daemon_alive(socket_path)
+ if not alive and (pid is None or not pid_alive(pid)):
+ remove_runtime_files(socket_path)
+ return "not running"
+
+ if alive: # 1) polite, lets the daemon free VRAM on its own terms
+ try:
+ request(socket_path, {"cmd": ":shutdown"}, timeout=grace)
+ except OSError:
+ pass
+ deadline = time.time() + grace
+ while time.time() < deadline:
+ if not daemon_alive(socket_path, timeout=0.5) and not (pid and pid_alive(pid)):
+ remove_runtime_files(socket_path)
+ return "stopped"
+ time.sleep(0.2)
+
+ if pid and pid_alive(pid): # 2) escalate: it's wedged or won't exit
+ for sig in (_signal.SIGTERM, _signal.SIGKILL):
+ try:
+ os.kill(pid, sig)
+ except OSError:
+ break
+ deadline = time.time() + grace
+ while time.time() < deadline and pid_alive(pid):
+ time.sleep(0.2)
+ if not pid_alive(pid):
+ break
+
+ remove_runtime_files(socket_path)
+ return "killed" if (pid and not pid_alive(pid)) else "stopped"
+
+
+def request_status(socket_path: str, timeout: float = 2.0) -> dict | None:
+ """Ask a daemon for its status payload, or None if it isn't answering."""
+ try:
+ resp = request(socket_path, {"cmd": ":status"}, timeout=timeout)
+ except OSError:
+ return None
+ return resp if resp.get("ok") else None
+
+
+def send_obj(conn: socket.socket, obj: dict) -> None:
+ conn.sendall((json.dumps(obj) + "\n").encode())
+
+
+def recv_obj(conn: socket.socket) -> dict:
+ """Read one newline-terminated JSON object. Returns {} on empty/closed."""
+ buf = bytearray()
+ while not buf.endswith(b"\n"):
+ chunk = conn.recv(65536)
+ if not chunk:
+ break
+ buf.extend(chunk)
+ line = buf.decode().strip()
+ return json.loads(line) if line else {}
+
+
+def request(socket_path: str, obj: dict, timeout: float | None = 600.0) -> dict:
+ """Connect to a running daemon, send one request, return its reply."""
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ if timeout is not None:
+ s.settimeout(timeout)
+ try:
+ s.connect(socket_path)
+ send_obj(s, obj)
+ return recv_obj(s)
+ finally:
+ s.close()
+
+
+def daemon_alive(socket_path: str, timeout: float = 2.0) -> bool:
+ """True if a daemon is listening on ``socket_path`` and answers a ping."""
+ try:
+ resp = request(socket_path, {"cmd": ":ping"}, timeout=timeout)
+ except OSError: # ConnectionRefused / FileNotFound / timeout all subclass OSError
+ return False
+ return bool(resp.get("ok"))
+
+
+def spawn_daemon(
+ model_path,
+ *,
+ recipe: str | None = None,
+ sfm: str | None = None,
+ view_source: str | None = None,
+ max_views: int | None = None,
+ vram_budget_gb: float | None = None,
+ up: str | None = None,
+ select: float | None = None,
+ compete: bool = False,
+ fast_views: int | None = None,
+ device: str | None = None,
+ socket_path: str | None = None,
+ log_path: str | None = None,
+ keep_alive: float | None = None,
+):
+ """Launch ``gls serve`` as a detached background process.
+
+ Returns ``(proc, log_path)``; ``proc.poll()`` lets the caller notice a crash
+ instead of waiting out the whole readiness timeout.
+ """
+ import subprocess
+ import sys
+
+ sock = socket_path or default_socket(model_path)
+ log = log_path or (sock + ".log")
+ open(log, "wb").close() # truncate previous log so we stream only this run
+ args = [sys.executable, "-m", "geolangsplat", "serve", str(model_path), "--socket-path", sock]
+ if recipe:
+ args += ["--recipe", recipe]
+ if sfm:
+ args += ["--sfm", sfm]
+ if view_source:
+ args += ["--view-source", view_source]
+ if max_views is not None:
+ args += ["--max-views", str(max_views)]
+ if vram_budget_gb is not None:
+ args += ["--vram-budget-gb", str(vram_budget_gb)]
+ if up:
+ args += ["--up", up]
+ if select is not None:
+ args += ["--select", str(select)]
+ if compete:
+ args += ["--compete"]
+ if fast_views is not None:
+ args += ["--fast-views", str(fast_views)]
+ if keep_alive is not None:
+ args += ["--keep-alive", str(keep_alive)]
+ if device:
+ args += ["--device", device]
+ fh = open(log, "ab")
+ proc = subprocess.Popen(
+ args,
+ stdout=fh,
+ stderr=fh,
+ stdin=subprocess.DEVNULL,
+ start_new_session=True, # detach: survives this CLI process exiting
+ )
+ return proc, log
+
+
+def wait_ready(
+ socket_path: str,
+ proc=None,
+ log_path: str | None = None,
+ timeout: float = 1800.0,
+ interval: float = 0.5,
+ stream: bool = True,
+) -> bool:
+ """Wait until the daemon answers a ping. Fails fast if ``proc`` exits first,
+ and (when ``stream``) tails the build log so the wait does not look hung."""
+ import time
+
+ t0 = time.time()
+ pos = 0
+ while time.time() - t0 < timeout:
+ if stream and log_path:
+ pos = _tail(log_path, pos)
+ if daemon_alive(socket_path, timeout=2.0):
+ if stream and log_path:
+ _tail(log_path, pos)
+ return True
+ if proc is not None and proc.poll() is not None: # daemon died during build
+ if stream and log_path:
+ _tail(log_path, pos)
+ return False
+ time.sleep(interval)
+ return False
+
+
+def _tail(log_path: str, pos: int) -> int:
+ """Print new bytes appended to ``log_path`` since ``pos``; return new offset."""
+ try:
+ with open(log_path, "r", errors="replace") as fh:
+ fh.seek(pos)
+ chunk = fh.read()
+ if chunk:
+ print(chunk, end="", flush=True)
+ return fh.tell()
+ except OSError:
+ return pos
+
+
+def handle_request(engine, req: dict) -> dict:
+ """Run one query request against a prebuilt engine and (optionally) write a file.
+
+ Reuses :func:`geolangsplat.api.segment` so output formats stay identical to
+ the one-shot CLI. Per-request ``select``/``compete`` overrides persist on the
+ engine config (a session naturally remembers your last setting).
+ """
+ from .api import segment
+
+ prompt = (req.get("prompt") or "").strip()
+ if not prompt:
+ return {"ok": False, "error": "empty prompt"}
+
+ if req.get("select") is not None:
+ engine.cfg.select = float(req["select"])
+ if req.get("margin") is not None:
+ engine.cfg.margin = float(req["margin"])
+ if req.get("compete") is not None:
+ engine.cfg.compete = bool(req["compete"])
+
+ output = req.get("output") or "mask"
+ out_path = req.get("out_path")
+ if output != "mask" and not out_path:
+ return {"ok": False, "error": f"output={output!r} requires out_path"}
+
+ res = segment(engine.model, prompt, engine=engine, output=output, out_path=out_path)
+ return {
+ "ok": True,
+ "prompt": prompt,
+ "n": res.num_selected,
+ "N": int(res.scores.shape[0]),
+ "t": res.stats.get("query_seconds"),
+ "output": output,
+ "path": (str(out_path) if output != "mask" else None),
+ }
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/lift.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/lift.py
new file mode 100644
index 0000000..754ff91
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/lift.py
@@ -0,0 +1,465 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Lift 2D SAM3 score maps onto per-Gaussian scores.
+
+The default, recommended lift is alpha-weighted back-projection: for each view we
+ask fVDB which Gaussians contribute to each pixel and by how much
+(``render_contributing_gaussian_ids``), then distribute that pixel's SAM3 score
+to those Gaussians weighted by their contribution. Summing over views and
+dividing by the accumulated weight gives a continuous per-Gaussian score (no
+binary "is this Gaussian in the mask" decision, which is what makes naive
+back-projection look splatty).
+
+A legacy depth-band lift (single visible Gaussian per pixel + multi-view
+consensus) is also provided for the rendered-orbit path.
+"""
+from __future__ import annotations
+
+import math
+import time
+from dataclasses import dataclass, field
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+from .cameras import _perp_basis, intrinsics, project, up_vector
+from .sam3 import encode_progress, encode_progress_done
+from .views import View, core_extent
+
+_EPS = 1e-8
+
+
+# --- alpha-weighted lift ---------------------------------------------------
+
+
+@dataclass
+class AlphaCache:
+ """Per-view contribution lists + accumulated per-Gaussian weight."""
+
+ a_ids: list = field(default_factory=list) # [Mi] long, gaussian id per contribution
+ a_pix: list = field(default_factory=list) # [Mi] long, pixel index per contribution
+ a_w: list = field(default_factory=list) # [Mi] half, contribution weight
+ a_hw: list = field(default_factory=list) # (h, w) render resolution per view
+ denom: torch.Tensor | None = None # [N] total accumulated weight per gaussian
+ seen: torch.Tensor | None = None # [N] number of views each gaussian appears in
+
+
+def _lift_resolution(height: int, width: int, lift_res: int) -> tuple[int, int]:
+ """Scale a view down so its long side is ``lift_res`` (caps render cost)."""
+ scale = lift_res / max(height, width) if max(height, width) > lift_res else 1.0
+ rh = max(int(round(height * scale)), 1)
+ rw = max(int(round(width * scale)), 1)
+ return rh, rw
+
+
+@torch.no_grad()
+def view_contrib(model, v: View, cfg, device: torch.device):
+ """Per-pixel top-k contributing Gaussians for one view.
+
+ Returns ``(ids[M] long, pix[M] long, w[M] float, (rh, rw))`` -- the flat list
+ of (gaussian, pixel, weight) contributions at the lift resolution. Shared by the
+ cached build (:func:`build_alpha_cache`) and the streaming lift, so both compute
+ the lift identically.
+ """
+ rh, rw = _lift_resolution(v.height, v.width, cfg.lift_res)
+ sx, sy = rw / v.width, rh / v.height
+ K = v.K.clone().float()
+ K[0, :] *= sx
+ K[1, :] *= sy
+ ids_j, w_j = model.render_contributing_gaussian_ids(
+ world_to_camera_matrices=v.w2c.unsqueeze(0).float(),
+ projection_matrices=K.unsqueeze(0),
+ image_width=rw,
+ image_height=rh,
+ near=0.01,
+ far=1e12,
+ top_k_contributors=cfg.top_k,
+ )
+ ids = ids_j.jdata.reshape(-1).long()
+ w = w_j.jdata.reshape(-1).float()
+ off = ids_j.joffsets.reshape(-1).long()
+ counts = off[1:] - off[:-1]
+ npix = rh * rw
+ pix = torch.repeat_interleave(torch.arange(counts.numel(), device=device) % npix, counts)
+ valid = ids >= 0
+ return ids[valid], pix[valid], w[valid], (rh, rw)
+
+
+@torch.no_grad()
+def build_alpha_cache(model, views: list[View], cfg, device: torch.device) -> AlphaCache:
+ """Render per-pixel top-k contributing Gaussians for every view and cache them."""
+ N = model.means.shape[0]
+ cache = AlphaCache(denom=torch.zeros(N, device=device), seen=torch.zeros(N, device=device))
+ t0 = time.time()
+ for v in views:
+ ids, pix, w, (rh, rw) = view_contrib(model, v, cfg, device)
+ cache.denom.scatter_add_(0, ids, w)
+ # seen counts distinct VIEWS per Gaussian (not per-pixel contributions),
+ # so it is bounded by len(views) and is a meaningful coverage signal.
+ cache.seen[torch.unique(ids)] += 1.0
+ cache.a_ids.append(ids)
+ cache.a_pix.append(pix)
+ cache.a_w.append(w.half())
+ cache.a_hw.append((rh, rw))
+ print(f"[lift] alpha cache for {len(views)} views in {time.time() - t0:.1f}s", flush=True)
+ return cache
+
+
+@torch.inference_mode()
+def aggregate_alpha(
+ scorer,
+ states,
+ cache: AlphaCache,
+ prompt: str,
+ cfg,
+ text_outputs=None,
+ want_peak: bool = False,
+ want_support: bool = False,
+):
+ """Alpha-weighted per-Gaussian score for ``prompt`` (mean over contributions),
+ optionally also the per-Gaussian peak and a multi-view support count.
+
+ Returns ``mean [N]``; with ``want_peak`` and/or ``want_support`` it returns the
+ extra tensors after ``mean`` in the order ``(mean[, peak][, support])``. ``support``
+ counts, per Gaussian, the number of views in which its best contributed score
+ cleared ``cfg.consensus_thr`` -- the signal the consensus gate thresholds on.
+ """
+ N = cache.denom.shape[0]
+ dev = cache.denom.device
+ num = torch.zeros(N, device=dev)
+ peak = torch.zeros(N, device=dev) if want_peak else None
+ support = torch.zeros(N, device=dev) if want_support else None
+ sthr = float(getattr(cfg, "consensus_thr", 0.3))
+ amp_dtype = getattr(scorer, "amp_dtype", torch.bfloat16)
+ with torch.autocast("cuda", dtype=amp_dtype, enabled=(dev.type == "cuda")):
+ if text_outputs is None:
+ text_outputs = scorer.forward_text(prompt)
+ for i, state in enumerate(states):
+ ids = cache.a_ids[i]
+ if ids.numel() == 0:
+ continue
+ rh, rw = cache.a_hw[i]
+ # Decode masks straight to the lift resolution: avoids upsampling every
+ # instance mask to full photo res just to immediately shrink it.
+ smap = scorer.scoremap(prompt, state, text_outputs, out_hw=(rh, rw))
+ if smap is None:
+ continue
+ if smap.shape != (rh, rw): # band/stand-in scorers may ignore out_hw
+ smap = F.interpolate(smap[None, None].float(), size=(rh, rw), mode="bilinear", align_corners=False)[
+ 0, 0
+ ]
+ vals = smap.reshape(-1)[cache.a_pix[i].long()]
+ w = cache.a_w[i].float()
+ num.scatter_add_(0, ids, w * vals)
+ if want_peak:
+ peak.scatter_reduce_(0, ids, vals, reduce="amax", include_self=True)
+ if want_support:
+ # per-view best score per Gaussian, then a vote if it clears the floor
+ vview = torch.zeros(N, device=dev)
+ vview.scatter_reduce_(0, ids, vals, reduce="amax", include_self=True)
+ support += (vview >= sthr).float()
+ mean = num / cache.denom.clamp_min(_EPS)
+ out = [mean]
+ if want_peak:
+ out.append(peak)
+ if want_support:
+ out.append(support)
+ return tuple(out) if len(out) > 1 else mean
+
+
+def alpha_qscore(scorer, states, cache: AlphaCache, prompt: str, cfg, text_outputs=None) -> torch.Tensor:
+ """Per-Gaussian query score blending mean and (optionally) peak per ``cfg.peak``."""
+ if cfg.peak > 0:
+ qmean, qpk = aggregate_alpha(scorer, states, cache, prompt, cfg, text_outputs, want_peak=True)
+ return (1 - cfg.peak) * qmean + cfg.peak * qpk
+ return aggregate_alpha(scorer, states, cache, prompt, cfg, text_outputs)
+
+
+def alpha_qscore_support(scorer, states, cache: AlphaCache, prompt: str, cfg, text_outputs=None):
+ """Like :func:`alpha_qscore` but also returns the per-Gaussian multi-view support
+ count for the consensus gate. Returns ``(qscore [N], support [N])``."""
+ if cfg.peak > 0:
+ qmean, qpk, support = aggregate_alpha(
+ scorer, states, cache, prompt, cfg, text_outputs, want_peak=True, want_support=True
+ )
+ return (1 - cfg.peak) * qmean + cfg.peak * qpk, support
+ qmean, support = aggregate_alpha(scorer, states, cache, prompt, cfg, text_outputs, want_support=True)
+ return qmean, support
+
+
+@torch.inference_mode()
+def aggregate_alpha_views(
+ scorer, states, cache: AlphaCache, prompt: str, cfg, view_idx, text_outputs=None
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Alpha aggregation over a SUBSET of views (the fast query path).
+
+ Identical math to :func:`aggregate_alpha`, but only the views in ``view_idx`` are
+ decoded and the denominator/seen counts are accumulated over *just that subset* (so
+ the per-Gaussian mean stays correct). The grounding decode is the per-query cost, so
+ decoding k of N views is ~N/k faster. Returns ``(qscore [N], denom [N], seen [N])``
+ -- denom/seen feed selection so it behaves like the full path on the chosen views.
+ """
+ N = cache.denom.shape[0]
+ dev = cache.denom.device
+ num = torch.zeros(N, device=dev)
+ denom = torch.zeros(N, device=dev)
+ seen = torch.zeros(N, device=dev)
+ peak = torch.zeros(N, device=dev) if cfg.peak > 0 else None
+ amp_dtype = getattr(scorer, "amp_dtype", torch.bfloat16)
+ with torch.autocast("cuda", dtype=amp_dtype, enabled=(dev.type == "cuda")):
+ if text_outputs is None:
+ text_outputs = scorer.forward_text(prompt)
+ for i in view_idx:
+ ids = cache.a_ids[i]
+ if ids.numel() == 0:
+ continue
+ rh, rw = cache.a_hw[i]
+ w = cache.a_w[i].float()
+ denom.scatter_add_(0, ids, w)
+ seen[torch.unique(ids)] += 1.0
+ smap = scorer.scoremap(prompt, states[i], text_outputs, out_hw=(rh, rw))
+ if smap is None:
+ continue
+ if smap.shape != (rh, rw):
+ smap = F.interpolate(smap[None, None].float(), size=(rh, rw), mode="bilinear", align_corners=False)[
+ 0, 0
+ ]
+ vals = smap.reshape(-1)[cache.a_pix[i].long()]
+ num.scatter_add_(0, ids, w * vals)
+ if peak is not None:
+ peak.scatter_reduce_(0, ids, vals, reduce="amax", include_self=True)
+ mean = num / denom.clamp_min(_EPS)
+ if peak is not None:
+ mean = (1 - cfg.peak) * mean + cfg.peak * peak
+ return mean, denom, seen
+
+
+# --- streaming (low-VRAM) alpha lift ---------------------------------------
+
+
+def _view_azimuth(v: View, center: np.ndarray, up: np.ndarray) -> float:
+ """Azimuth (deg) of a view's camera around the up axis, in the perpendicular
+ plane. Used to measure whether agreeing views come from *different sides*."""
+ c2w = torch.linalg.inv(v.w2c.float()).cpu().numpy()
+ d = c2w[:3, 3].astype(np.float64) - center
+ d = d - float(d @ up) * up # project onto the plane perpendicular to up
+ e1, e2 = _perp_basis(up)
+ return math.degrees(math.atan2(float(d @ e2), float(d @ e1)))
+
+
+def _azimuth_coverage(azimuths: list[float]) -> float:
+ """How much of the 360 deg ring the azimuths span = 360 - largest gap.
+
+ Two views on opposite sides -> ~180; views clustered on one side -> small. This
+ is the "across each other enough" guard so we never early-stop from a cluster of
+ near-identical viewpoints all looking from the same direction.
+ """
+ if len(azimuths) < 2:
+ return 0.0
+ a = sorted(x % 360.0 for x in azimuths)
+ gaps = [a[i + 1] - a[i] for i in range(len(a) - 1)] + [a[0] + 360.0 - a[-1]]
+ return 360.0 - max(gaps)
+
+
+@torch.inference_mode()
+def stream_scores(
+ model,
+ scorer,
+ views: list[View],
+ cfg,
+ prompts,
+ device: torch.device,
+ *,
+ want_peak: bool = False,
+ early_stop: bool = False,
+):
+ """Low-VRAM lift: render + encode + score + accumulate + EVICT, one view at a time.
+
+ Equivalent to building the full cache and running :func:`aggregate_alpha` for each
+ prompt, but it never holds more than a single view's embedding resident -- so peak
+ VRAM is bounded to ``SAM3 weights + one view`` regardless of view count (vs. the
+ warm path's all-views embedding cache). The trade is no reuse: every call re-encodes,
+ so this backs the one-shot path while ``gls serve`` keeps the fast warm cache.
+
+ With a single prompt and ``early_stop``, it stops once ``cfg.agree_k`` views with a
+ hit span ``cfg.min_azimuth_spread`` of azimuth (agreement from different sides) and
+ the selection has converged -- bounding work to "as many views as the object needs".
+ Early-stop is only sound for a *compact* subject; the engine gates it to
+ object/dome captures (see ``GeoLangSplat._stream_early_stop``).
+
+ Views are processed in chunks of ``cfg.stream_chunk``: each chunk is encoded in one
+ batched SAM3 call (fast) then evicted before the next, so peak VRAM is bounded to
+ ``SAM3 weights + one chunk`` regardless of view count -- vs. the warm path's
+ all-views embedding cache. The trade is no reuse: every call re-encodes, so this
+ backs the one-shot path while ``gls serve`` keeps the fast warm cache.
+
+ Returns ``(scores [C, N], denom [N], seen [N], stats)``.
+ """
+ N = model.means.shape[0]
+ C = len(prompts)
+ num = torch.zeros(C, N, device=device)
+ denom = torch.zeros(N, device=device)
+ seen = torch.zeros(N, device=device)
+ peak = torch.zeros(C, N, device=device) if want_peak else None
+
+ cap = int(getattr(cfg, "view_cap", 0) or 0)
+ cap = min(cap, len(views)) if cap > 0 else len(views)
+ chunk = max(1, int(getattr(cfg, "stream_chunk", 8)))
+ batch_pref = bool(getattr(cfg, "batch_encode", True))
+ agree_k = int(getattr(cfg, "agree_k", 12))
+ min_spread = float(getattr(cfg, "min_azimuth_spread", 120.0))
+ conv = float(getattr(cfg, "converge_frac", 0.02))
+ center, _r = core_extent(model.means)
+ up = up_vector(getattr(cfg, "up", "+z"))
+ hit_az: list[float] = []
+ prev_sel = -1
+ used = 0
+ stopped = False
+ amp_dtype = getattr(scorer, "amp_dtype", torch.bfloat16)
+
+ def _encode_chunk(vlist):
+ """Batched encode, grouping by resolution so a single odd-sized view in the
+ chunk doesn't force the whole chunk onto the slow per-view path (matters for
+ aerial real photos, which can vary in size). Order is preserved."""
+ if not batch_pref or len(vlist) <= 1:
+ return [scorer.encode([v.pil], v.height, v.width, batch_encode=False)[0] for v in vlist]
+ groups: dict = {}
+ for i, v in enumerate(vlist):
+ groups.setdefault((v.height, v.width), []).append(i)
+ states = [None] * len(vlist)
+ for (h, w), idxs in groups.items():
+ pils = [vlist[i].pil for i in idxs]
+ enc = scorer.encode(pils, h, w, batch_encode=(len(idxs) > 1), batch_size=len(idxs))
+ for k, i in enumerate(idxs):
+ states[i] = enc[k]
+ return states
+
+ with torch.autocast("cuda", dtype=amp_dtype, enabled=(device.type == "cuda")):
+ text_outs = [scorer.forward_text(p) for p in prompts]
+ for c0 in range(0, cap, chunk):
+ chunk_views = [views[vi] for vi in range(c0, min(c0 + chunk, cap))]
+ # 1) render contributions for the chunk (cheap, no embeddings resident)
+ contribs = [view_contrib(model, v, cfg, device) for v in chunk_views]
+ # 2) one batched SAM3 encode for the whole chunk
+ states = _encode_chunk(chunk_views)
+ # 3) score + accumulate, then evict the chunk
+ for v, (ids, pix, w, (rh, rw)), state in zip(chunk_views, contribs, states):
+ used += 1
+ if ids.numel() == 0:
+ continue
+ denom.scatter_add_(0, ids, w)
+ seen[torch.unique(ids)] += 1.0
+ hit = False
+ for c, p in enumerate(prompts):
+ smap = scorer.scoremap(p, state, text_outs[c], out_hw=(rh, rw))
+ if smap is None:
+ continue
+ if smap.shape != (rh, rw):
+ smap = F.interpolate(
+ smap[None, None].float(), size=(rh, rw), mode="bilinear", align_corners=False
+ )[0, 0]
+ vals = smap.reshape(-1)[pix]
+ num[c].scatter_add_(0, ids, w * vals)
+ if want_peak:
+ peak[c].scatter_reduce_(0, ids, vals, reduce="amax", include_self=True)
+ if c == 0 and float((vals > cfg.sam_conf).sum()) > 0:
+ hit = True
+ if early_stop and C == 1 and hit:
+ hit_az.append(_view_azimuth(v, center, up))
+ del contribs, states # evict the chunk's footprint before the next
+ encode_progress(min(c0 + chunk, cap), cap)
+
+ if early_stop and C == 1 and len(hit_az) >= agree_k and _azimuth_coverage(hit_az) >= min_spread:
+ cur = int(((num[0] / denom.clamp_min(_EPS)) >= cfg.select).sum())
+ if prev_sel >= 0 and abs(cur - prev_sel) <= conv * max(prev_sel, 1):
+ stopped = True
+ break
+ prev_sel = cur
+ encode_progress_done()
+
+ mean = num / denom.clamp_min(_EPS)
+ if want_peak:
+ mean = (1 - cfg.peak) * mean + cfg.peak * peak
+ stats = {"views_used": used, "views_total": len(views), "early_stopped": stopped}
+ return mean, denom, seen, stats
+
+
+# --- legacy depth-band lift (rendered-orbit path) --------------------------
+
+
+@dataclass
+class BandCache:
+ """Single-visible-Gaussian-per-pixel cache for the depth-band lift."""
+
+ cidx: list = field(default_factory=list) # [Ki] gaussian ids visible in view i
+ cvv: list = field(default_factory=list) # [Ki] pixel rows
+ cuu: list = field(default_factory=list) # [Ki] pixel cols
+ hw: list = field(default_factory=list)
+ seen: torch.Tensor | None = None
+
+
+@torch.no_grad()
+def build_band_cache(model, views: list[View], cfg, device: torch.device) -> BandCache:
+ """Project means into each view and keep the depth-visible ones."""
+ means = model.means.detach()
+ N = means.shape[0]
+ cache = BandCache(seen=torch.zeros(N, device=device))
+ for v in views:
+ H, W = v.height, v.width
+ f = float(v.K[0, 0])
+ cx, cy = float(v.K[0, 2]), float(v.K[1, 2])
+ img, alpha = model.render_images_and_depths(
+ world_to_camera_matrices=v.w2c.unsqueeze(0).float(),
+ projection_matrices=v.K.unsqueeze(0).float(),
+ image_width=W,
+ image_height=H,
+ near=0.01,
+ far=1e12,
+ )
+ depth = img[0, ..., 3]
+ a0 = alpha[0, ..., 0]
+ u, vv, zc = project(means, v.w2c.float(), f, cx, cy)
+ ui, vj = u.round().long(), vv.round().long()
+ infr = (zc > 0.01) & (ui >= 0) & (ui < W) & (vj >= 0) & (vj < H)
+ uic, vjc = ui.clamp(0, W - 1), vj.clamp(0, H - 1)
+ vis = infr & (a0[vjc, uic] > 0.5) & (zc <= depth[vjc, uic] * (1.0 + cfg.depth_tol))
+ idx = vis.nonzero(as_tuple=True)[0]
+ cache.cidx.append(idx)
+ cache.cvv.append(vjc[idx])
+ cache.cuu.append(uic[idx])
+ cache.hw.append((H, W))
+ cache.seen[idx] += 1
+ return cache
+
+
+@torch.inference_mode()
+def aggregate_band(
+ scorer, states, cache: BandCache, prompt: str, cfg, text_outputs=None
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Footprint-sampled, multi-view-consensus per-Gaussian score. Returns
+ ``(smax [N], hits [N])`` where ``smax`` is the best per-view score and
+ ``hits`` counts views scoring above ``cfg.view_thresh``."""
+ N = cache.seen.shape[0]
+ dev = cache.seen.device
+ smax = torch.zeros(N, device=dev)
+ hits = torch.zeros(N, device=dev)
+ k = 2 * int(cfg.foot) + 1
+ amp_dtype = getattr(scorer, "amp_dtype", torch.bfloat16)
+ with torch.autocast("cuda", dtype=amp_dtype, enabled=(dev.type == "cuda")):
+ if text_outputs is None:
+ text_outputs = scorer.forward_text(prompt)
+ for i, state in enumerate(states):
+ idx = cache.cidx[i]
+ if idx.numel() == 0:
+ continue
+ smap = scorer.scoremap(prompt, state, text_outputs)
+ if smap is None:
+ continue
+ if k > 1:
+ smap = F.max_pool2d(smap[None, None].float(), kernel_size=k, stride=1, padding=int(cfg.foot))[0, 0]
+ vals = smap[cache.cvv[i], cache.cuu[i]].float()
+ smax[idx] = torch.maximum(smax[idx], vals)
+ hits[idx] += (vals >= cfg.view_thresh).float()
+ return smax, hits
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/outputs.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/outputs.py
new file mode 100644
index 0000000..fd8acad
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/outputs.py
@@ -0,0 +1,161 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Output writers: segmented ``.ply``, highlight-overlay ``.ply``, and
+per-Gaussian reports (``.csv`` / ``.npz``).
+
+All per-Gaussian arrays stay in the input ``.ply`` order so a consumer (e.g. a
+co-worker holding the same ``.ply``) can index directly by row.
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+
+import numpy as np
+import torch
+
+SH_C0 = 0.28209479177387814
+
+# A reusable categorical palette (RGB 0..1) for label recolouring.
+_PALETTE = np.array(
+ [
+ [0.90, 0.10, 0.10],
+ [0.10, 0.70, 0.20],
+ [0.15, 0.45, 0.95],
+ [0.95, 0.75, 0.10],
+ [0.65, 0.25, 0.85],
+ [0.10, 0.80, 0.80],
+ [0.95, 0.45, 0.10],
+ [0.55, 0.55, 0.55],
+ [0.85, 0.30, 0.55],
+ [0.40, 0.30, 0.20],
+ ],
+ dtype=np.float32,
+)
+
+
+def _gs():
+ from fvdb import GaussianSplat3d
+
+ return GaussianSplat3d
+
+
+def _save_ply(disp, path: str, metadata: dict | None) -> None:
+ """Save a splat, round-tripping metadata when the backend accepts it.
+
+ fvdb's ``save_ply`` only takes scalar/tensor metadata values; if the dict it
+ came with has anything richer (e.g. nested camera info from a checkpoint), we
+ retry without metadata rather than failing the whole write.
+ """
+ if metadata:
+ try:
+ disp.save_ply(path, metadata=metadata)
+ return
+ except Exception:
+ pass
+ disp.save_ply(path)
+
+
+def _subset(model, mask: torch.Tensor):
+ return _gs().from_tensors(
+ means=model.means.detach()[mask],
+ quats=model.quats.detach()[mask],
+ log_scales=model.log_scales.detach()[mask],
+ logit_opacities=model.logit_opacities.detach()[mask],
+ sh0=model.sh0.detach()[mask],
+ shN=model.shN.detach()[mask],
+ )
+
+
+def write_ply_segmented(model, selected: torch.Tensor, path, metadata: dict | None = None) -> int:
+ """Write a ``.ply`` containing only the selected Gaussians."""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ sub = _subset(model, selected)
+ _save_ply(sub, str(path), metadata)
+ return int(selected.sum())
+
+
+def _recolor_sh0(sh0: torch.Tensor, mask: torch.Tensor, color, blend: float) -> torch.Tensor:
+ tgt = torch.tensor(color, device=sh0.device, dtype=sh0.dtype).view(1, 1, 3)
+ cur = 0.5 + SH_C0 * sh0[mask]
+ sh0 = sh0.clone()
+ sh0[mask] = ((1 - blend) * cur + blend * tgt - 0.5) / SH_C0
+ return sh0
+
+
+def write_ply_overlay(
+ model, selected: torch.Tensor, path, color=(1.0, 0.95, 0.1), blend: float = 0.75, metadata: dict | None = None
+) -> int:
+ """Write the full ``.ply`` with the selected Gaussians tinted ``color``."""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ sh0 = model.sh0.detach()
+ shN = model.shN.detach().clone()
+ if bool(selected.any()):
+ sh0 = _recolor_sh0(sh0, selected, color, blend)
+ shN[selected] = shN[selected] * (1 - blend)
+ else:
+ sh0 = sh0.clone()
+ disp = _gs().from_tensors(
+ means=model.means.detach(),
+ quats=model.quats.detach(),
+ log_scales=model.log_scales.detach(),
+ logit_opacities=model.logit_opacities.detach(),
+ sh0=sh0,
+ shN=shN,
+ )
+ _save_ply(disp, str(path), metadata)
+ return int(selected.sum())
+
+
+def write_ply_labels(model, label_ids: torch.Tensor, path, blend: float = 0.85, metadata: dict | None = None) -> int:
+ """Recolour the full ``.ply`` by per-Gaussian label (unlabeled stay original)."""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ sh0 = model.sh0.detach().clone()
+ shN = model.shN.detach().clone()
+ labels = label_ids.detach().cpu().numpy()
+ for lid in np.unique(labels):
+ if lid < 0:
+ continue
+ mask = torch.from_numpy(labels == lid).to(sh0.device)
+ color = tuple(_PALETTE[int(lid) % len(_PALETTE)].tolist())
+ sh0 = _recolor_sh0(sh0, mask, color, blend)
+ shN[mask] = shN[mask] * (1 - blend)
+ disp = _gs().from_tensors(
+ means=model.means.detach(),
+ quats=model.quats.detach(),
+ log_scales=model.log_scales.detach(),
+ logit_opacities=model.logit_opacities.detach(),
+ sh0=sh0,
+ shN=shN,
+ )
+ _save_ply(disp, str(path), metadata)
+ return int((label_ids >= 0).sum())
+
+
+def write_report_csv(means: np.ndarray, label_ids: np.ndarray, scores: np.ndarray, vocab, path) -> None:
+ """Write ``index,x,y,z,label_id,label,score`` plus a sibling ``legend.json``."""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ vocab = list(vocab)
+ lines = ["index,x,y,z,label_id,label,score"]
+ for i in range(means.shape[0]):
+ lid = int(label_ids[i])
+ name = vocab[lid] if 0 <= lid < len(vocab) else ""
+ sc = float(scores[i]) if scores.ndim == 1 else float(scores[i, lid]) if lid >= 0 else 0.0
+ x, y, z = means[i]
+ lines.append(f"{i},{x:.6f},{y:.6f},{z:.6f},{lid},{name},{sc:.4f}")
+ path.write_text("\n".join(lines) + "\n")
+ legend = {str(i): v for i, v in enumerate(vocab)}
+ legend["-1"] = "unlabeled"
+ (path.parent / "legend.json").write_text(json.dumps(legend, indent=2))
+
+
+def write_report_npz(means: np.ndarray, label_ids: np.ndarray, scores: np.ndarray, vocab, path) -> None:
+ """Write a compact ``.npz`` (means, label_ids, scores, vocab)."""
+ path = pathlib.Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ np.savez_compressed(str(path), means=means, label_ids=label_ids, scores=scores, vocab=np.array(list(vocab)))
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/profile.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/profile.py
new file mode 100644
index 0000000..7fff1b7
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/profile.py
@@ -0,0 +1,62 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""A tiny, always-on step profiler for the engine build / cache-load paths.
+
+GPU work is asynchronous, so wall-clock around a CUDA call under-counts it. Each
+span therefore ``torch.cuda.synchronize()``-es on enter and exit, giving an
+honest per-step breakdown (load splat, render views, alpha lift, SAM3 load, SAM3
+encode, cache read/restore). The overhead (a couple of syncs per major step) is
+negligible next to the steps themselves, so it stays on by default -- the whole
+point of this exercise is to know where the seconds go.
+"""
+from __future__ import annotations
+
+import time
+from contextlib import contextmanager
+
+import torch
+
+
+class Stopwatch:
+ """Accumulates named, CUDA-synchronized spans and prints a breakdown."""
+
+ def __init__(self, device=None, enabled: bool = True):
+ self.device = device
+ self.enabled = enabled
+ self.spans: list[tuple[str, float]] = []
+ self._cuda = bool(device is not None and getattr(device, "type", None) == "cuda")
+
+ @contextmanager
+ def span(self, name: str):
+ if not self.enabled:
+ yield
+ return
+ if self._cuda:
+ torch.cuda.synchronize(self.device)
+ t = time.perf_counter()
+ try:
+ yield
+ finally:
+ if self._cuda:
+ torch.cuda.synchronize(self.device)
+ self.spans.append((name, time.perf_counter() - t))
+
+ def add(self, name: str, seconds: float) -> None:
+ """Record a pre-measured span (for time spent outside a ``span`` block)."""
+ self.spans.append((name, float(seconds)))
+
+ def total(self) -> float:
+ return sum(s for _, s in self.spans)
+
+ def report(self, title: str = "profile") -> str:
+ if not self.spans:
+ return ""
+ tot = self.total() or 1e-9
+ w = max(len(n) for n, _ in self.spans)
+ lines = [f"[{title}] step breakdown (cuda-synced):"]
+ for n, s in self.spans:
+ bar = "#" * int(round(28 * s / tot))
+ lines.append(f" {n.ljust(w)} {s:8.2f}s {100 * s / tot:5.1f}% {bar}")
+ lines.append(f" {'TOTAL'.ljust(w)} {tot:8.2f}s")
+ return "\n".join(lines)
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/sam3.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/sam3.py
new file mode 100644
index 0000000..dfc589b
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/sam3.py
@@ -0,0 +1,340 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Thin SAM3 wrapper: build the image model, cache per-view image embeddings,
+and turn a text prompt into a per-pixel score map.
+
+SAM3 is mixed precision and must run under autocast. The class is duck-typed so
+tests can substitute a lightweight stand-in (anything exposing ``encode`` and
+``scoremap``); see ``tests/conftest.py``.
+"""
+from __future__ import annotations
+
+import contextlib
+import os
+import pathlib
+import warnings
+from typing import Any
+
+import torch
+import torch.nn.functional as F
+
+from .config import DEFAULT_DISTRACTORS
+from .errors import GeoLangSplatError
+
+
+@contextlib.contextmanager
+def _quiet_load():
+ """Silence SAM3's import/load chatter (timm/pkg_resources FutureWarnings and the
+ model loader's ``missing_keys`` dump) while still letting real exceptions raise."""
+ with (
+ warnings.catch_warnings(),
+ open(os.devnull, "w") as _dn,
+ contextlib.redirect_stdout(_dn),
+ contextlib.redirect_stderr(_dn),
+ ):
+ warnings.simplefilter("ignore")
+ yield
+
+
+def encode_progress(done: int, total: int, label: str = "encode") -> None:
+ """Draw a single in-place progress line (carriage return, no newline)."""
+ width = 24
+ filled = int(width * done / max(total, 1))
+ bar = "#" * filled + "-" * (width - filled)
+ print(f"\r[{label}] [{bar}] {done}/{total} views", end="", flush=True)
+
+
+def encode_progress_done() -> None:
+ """Terminate a progress line started by :func:`encode_progress`."""
+ print(flush=True)
+
+
+def _resolve_amp_dtype(amp: str, device: str) -> torch.dtype:
+ """bf16 by default, but fall back to fp16 on GPUs that lack bf16 support."""
+ want = amp.lower()
+ if want == "fp16":
+ return torch.float16
+ if str(device).startswith("cuda") and torch.cuda.is_available():
+ try:
+ if not torch.cuda.is_bf16_supported():
+ print("[sam3] bf16 unsupported on this GPU -> using fp16", flush=True)
+ return torch.float16
+ except Exception:
+ pass
+ return torch.bfloat16
+
+
+class Sam3Scorer:
+ """Wraps a SAM3 image model + processor for promptable per-pixel scoring."""
+
+ def __init__(
+ self,
+ sam_ckpt: str,
+ sam_res: int = 1008,
+ sam_conf: float = 0.20,
+ amp: str = "bf16",
+ device: str = "cuda",
+ dual_head: bool = False,
+ sem_weight: float = 0.5,
+ sem_mode: str = "mean",
+ ):
+ if not str(sam_ckpt).strip():
+ raise GeoLangSplatError(
+ "SAM3 checkpoint is not set. Point GeoLangSplat at the weights via the "
+ "GEOLANGSPLAT_SAM_CKPT environment variable, e.g.\n"
+ " export GEOLANGSPLAT_SAM_CKPT=/path/to/sam3.1_multiplex.pt\n"
+ "(or pass GeoLangSplatConfig(sam_ckpt=...)). Geometry-only `gls check` needs no weights."
+ )
+ if not pathlib.Path(sam_ckpt).exists():
+ raise GeoLangSplatError(
+ f"SAM3 checkpoint not found: {sam_ckpt}\n"
+ "Set it via GEOLANGSPLAT_SAM_CKPT or GeoLangSplatConfig(sam_ckpt=...). SAM3 weights "
+ "are required for segmentation; geometry-only `gls check` does not need them."
+ )
+ try:
+ with _quiet_load():
+ from sam3.model.sam3_image_processor import Sam3Processor
+ from sam3.model_builder import build_sam3_image_model
+ except Exception as ex: # SAM3 package not installed in this env
+ raise GeoLangSplatError(
+ f"the `sam3` package is required for segmentation but could not be imported ({ex}).\n"
+ "Install SAM3 in this environment, or use geometry-only `gls check`."
+ ) from ex
+
+ with _quiet_load():
+ model = build_sam3_image_model(device=device, checkpoint_path=sam_ckpt, load_from_HF=False)
+ self.proc = Sam3Processor(model, resolution=sam_res, device=device, confidence_threshold=sam_conf)
+ self.model = model
+ self.amp_dtype = _resolve_amp_dtype(amp, device)
+ self.device = device
+ self.dual_head = bool(dual_head) # fuse the dense semantic head into the score map
+ self.sem_weight = float(sem_weight)
+ self.sem_mode = str(sem_mode)
+ self._dummy_prompt = None # cached image-independent dummy geometric prompt
+ self._batch_ok = None # tri-state: None=unverified, True/False=cached split-vs-ref check
+
+ # -- embedding cache ----------------------------------------------------
+
+ @staticmethod
+ def _split_backbone(bb: Any, i: int) -> Any:
+ """Index batch element ``i`` out of a (possibly nested) backbone_out."""
+ if isinstance(bb, dict):
+ return {k: Sam3Scorer._split_backbone(v, i) for k, v in bb.items()}
+ if isinstance(bb, (list, tuple)):
+ return type(bb)(Sam3Scorer._split_backbone(v, i) for v in bb)
+ if torch.is_tensor(bb):
+ if bb.dim() >= 1 and bb.shape[0] > i:
+ return bb[i : i + 1].contiguous()
+ return bb
+ return bb
+
+ def _verify_batch_split(self, pils, height: int, width: int) -> bool:
+ """One-time check that indexing a batched encode matches the per-view encode.
+
+ Cached on ``self._batch_ok`` so it runs ONCE per process, not per encode call.
+ The streaming lift encodes in many small chunks, and re-validating every chunk
+ was ~3 wasted encodes each -- the dominant overhead vs. the warm build (which
+ validates once). Correctness still never depends on the batched split: if the
+ check fails we fall back to per-view encoding everywhere.
+ """
+ if self._batch_ok is not None:
+ return self._batch_ok
+ try:
+ sb = self.proc.set_image_batch(pils[:2])
+ v1 = {
+ "backbone_out": self._split_backbone(sb["backbone_out"], 1),
+ "original_height": height,
+ "original_width": width,
+ }
+ ref1 = self.proc.set_image(pils[1])
+ a = self.scoremap("building", v1)
+ b = self.scoremap("building", ref1)
+ self._batch_ok = (a is None and b is None) or (
+ a is not None
+ and b is not None
+ and a.shape == b.shape
+ and torch.allclose(a.float(), b.float(), atol=2e-2, rtol=2e-2)
+ )
+ except Exception as ex:
+ print(f"[encode] batched encode unavailable ({ex}) -> per-view", flush=True)
+ self._batch_ok = False
+ return self._batch_ok
+
+ @torch.inference_mode()
+ def encode(
+ self, pils, height: int, width: int, batch_encode: bool = True, batch_size: int = 8, progress: bool = False
+ ):
+ """Run the SAM3 image encoder once per view and cache the result.
+
+ Tries a batched encode for a faster build; the batched-split-vs-per-view check
+ is validated once per process (:meth:`_verify_batch_split`) and on any mismatch
+ or error it falls back to per-view, so correctness never depends on batching.
+ ``progress`` draws a single in-place progress line over the encode (used by the
+ warm build; the streaming lift draws its own across chunks).
+ """
+ n = len(pils)
+ if n == 0:
+ return []
+ with torch.autocast("cuda", dtype=self.amp_dtype):
+ if batch_encode and n > 1 and self._verify_batch_split(pils, height, width):
+ states: list = [None] * n
+ for c0 in range(0, n, batch_size):
+ chunk = pils[c0 : c0 + batch_size]
+ sb = self.proc.set_image_batch(chunk)
+ for j in range(len(chunk)):
+ states[c0 + j] = {
+ "backbone_out": self._split_backbone(sb["backbone_out"], j),
+ "original_height": height,
+ "original_width": width,
+ }
+ if progress:
+ encode_progress(min(c0 + batch_size, n), n)
+ if progress:
+ encode_progress_done()
+ return states
+ states = []
+ for k, p in enumerate(pils):
+ states.append(self.proc.set_image(p))
+ if progress:
+ encode_progress(k + 1, n)
+ if progress:
+ encode_progress_done()
+ return states
+
+ # -- scoring ------------------------------------------------------------
+
+ def forward_text(self, prompt: str):
+ """Encode the text prompt once (image-independent); reuse across views."""
+ return self.proc.model.backbone.forward_text([prompt], device=self.device)
+
+ def _decode(self, prompt: str, state: dict, text_outputs=None, out_hw=None):
+ """Run SAM3's grounding decode for ``prompt`` on a cached embedding and return
+ the raw output dict (``masks_logits`` [K,1,h,w] probs, ``scores`` [K], ...).
+
+ Critically we decode on a *throwaway* working dict, not on ``state`` itself:
+ ``_forward_grounding`` writes the (full-stack) mask tensors back into the dict
+ it is given, and ``state`` lives in the resident per-view embedding cache.
+ Mutating it would keep every view's masks alive for the whole query (they would
+ accumulate across all views instead of being freed per view) -- the original
+ cause of the runaway query VRAM.
+ """
+ if text_outputs is None:
+ text_outputs = self.forward_text(prompt)
+ if self._dummy_prompt is None:
+ self._dummy_prompt = self.proc.model._get_dummy_prompt()
+ backbone_out = dict(state["backbone_out"]) # shallow copy: shares tensors, new keys are local
+ backbone_out.update(text_outputs)
+ work = {
+ "backbone_out": backbone_out,
+ "original_height": out_hw[0] if out_hw else state["original_height"],
+ "original_width": out_hw[1] if out_hw else state["original_width"],
+ "geometric_prompt": self._dummy_prompt,
+ }
+ return self.proc._forward_grounding(work)
+
+ def scoremap(self, prompt: str, state: dict, text_outputs=None, out_hw=None):
+ """Text + grounding decode on a cached image embedding -> ``[H, W]`` scores
+ (or ``None`` if SAM3 returns no masks).
+
+ ``out_hw=(h, w)`` decodes the masks straight to that size instead of the
+ original photo resolution. The alpha lift downsamples to its lift
+ resolution anyway, so passing the lift size here avoids upsampling every
+ instance mask to full photo res and then throwing it away -- a large
+ memory + time saving on high-resolution aerial captures.
+
+ With ``dual_head`` the dense semantic head is fused in (see
+ :meth:`_fuse_heads`); otherwise this is the presence-gated instance head only.
+ """
+ if self.dual_head:
+ h = out_hw[0] if out_hw else state["original_height"]
+ w = out_hw[1] if out_hw else state["original_width"]
+ outputs = self._grounding_outputs(prompt, state, text_outputs)
+ return self._fuse_heads(outputs, h, w)
+ st = self._decode(prompt, state, text_outputs, out_hw)
+ ml = st.get("masks_logits")
+ if ml is None or ml.shape[0] == 0:
+ return None
+ sc = st["scores"]
+ return (ml[:, 0] * sc.view(-1, 1, 1)).amax(0).float() # [H, W]
+
+ # -- dual-head (instance + semantic) fusion -----------------------------
+
+ def _grounding_outputs(self, prompt: str, state: dict, text_outputs=None) -> dict:
+ """Run SAM3's grounding model once and return its *raw* output dict.
+
+ Unlike :meth:`_decode` (which routes through the processor's
+ ``_forward_grounding`` and keeps only instance masks/scores), this keeps the
+ full output -- including ``semantic_seg`` -- so both heads come from a single
+ forward. As in :meth:`_decode`, we build a throwaway working ``backbone_out``
+ so we never mutate the resident per-view embedding cache.
+ """
+ if text_outputs is None:
+ text_outputs = self.forward_text(prompt)
+ if self._dummy_prompt is None:
+ self._dummy_prompt = self.proc.model._get_dummy_prompt()
+ backbone_out = dict(state["backbone_out"]) # shallow copy: shares tensors
+ backbone_out.update(text_outputs)
+ return self.proc.model.forward_grounding(
+ backbone_out=backbone_out,
+ find_input=self.proc.find_stage,
+ geometric_prompt=self._dummy_prompt,
+ find_target=None,
+ )
+
+ def _instance_from_outputs(self, outputs: dict, img_h: int, img_w: int):
+ """Presence-gated instance head -> ``[H, W]`` in [0, 1] (or ``None``).
+
+ Mirrors the processor's ``_forward_grounding`` exactly (presence multiply +
+ ``sam_conf`` keep), minus the box bookkeeping we don't need.
+ """
+ out_logits = outputs["pred_logits"]
+ out_masks = outputs["pred_masks"]
+ out_probs = out_logits.sigmoid()
+ presence = outputs["presence_logit_dec"].sigmoid().unsqueeze(1)
+ out_probs = (out_probs * presence).squeeze(-1)
+ keep = out_probs > self.proc.confidence_threshold
+ out_probs = out_probs[keep]
+ out_masks = out_masks[keep]
+ if out_masks.shape[0] == 0:
+ return None
+ out_masks = F.interpolate(
+ out_masks.unsqueeze(1).float(), (img_h, img_w), mode="bilinear", align_corners=False
+ ).sigmoid()
+ return (out_masks[:, 0] * out_probs.view(-1, 1, 1).float()).amax(0).float()
+
+ def _semantic_from_outputs(self, outputs: dict, img_h: int, img_w: int):
+ """Dense prompt-conditioned semantic head -> ``[H, W]`` in [0, 1] (or ``None``).
+
+ ``semantic_seg`` is a single-channel logit map (``Conv2d(.., 1, 1)``); we take
+ the first image, sigmoid, and resize to the requested resolution.
+ """
+ sem = outputs.get("semantic_seg")
+ if sem is None:
+ return None
+ sem = sem.float()
+ while sem.dim() < 4: # [h,w]/[1,h,w] -> [1,1,h,w]
+ sem = sem.unsqueeze(0)
+ sem = F.interpolate(sem[:1], (img_h, img_w), mode="bilinear", align_corners=False).sigmoid()
+ return sem[0, 0]
+
+ def _fuse_heads(self, outputs: dict, img_h: int, img_w: int):
+ """Blend the instance and semantic heads into one ``[H, W]`` score (or ``None``).
+
+ Missing heads count as 0, so the blend is continuous when either head fires
+ alone -- the semantic-only case (instance head filtered out by ``sam_conf``) is
+ exactly the recall the dense head is meant to recover. ``sem_mode='mean'`` is a
+ convex blend; ``'max'`` takes the stronger weighted head per pixel.
+ """
+ inst = self._instance_from_outputs(outputs, img_h, img_w)
+ sem = self._semantic_from_outputs(outputs, img_h, img_w)
+ if inst is None and sem is None:
+ return None
+ w = self.sem_weight
+ if inst is None:
+ inst = torch.zeros_like(sem)
+ if sem is None:
+ sem = torch.zeros_like(inst)
+ if self.sem_mode == "max":
+ return torch.maximum((1.0 - w) * inst, w * sem)
+ return (1.0 - w) * inst + w * sem
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/select.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/select.py
new file mode 100644
index 0000000..aa6e884
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/select.py
@@ -0,0 +1,212 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Turn per-Gaussian scores into selections / labels.
+
+* :func:`select_query` - threshold + concept competition for a single prompt.
+* :func:`spatial_cleanup` - voxel cull of isolated selected Gaussians (floaters).
+* :func:`assign_labels` - multi-class argmax with confidence/ambiguity/weight gates.
+"""
+from __future__ import annotations
+
+import torch
+
+_EPS = 1e-8
+
+# Coarse "is a kind of" map so we don't compete a query against its own
+# hypernym/synonym (e.g. don't let the "vehicle" distractor suppress "car").
+HYPERNYMS: dict[str, set[str]] = {
+ "car": {"vehicle"},
+ "bus": {"vehicle"},
+ "truck": {"vehicle"},
+ "van": {"vehicle"},
+ "house": {"building"},
+ "garage": {"building"},
+ "road": {"pavement", "ground"},
+ "sidewalk": {"pavement", "ground"},
+ "pavement": {"ground"},
+ "grass": {"vegetation", "ground"},
+ "tree": {"vegetation"},
+}
+
+
+def _sing(w: str) -> str:
+ """Crude singularizer so 'cars' competes like 'car'."""
+ w = w.lower().strip()
+ return w[:-1] if w.endswith("s") and len(w) > 3 else w
+
+
+def competitor_idx(query: str, dist_names) -> list[int]:
+ """Indices of distractors the query should compete against (drops the query
+ itself, substring synonyms, and the query's hypernyms/co-hyponyms)."""
+ ql = _sing(query)
+ q_parents = HYPERNYMS.get(ql, set())
+ keep: list[int] = []
+ for i, n in enumerate(dist_names):
+ nl = n.lower().strip()
+ if nl in query.lower() or query.lower() in nl:
+ continue
+ if _sing(nl) == ql or _sing(nl) in q_parents:
+ continue
+ keep.append(i)
+ return keep
+
+
+def smooth_scores(
+ means: torch.Tensor,
+ qscore: torch.Tensor,
+ denom: torch.Tensor | None,
+ cfg,
+) -> torch.Tensor:
+ """Voxel-grid smoothing of per-Gaussian scores (training-free regularization).
+
+ Blends each Gaussian's score with the (optionally render-weighted) mean score of
+ the Gaussians sharing its voxel. This denoises the lifted score and fills objects
+ -- the cheap analog of LangSplat's per-scene field optimization -- and is applied
+ *before* thresholding so it lifts both recall (filled interiors) and precision
+ (isolated false positives get pulled down toward their empty neighborhood).
+ """
+ if not getattr(cfg, "smooth", False) or float(cfg.smooth_beta) <= 0:
+ return qscore
+ span = float((means.max(dim=0).values - means.min(dim=0).values).max())
+ vox = max(cfg.smooth_vox_frac * span, 1e-9)
+ keys = torch.floor(means / vox).long()
+ _uniq, inv = torch.unique(keys, dim=0, return_inverse=True)
+ M = int(inv.max()) + 1 if inv.numel() else 0
+ if M == 0:
+ return qscore
+ if denom is not None and getattr(cfg, "smooth_weighted", True):
+ w = denom.clamp_min(_EPS)
+ else:
+ w = torch.ones_like(qscore)
+ num = torch.zeros(M, device=qscore.device, dtype=qscore.dtype).scatter_add_(0, inv, qscore * w)
+ den = torch.zeros(M, device=qscore.device, dtype=qscore.dtype).scatter_add_(0, inv, w)
+ local = (num / den.clamp_min(_EPS))[inv]
+ beta = float(cfg.smooth_beta)
+ return (1.0 - beta) * qscore + beta * local
+
+
+def select_query(
+ qscore: torch.Tensor,
+ seen: torch.Tensor,
+ cfg,
+ *,
+ query: str = "",
+ select: float | None = None,
+ margin: float | None = None,
+ compete: bool | None = None,
+ denom: torch.Tensor | None = None,
+ dist_scores: torch.Tensor | None = None,
+ dist_names=None,
+ select_mode: str | None = None,
+ select_rel: float | None = None,
+ min_keep: int | None = None,
+ support: torch.Tensor | None = None,
+) -> torch.Tensor:
+ """Boolean ``[N]`` selection mask for a single prompt.
+
+ The candidate set (observed, well-observed, competition-surviving Gaussians) is
+ fixed first; the score threshold is then applied within it. ``select_mode``:
+
+ * ``"fixed"`` -- keep candidates with ``qscore >= select`` (absolute floor).
+ * ``"relative"`` -- keep candidates with ``qscore >= select_rel * max(qscore)``
+ over the candidates. Robust to prompts whose grounding scores are globally
+ weak (the main recall killer); always keeps at least the peak candidate.
+
+ ``min_keep`` is a non-empty guard for ``fixed`` mode: if the threshold selects
+ nothing, the top-``min_keep`` candidates by ``qscore`` are kept instead.
+ """
+ select = cfg.select if select is None else select
+ margin = cfg.margin if margin is None else margin
+ compete = cfg.compete if compete is None else compete
+ select_mode = getattr(cfg, "select_mode", "fixed") if select_mode is None else select_mode
+ select_rel = getattr(cfg, "select_rel", 0.5) if select_rel is None else select_rel
+ min_keep = getattr(cfg, "min_keep", 0) if min_keep is None else min_keep
+
+ cand = seen > 0
+ if denom is not None and cfg.min_weight > 0:
+ cand = cand & (denom >= cfg.min_weight)
+ if support is not None and getattr(cfg, "consensus", False):
+ need = torch.clamp(
+ float(getattr(cfg, "consensus_frac", 0.0)) * seen,
+ min=float(getattr(cfg, "consensus_min", 1)),
+ )
+ cand = cand & (support >= need)
+ if compete and dist_scores is not None and dist_names:
+ keep = competitor_idx(query, dist_names)
+ if keep:
+ dmax = dist_scores[:, keep].max(dim=1).values
+ cand = cand & (qscore >= dmax + margin)
+
+ if select_mode == "relative":
+ cand_scores = qscore[cand]
+ thr = select_rel * float(cand_scores.max()) if cand_scores.numel() else select
+ sel = cand & (qscore >= thr)
+ else:
+ sel = cand & (qscore >= select)
+
+ if min_keep > 0 and int(sel.sum()) == 0 and int(cand.sum()) > 0:
+ cand_idx = cand.nonzero(as_tuple=True)[0]
+ k = min(int(min_keep), cand_idx.numel())
+ top = qscore[cand_idx].topk(k).indices
+ sel = torch.zeros_like(cand)
+ sel[cand_idx[top]] = True
+ return sel
+
+
+def spatial_cleanup(means: torch.Tensor, sel: torch.Tensor, cfg) -> torch.Tensor:
+ """Drop isolated selected Gaussians: voxelize, keep voxels with >= min_pts."""
+ if not cfg.clean3d or int(sel.sum()) == 0:
+ return sel
+ pts = means[sel]
+ span = float((means.max(dim=0).values - means.min(dim=0).values).max())
+ vox = max(cfg.voxel_frac * span, 1e-9)
+ keys = torch.floor(pts / vox).long()
+ _uniq, inv, counts = torch.unique(keys, dim=0, return_inverse=True, return_counts=True)
+ survive = counts[inv] >= cfg.min_pts
+ out = sel.clone()
+ out[sel.nonzero(as_tuple=True)[0]] = survive
+ return out
+
+
+def assign_labels(
+ scores: torch.Tensor,
+ denom: torch.Tensor,
+ min_weight: float,
+ tau: float,
+ delta: float,
+) -> tuple[torch.Tensor, dict]:
+ """Multi-class label per Gaussian via argmax with three gates.
+
+ ``scores`` is ``[N, C]`` (one column per vocabulary word). A Gaussian is
+ labeled with its top class only if: the top score clears ``tau`` (confidence),
+ the top1-top2 margin clears ``delta`` (unambiguous), and the accumulated
+ render weight clears ``min_weight`` (well-observed). Otherwise label = -1.
+
+ Returns ``(label_ids [N] long, stats)``.
+ """
+ N, C = scores.shape
+ dev = scores.device
+ k = min(2, C)
+ top = scores.topk(k, dim=1)
+ top1v = top.values[:, 0]
+ top2v = top.values[:, 1] if C > 1 else torch.zeros(N, device=dev)
+ lab = top.indices[:, 0].clone().long()
+
+ confident = top1v >= tau
+ unambiguous = (top1v - top2v) >= delta if C > 1 else torch.ones(N, dtype=torch.bool, device=dev)
+ enough = denom >= min_weight
+ multi = (scores >= tau).sum(dim=1) >= 2 # how many gaussians have >=2 classes above tau
+
+ keep = confident & unambiguous & enough
+ lab[~keep] = -1
+
+ stats = {
+ "total": int(N),
+ "well_observed": int(enough.sum()),
+ "confident": int((confident & enough).sum()),
+ "ambiguous_dropped": int((confident & enough & ~unambiguous).sum()),
+ "multi_class_candidates": int((multi & enough).sum()),
+ "labeled": int((lab >= 0).sum()),
+ }
+ return lab, stats
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/__init__.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/__init__.py
new file mode 100644
index 0000000..503a720
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/__init__.py
@@ -0,0 +1,9 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Internal interactive viewer (test harness only, not part of the product API)."""
+from __future__ import annotations
+
+from .server import run_viewer
+
+__all__ = ["run_viewer"]
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/server.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/server.py
new file mode 100644
index 0000000..b30f3b6
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/server.py
@@ -0,0 +1,365 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Interactive web catalog browser for GeoLangSplat (backs ``gls explore``).
+
+A small web UI drives the object catalog and the fvdb viewer renders the result.
+The flow is one word at a time:
+
+1. type a class ("building", "car") and Search,
+2. every instance of it lights up in its own colour in the 3D view,
+3. click an instance in the list -- the viewer swaps to a **cutout** of just that
+ object (the rest of the scene is removed),
+4. Back returns to the highlighted view so you can pick another or search again.
+
+An optional visual companion for eyeballing and curation; the stable surface is the
+``segment`` / ``build_catalog`` APIs and the ``gls`` CLI.
+"""
+from __future__ import annotations
+
+import json
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import parse_qs, urlparse
+
+import torch
+
+from ..cameras import orbit_cameras
+from ..config import GeoLangSplatConfig, apply_recipe
+from ..engine import GeoLangSplatEngine
+from ..outputs import SH_C0
+
+
+def _frame_viewer(scene, engine, cfg) -> None:
+ """Orient + frame the viewer camera for the scene.
+
+ Sets the up axis (object/COLMAP plys aren't z-up), shrinks the near clip plane
+ (the auto default is too far to zoom *into* small object scenes), and starts the
+ camera at a close, low vantage. Note: fvdb.viz's orbit control clamps how far
+ *below* the orbit plane you can drag (a ground-plane assumption), so the
+ rendered orbit -- which includes below-horizon views for object captures -- is
+ the source of truth; the viewer is just for eyeballing.
+ """
+ import numpy as np
+
+ from .. import autoview
+
+ up = autoview.resolve_up(getattr(cfg, "up", "auto"), engine.means)
+ try:
+ m = engine.means.detach().float().cpu().numpy()
+ lo = np.quantile(m, 0.02, axis=0)
+ hi = np.quantile(m, 0.98, axis=0)
+ center = (lo + hi) / 2
+ radius = max(float(np.linalg.norm(hi - lo)) / 2, 0.001)
+ scene.camera_up_direction = up
+ scene.camera_near = max(radius * 0.002, 0.0001)
+ scene.camera_far = radius * 100
+ c2w = orbit_cameras(center, 18, radius * 2.2, num_azimuth=1, azimuth_offset_deg=35, up=up)[0][1]
+ scene.set_camera_lookat(c2w[:3, 3], center, up)
+ except Exception as e:
+ print(f"[viewer] could not frame scene (up={getattr(cfg, 'up', '+z')!r}): {e}", flush=True)
+
+
+_CHIPS = ("building", "car", "tree", "road")
+
+
+def _object_palette(k: int) -> list[tuple[float, float, float]]:
+ """``k`` visually distinct RGB colors (golden-ratio hue walk) for object recolor."""
+ import colorsys
+
+ return [colorsys.hsv_to_rgb((i * 0.61803398875) % 1.0, 0.65, 1.0) for i in range(max(k, 1))]
+
+
+PAGE = """GeoLangSplat catalog
+
+
+type a class and Search — each instance lights up; click one to cut it out
+
+
+
← Back to all Export all
+
+
+
+
:{cfg.web_port} [viewer] http://:{cfg.viewer_port}", flush=True)
+ httpd.serve_forever()
diff --git a/open_vocabulary_segmentation/geolangsplat/geolangsplat/views.py b/open_vocabulary_segmentation/geolangsplat/geolangsplat/views.py
new file mode 100644
index 0000000..4ca5d24
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/geolangsplat/views.py
@@ -0,0 +1,596 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Unified view generation.
+
+Two ways to get the 2D views that SAM3 segments and that we back-project from:
+
+* ``render`` - synthesize a multi-tier grid of orbit cameras over the splat and
+ render RGB for each (good when there are no photos, or for uniform coverage).
+* ``images`` - use the scene's ground-truth SfM photos and their COLMAP camera
+ poses (SAM3 segments real photos most cleanly).
+
+Both produce a list of :class:`View` (image + camera), which the lift consumes
+identically, so everything downstream is view-source agnostic.
+"""
+from __future__ import annotations
+
+import math
+import pathlib
+import struct
+import time
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+from PIL import Image
+
+from .cameras import inside_out_cameras, intrinsics, orbit_cameras, up_vector
+from .errors import GeoLangSplatError
+
+# COLMAP camera model id -> number of intrinsic params.
+_COLMAP_MODEL_NUM_PARAMS = {0: 3, 1: 4, 2: 4, 3: 5, 4: 8, 5: 8, 6: 12, 7: 5, 8: 4, 9: 5, 10: 12}
+
+
+@dataclass
+class View:
+ """One 2D view of the scene: an image plus its camera (world->camera)."""
+
+ pil: Image.Image
+ w2c: torch.Tensor # [4, 4] float
+ K: torch.Tensor # [3, 3] float
+ height: int
+ width: int
+
+
+def scene_span(means: torch.Tensor) -> float:
+ """Diagonal extent of the scene's axis-aligned bounding box (world units)."""
+ return float((means.float().max(dim=0).values - means.float().min(dim=0).values).norm())
+
+
+def _grid_centers(means: torch.Tensor, grid: int, grid_frac: float) -> list[np.ndarray]:
+ """Footprint-grid of look-at centers at the scene's median height."""
+ lo = 0.5 - grid_frac / 2.0
+ hi = 0.5 + grid_frac / 2.0
+ qx = torch.quantile(means[:, 0], torch.tensor([lo, hi], device=means.device))
+ qy = torch.quantile(means[:, 1], torch.tensor([lo, hi], device=means.device))
+ zc = float(torch.median(means[:, 2]))
+ if grid <= 1:
+ return [np.array([float(qx.mean()), float(qy.mean()), zc])]
+ xs = torch.linspace(float(qx[0]), float(qx[1]), grid)
+ ys = torch.linspace(float(qy[0]), float(qy[1]), grid)
+ return [np.array([float(x), float(y), zc]) for x in xs for y in ys]
+
+
+@torch.no_grad()
+def render_orbit_views(model, cfg, device: torch.device) -> list[View]:
+ """Synthesize and render the multi-tier orbit grid (``view_source='render'``)."""
+ means = model.means.detach()
+ up = up_vector(getattr(cfg, "up", "+z"))
+ views: list[View] = []
+ t0 = time.time()
+ for tier in cfg.tiers:
+ K_np = intrinsics(tier.fov_deg, cfg.size, cfg.size)
+ K = torch.from_numpy(K_np).float().unsqueeze(0).to(device)
+ rad = tier.radius * cfg.zoom
+ for center in _grid_centers(means, tier.grid, cfg.grid_frac):
+ for elev in tier.elevations:
+ for ai in range(tier.num_azimuth):
+ az = 360.0 * ai / tier.num_azimuth
+ w2c_np = orbit_cameras(center, elev, rad, num_azimuth=1, azimuth_offset_deg=az, up=up)[0][0]
+ w2c = torch.from_numpy(w2c_np).float().to(device)
+ img, _alpha = model.render_images_and_depths(
+ world_to_camera_matrices=w2c.unsqueeze(0),
+ projection_matrices=K,
+ image_width=cfg.size,
+ image_height=cfg.size,
+ near=0.01,
+ far=1e12,
+ )
+ rgb = img[0, ..., :3]
+ pil = Image.fromarray((rgb.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8))
+ views.append(View(pil=pil, w2c=w2c, K=K[0], height=cfg.size, width=cfg.size))
+ print(f"[views] rendered {len(views)} orbit views in {time.time() - t0:.1f}s", flush=True)
+ return views
+
+
+@torch.no_grad()
+def render_inside_out_views(model, cfg, device: torch.device) -> list[View]:
+ """Render from the scene core looking OUTWARD (interior / unbounded scenes).
+
+ For rooms and unbounded captures (Mip-NeRF360 indoor), an outside-in orbit only
+ sees the backs of walls. We anchor the eye at the robust scene center (per-axis
+ median, ignoring the background shell) and sweep a wide-FOV azimuth ring at a few
+ pitches, so the surrounding content is framed.
+ """
+ means = model.means.detach()
+ up = up_vector(getattr(cfg, "up", "+z"))
+ center = means.float().median(dim=0).values.cpu().numpy().astype(np.float64)
+ elevations = (-25.0, 0.0, 25.0)
+ budget = int(getattr(cfg, "max_views", 0) or 120)
+ num_az = max(6, budget // len(elevations))
+ fov = float(getattr(cfg, "inside_out_fov", 80.0))
+ K_np = intrinsics(fov, cfg.size, cfg.size)
+ K = torch.from_numpy(K_np).float().unsqueeze(0).to(device)
+ views: list[View] = []
+ t0 = time.time()
+ for w2c_np, _c2w in inside_out_cameras(center, elevations, num_az, up=up):
+ w2c = torch.from_numpy(w2c_np).float().to(device)
+ img, _alpha = model.render_images_and_depths(
+ world_to_camera_matrices=w2c.unsqueeze(0),
+ projection_matrices=K,
+ image_width=cfg.size,
+ image_height=cfg.size,
+ near=0.01,
+ far=1e12,
+ )
+ rgb = img[0, ..., :3]
+ pil = Image.fromarray((rgb.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8))
+ views.append(View(pil=pil, w2c=w2c, K=K[0], height=cfg.size, width=cfg.size))
+ print(
+ f"[views] rendered {len(views)} inside-out views ({len(elevations)} pitch x {num_az} az, "
+ f"fov {fov:.0f}) in {time.time() - t0:.1f}s",
+ flush=True,
+ )
+ return views
+
+
+@torch.no_grad()
+# --- globe orbit: auto-framed radius + full-sphere rings -------------------
+
+
+@torch.no_grad()
+def core_extent(means: torch.Tensor, q_lo: float = 0.1, q_hi: float = 0.9):
+ """Center + radius of the scene's DENSE core (quantile-trimmed).
+
+ Unbounded 360/interior captures carry a sparse far halo of Gaussians that
+ blows up the bbox; the q10-q90 box ignores it and tracks the actual subject.
+ Returns ``(center[3] np.float64, radius float)`` where radius is half the core
+ box diagonal.
+ """
+ m = means.detach().float()
+ q = torch.tensor([q_lo, q_hi], device=m.device, dtype=m.dtype)
+ lo_hi = torch.quantile(m, q, dim=0)
+ lo, hi = lo_hi[0], lo_hi[1]
+ center = ((lo + hi) / 2.0).cpu().numpy().astype(np.float64)
+ radius = 0.5 * float((hi - lo).norm())
+ return center, max(radius, 1e-4)
+
+
+# Default frame fill: <1 lets the dense core overfill the frame so the subject reads
+# large -- this is what SAM3 wants (small objects like a mitten get enough pixels to
+# fire). Shared by the `globe` segment views and the `render` preview so what you
+# eyeball is exactly what SAM sees. Tune per-call with --zoom.
+_FRAME_FILL = 0.55
+
+
+def auto_frame_radius(model, center, up, *, fov: float, zoom: float = 1.0, **_ignored) -> float:
+ """Look-at distance that frames the scene's dense core to ~fill the view.
+
+ Frames the q10-q90 core (not the floater-inflated bbox, which is what parked the
+ camera absurdly far on 360 scenes). Analytic and floater-robust: distance so the
+ core's angular size matches the field of view, scaled by ``_FRAME_FILL`` (a fixed
+ "sit a little closer" default) and the caller's ``zoom`` (``<1`` pulls in, ``>1``
+ backs out). ``center``/``up``/extra kwargs are accepted for call-site
+ compatibility but the framing distance only depends on the core size.
+ """
+ _center, core_radius = core_extent(model.means)
+ half = math.radians(fov) / 2.0
+ dist = core_radius / max(math.tan(half), 1e-3)
+ return max(dist * _FRAME_FILL * zoom, core_radius * 0.4)
+
+
+# Dome view-quality tuning (see config.dome_low_zoom / reject_blur / blur_rel).
+_DOME_LOW_ZOOM = 0.72 # default radius scale at the lowest ring (1.0 at nadir)
+_DOME_RESAMPLE_ZOOM = 0.82 # pull a rejected view this much closer on its one retry
+_BLUR_MAX_REJECT = 0.45 # never drop more than this fraction of the dome (safety)
+
+
+def dome_radius_scale(elev: float, elev_min: float, elev_max: float, low: float) -> float:
+ """Per-ring radius multiplier: ``low`` at the lowest (oblique) ring rising to
+ 1.0 at the top (near-nadir) ring. Low/oblique rings see through more foreground
+ clutter, so pulling them closer reframes the subject larger."""
+ if elev_max <= elev_min:
+ return 1.0
+ t = (elev - elev_min) / (elev_max - elev_min)
+ return low + (1.0 - low) * max(0.0, min(1.0, t))
+
+
+def view_sharpness(pil: Image.Image) -> float:
+ """Cheap detail/blur metric: variance of the image Laplacian (grayscale).
+
+ Sharp, well-framed renders carry high-frequency edges -> high variance; a view
+ that renders mostly smooth blur or empty background -> near-zero. Used relative
+ to the median across the view set, so it adapts per scene with no magic absolute
+ threshold. Pure numpy (no cv2/scipy)."""
+ g = np.asarray(pil.convert("L"), dtype=np.float32) / 255.0
+ if g.shape[0] < 3 or g.shape[1] < 3:
+ return 0.0
+ lap = -4.0 * g[1:-1, 1:-1] + g[:-2, 1:-1] + g[2:, 1:-1] + g[1:-1, :-2] + g[1:-1, 2:]
+ return float(lap.var())
+
+
+@torch.no_grad()
+def _render_w2c(model, w2c_np: np.ndarray, K: torch.Tensor, size: int, device: torch.device):
+ """Render one orbit camera; returns ``(pil, w2c_tensor, depth[H,W], alpha[H,W])``.
+
+ Depth and alpha come free from the same rasterization (depth is RGBA channel 3,
+ alpha is the separate coverage buffer) and drive the dome view-quality gate.
+ """
+ w2c = torch.from_numpy(w2c_np).float().to(device)
+ img, a = model.render_images_and_depths(
+ world_to_camera_matrices=w2c.unsqueeze(0),
+ projection_matrices=K,
+ image_width=size,
+ image_height=size,
+ near=0.01,
+ far=1e12,
+ )
+ rgb = img[0, ..., :3]
+ depth = img[0, ..., 3]
+ alpha = a[0, ..., 0]
+ pil = Image.fromarray((rgb.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8))
+ return pil, w2c, depth, alpha
+
+
+def dome_view_metrics(
+ pil: Image.Image, depth: torch.Tensor, alpha: torch.Tensor, radius: float, center_frac: float = 0.6
+):
+ """Quality metrics for a dome view, from the central region where the subject sits.
+
+ Returns ``(coverage, occlusion, sharpness)``:
+
+ * ``coverage`` - fraction of the center actually covered by the splat (low =>
+ the camera is staring through to empty/see-through background).
+ * ``occlusion`` - fraction of the covered center that sits *much* closer than the
+ orbit radius, i.e. a foreground floater/furniture blob blocking the subject
+ (this is the dark-blob failure on the low/oblique rings).
+ * ``sharpness`` - detail energy (variance of Laplacian), a soft-blur backstop.
+
+ Coverage/occlusion are physically grounded (alpha + depth vs. the known camera
+ distance), so the gate generalizes across scenes without appearance tuning.
+ """
+ H, W = alpha.shape
+ m = center_frac
+ y0, y1 = int(H * (0.5 - m / 2)), int(H * (0.5 + m / 2))
+ x0, x1 = int(W * (0.5 - m / 2)), int(W * (0.5 + m / 2))
+ a = alpha[y0:y1, x0:x1]
+ d = depth[y0:y1, x0:x1]
+ solid = a > 0.5
+ nsolid = float(solid.sum())
+ coverage = nsolid / float(a.numel())
+ occlusion = float((solid & (d < 0.5 * radius)).sum()) / nsolid if nsolid > 0 else 1.0
+ return coverage, occlusion, view_sharpness(pil)
+
+
+@torch.no_grad()
+def render_globe_views(model, cfg, device: torch.device) -> list[View]:
+ """Render an upper-dome orbit around the scene core (``view_source='globe'``).
+
+ Auto-frames the radius (see :func:`auto_frame_radius`) then sweeps full azimuth
+ rings at elevations looking DOWN at the core (oblique -> near-nadir). A bare
+ splat only renders sharply near its original (inward/above) camera region, so the
+ dome stays there and avoids the blurry see-through-background angles. This is the
+ generalizable path for object/interior captures with no embedded cameras.
+ """
+ from .autoview import globe_rings
+
+ up = up_vector(getattr(cfg, "up", "+z"))
+ center, _r = core_extent(model.means)
+ fov = 55.0
+ base_radius = auto_frame_radius(model, center, up, fov=fov, zoom=float(getattr(cfg, "zoom", 1.0) or 1.0))
+ budget = int(getattr(cfg, "max_views", 0) or 200)
+ elevs, az_counts = globe_rings(budget)
+ e_lo, e_hi = min(elevs), max(elevs)
+ low_zoom = float(getattr(cfg, "dome_low_zoom", _DOME_LOW_ZOOM))
+ K = torch.from_numpy(intrinsics(fov, cfg.size, cfg.size)).float().unsqueeze(0).to(device)
+ gate = bool(getattr(cfg, "reject_blur", True))
+ min_cov = float(getattr(cfg, "view_min_coverage", 0.22))
+ max_occ = float(getattr(cfg, "view_max_occlusion", 0.45))
+ t0 = time.time()
+
+ def _render(elev, ai, naz, radius):
+ cams = orbit_cameras(center, elev, radius, num_azimuth=naz, up=up)
+ return _render_w2c(model, cams[ai % len(cams)][0], K, cfg.size, device)
+
+ # Render every ring; pull low/oblique rings closer (dome_radius_scale) so blocked
+ # side-on subjects fill more pixels. Gate each view on coverage + occlusion, and
+ # give a failing view one retry from a HIGHER, closer angle (clears floor clutter
+ # and near floaters -- exactly the unusable low-ring failure). This per-view
+ # quality control is what makes the dome generalize to a new scene unattended.
+ total = sum(az_counts)
+ pils: list[Image.Image] = []
+ w2cs: list[torch.Tensor] = []
+ sharps: list[float] = []
+ resampled = dropped = 0
+ for elev, naz in zip(elevs, az_counts):
+ ring_radius = base_radius * dome_radius_scale(elev, e_lo, e_hi, low_zoom)
+ for ai in range(naz):
+ pil, w2c, depth, alpha = _render(elev, ai, naz, ring_radius)
+ cov, occ, sharp = dome_view_metrics(pil, depth, alpha, ring_radius)
+ ok = (not gate) or (cov >= min_cov and occ <= max_occ)
+ if not ok:
+ # rise above the clutter (+ closer) and try once more
+ pil2, w2c2, depth2, alpha2 = _render(elev + 8.0, ai, naz, ring_radius * _DOME_RESAMPLE_ZOOM)
+ cov2, occ2, sharp2 = dome_view_metrics(pil2, depth2, alpha2, ring_radius * _DOME_RESAMPLE_ZOOM)
+ if cov2 >= min_cov and occ2 <= max_occ:
+ pil, w2c, sharp, ok = pil2, w2c2, sharp2, True
+ resampled += 1
+ if ok:
+ pils.append(pil)
+ w2cs.append(w2c)
+ sharps.append(sharp)
+ else:
+ dropped += 1
+
+ # Soft-blur backstop on survivors (catch blur the coverage gate let through),
+ # but never let total drops exceed the safety cap.
+ keep = list(range(len(pils)))
+ if gate and len(pils) >= 4:
+ sh = np.array(sharps, dtype=np.float64)
+ thr = float(getattr(cfg, "blur_rel", 0.40)) * float(np.median(sh))
+ budget = max(0, int(_BLUR_MAX_REJECT * total) - dropped)
+ worst = [i for i in sorted(range(len(pils)), key=lambda i: sh[i]) if sh[i] < thr]
+ drop2 = set(worst[:budget])
+ dropped += len(drop2)
+ keep = [i for i in keep if i not in drop2]
+
+ views = [View(pil=pils[i], w2c=w2cs[i], K=K[0], height=cfg.size, width=cfg.size) for i in keep]
+ extra = f", {resampled} resampled higher, {dropped} dropped (blocked/empty/blur)" if (resampled or dropped) else ""
+ print(
+ f"[views] rendered {len(views)} dome views (auto-framed radius {base_radius:.2f}, "
+ f"low rings x{low_zoom:.2f} closer, {len(elevs)} rings {e_lo:.0f}..{e_hi:.0f} deg "
+ f"looking down{extra}) in {time.time() - t0:.1f}s",
+ flush=True,
+ )
+ return views
+
+
+# --- COLMAP sparse-model reader (ground-truth image views) -----------------
+
+
+def _qvec2rotmat(q: np.ndarray) -> np.ndarray:
+ w, x, y, z = q
+ return np.array(
+ [
+ [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
+ [2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
+ [2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
+ ]
+ )
+
+
+def _read_colmap_cameras(path: pathlib.Path) -> dict:
+ cams = {}
+ with open(path, "rb") as f:
+ (n,) = struct.unpack(" np.ndarray:
+ p = cam["params"]
+ mid = cam["model_id"]
+ if mid == 0: # SIMPLE_PINHOLE: f, cx, cy
+ f, cx, cy = p[0], p[1], p[2]
+ fx = fy = f
+ else: # PINHOLE / radial / opencv all start fx, fy(or f), cx, cy
+ if mid in (1, 4, 6, 10): # fx, fy, cx, cy, ...
+ fx, fy, cx, cy = p[0], p[1], p[2], p[3]
+ else: # SIMPLE_RADIAL etc: f, cx, cy, ...
+ fx = fy = p[0]
+ cx, cy = p[1], p[2]
+ return np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]], dtype=np.float64)
+
+
+def _read_colmap_images(path: pathlib.Path) -> list[dict]:
+ imgs = []
+ with open(path, "rb") as f:
+ (n,) = struct.unpack(" tuple[pathlib.Path, pathlib.Path, pathlib.Path]:
+ """Locate (cameras.bin, images.bin, images_dir) under a scene directory."""
+ for sub in ("sparse/0", "sparse", "colmap/sparse/0", "."):
+ d = sfm / sub
+ if (d / "cameras.bin").is_file() and (d / "images.bin").is_file():
+ for img_sub in ("images", "../images", "../../images"):
+ cand = (d / img_sub).resolve()
+ if cand.is_dir():
+ return d / "cameras.bin", d / "images.bin", cand
+ return d / "cameras.bin", d / "images.bin", sfm / "images"
+ raise FileNotFoundError(f"no COLMAP cameras.bin/images.bin found under {sfm}")
+
+
+def _to_np(x) -> np.ndarray:
+ """Tensor/array-like -> float64 numpy on CPU."""
+ if isinstance(x, torch.Tensor):
+ return x.detach().cpu().numpy()
+ return np.asarray(x)
+
+
+def _colmap_c2w(rec: dict) -> np.ndarray:
+ """COLMAP record (world->camera) -> 4x4 camera->world in the ORIGINAL frame."""
+ w2c = np.eye(4, dtype=np.float64)
+ w2c[:3, :3] = _qvec2rotmat(rec["qvec"])
+ w2c[:3, 3] = rec["tvec"]
+ return np.linalg.inv(w2c)
+
+
+def match_metadata_to_colmap(meta_c2w: np.ndarray, col_c2w: np.ndarray, nt: np.ndarray) -> np.ndarray:
+ """Pair each training (metadata) camera with its source photo.
+
+ The splat's metadata cameras live in the *normalized* splat frame; the COLMAP
+ poses live in the scene's *original* (e.g. UTM) frame. We bring the COLMAP
+ camera centers into the normalized frame via ``normalization_transform`` and,
+ for each metadata camera, pick the nearest COLMAP image by position. Returns an
+ index array ``match[M]`` into the COLMAP set. (Faithful to the validated
+ LangSplat-Aerial ``_match_real_images``.)
+ """
+ norm_c2w = nt[None] @ col_c2w # [P, 4, 4] colmap cameras -> normalized frame
+ mt = meta_c2w[:, :3, 3] # [M, 3]
+ nc = norm_c2w[:, :3, 3] # [P, 3]
+ d = np.linalg.norm(mt[:, None, :] - nc[None, :, :], axis=2) # [M, P]
+ return d.argmin(axis=1)
+
+
+def _load_metadata_image_views(cfg, device: torch.device, metadata: dict) -> list[View]:
+ """Build views from the splat's *own* training cameras (correct frame).
+
+ A splat saved during training carries its camera poses (already in the
+ normalized splat frame), intrinsics, and image sizes in metadata. We render
+ the per-Gaussian lift from those poses and load pixels from the matching
+ source photos -- so the splat and the cameras are always in the same frame
+ (unlike raw COLMAP poses, which sit in the original geospatial frame and would
+ project nothing onto a recentered splat).
+ """
+ if not cfg.sfm:
+ raise GeoLangSplatError(
+ "view-source 'images' (the 'aerial' recipe) needs the scene's source photos: "
+ "pass --sfm (the COLMAP folder with sparse/ and images/). "
+ "If you have no photos, use --view-source render instead."
+ )
+ sfm = pathlib.Path(cfg.sfm)
+ if not sfm.is_dir():
+ raise GeoLangSplatError(f"--sfm path does not exist or is not a directory: {sfm}")
+ _cam_path, img_path, img_dir = _find_colmap(sfm)
+
+ meta_c2w = _to_np(metadata["camera_to_world_matrices"]).astype(np.float64) # [M, 4, 4]
+ Kall = _to_np(metadata["projection_matrices"]).astype(np.float64) # [M, 3, 3]
+ isz = _to_np(metadata["image_sizes"]).astype(np.int64) # [M, 2] = (H, W)
+ nt = metadata.get("normalization_transform")
+ nt = _to_np(nt).astype(np.float64) if nt is not None else np.eye(4)
+
+ recs = sorted(_read_colmap_images(img_path), key=lambda d: d["name"])
+ if not recs:
+ raise GeoLangSplatError(f"no images found in {img_path}")
+ names = [r["name"] for r in recs]
+ col_c2w = np.stack([_colmap_c2w(r) for r in recs], axis=0)
+ match = match_metadata_to_colmap(meta_c2w, col_c2w, nt)
+
+ M = meta_c2w.shape[0]
+ n = min(cfg.n_views, M) if cfg.n_views > 0 else M
+ pick = np.unique(np.linspace(0, M - 1, n).round().astype(int))
+
+ views: list[View] = []
+ for i in pick:
+ H, W = int(isz[i][0]), int(isz[i][1])
+ w2c = np.linalg.inv(meta_c2w[i])
+ pil = Image.open(img_dir / names[int(match[i])]).convert("RGB")
+ if pil.size != (W, H):
+ pil = pil.resize((W, H), Image.BILINEAR)
+ views.append(
+ View(
+ pil=pil,
+ w2c=torch.from_numpy(w2c).float().to(device),
+ K=torch.from_numpy(Kall[i]).float().to(device),
+ height=H,
+ width=W,
+ )
+ )
+ print(
+ f"[views] loaded {len(views)} ground-truth views (metadata cameras, photos from {img_dir})",
+ flush=True,
+ )
+ return views
+
+
+def _load_colmap_image_views(cfg, device: torch.device) -> list[View]:
+ """Legacy path: use raw COLMAP poses directly.
+
+ Only correct when the splat is in the *same* frame as the COLMAP
+ reconstruction (no training-time normalization). Used as a fallback when the
+ splat has no camera metadata.
+ """
+ sfm = pathlib.Path(cfg.sfm)
+ cam_path, img_path, img_dir = _find_colmap(sfm)
+ cameras = _read_colmap_cameras(cam_path)
+ images = sorted(_read_colmap_images(img_path), key=lambda d: d["name"])
+ if not images:
+ raise GeoLangSplatError(f"no images found in {img_path}")
+
+ n = min(cfg.n_views, len(images)) if cfg.n_views > 0 else len(images)
+ pick = np.unique(np.linspace(0, len(images) - 1, n).round().astype(int))
+
+ views: list[View] = []
+ for i in pick:
+ rec = images[i]
+ cam = cameras[rec["cam_id"]]
+ R = _qvec2rotmat(rec["qvec"])
+ w2c = np.eye(4, dtype=np.float64)
+ w2c[:3, :3] = R
+ w2c[:3, 3] = rec["tvec"]
+ K = _colmap_K(cam)
+ pil = Image.open(img_dir / rec["name"]).convert("RGB")
+ views.append(
+ View(
+ pil=pil,
+ w2c=torch.from_numpy(w2c).float().to(device),
+ K=torch.from_numpy(K).float().to(device),
+ height=cam["height"],
+ width=cam["width"],
+ )
+ )
+ print(f"[views] loaded {len(views)} ground-truth image views from {img_dir}", flush=True)
+ return views
+
+
+def load_image_views(cfg, device: torch.device, metadata: dict | None = None) -> list[View]:
+ """Load ground-truth photo views (``view_source='images'``).
+
+ Prefers the splat's own training cameras from ``metadata`` (which are in the
+ splat's frame); falls back to raw COLMAP poses when no camera metadata exists.
+ """
+ if not cfg.sfm:
+ raise GeoLangSplatError(
+ "view-source 'images' (the 'aerial' recipe) needs the scene's source photos: "
+ "pass --sfm (the COLMAP folder with sparse/ and images/). "
+ "If you have no photos, use --view-source render instead."
+ )
+ sfm = pathlib.Path(cfg.sfm)
+ if not sfm.is_dir():
+ raise GeoLangSplatError(f"--sfm path does not exist or is not a directory: {sfm}")
+ if metadata and "camera_to_world_matrices" in metadata:
+ return _load_metadata_image_views(cfg, device, metadata)
+ return _load_colmap_image_views(cfg, device)
+
+
+def generate_views(model, cfg, device: torch.device, metadata: dict | None = None) -> list[View]:
+ """Dispatch on ``cfg.view_source`` to produce the view list.
+
+ ``render`` synthesizes an orbit/ladder from scene geometry; ``globe`` synthesizes
+ an object/interior close dome; ``images`` loads the ground-truth SfM photos.
+ """
+ # Explicit inside-out wins (the user asked for interior coverage), regardless of source.
+ if getattr(cfg, "inside_out", False):
+ return render_inside_out_views(model, cfg, device)
+ if cfg.view_source == "globe":
+ return render_globe_views(model, cfg, device)
+ if cfg.view_source == "render":
+ return render_orbit_views(model, cfg, device)
+ if cfg.view_source == "images":
+ return load_image_views(cfg, device, metadata)
+ raise ValueError(f"unknown view_source {cfg.view_source!r} (expected 'render', 'globe', or 'images')")
diff --git a/open_vocabulary_segmentation/geolangsplat/pyproject.toml b/open_vocabulary_segmentation/geolangsplat/pyproject.toml
new file mode 100644
index 0000000..fc78a64
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/pyproject.toml
@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fvdb-geolangsplat"
+version = "0.0.1"
+description = "Training-free, open-vocabulary 3D segmentation for Gaussian splats"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Amrik Krishnakumar" }]
+
+dependencies = [
+ "fvdb-reality-capture",
+ "numpy",
+ "pandas",
+ "pillow",
+ "torch",
+ "tyro",
+]
+
+# SAM3 is the promptable 2D segmenter used at query time. It is not on PyPI;
+# install it from its source checkout (see README) and make it importable.
+
+[project.optional-dependencies]
+test = ["pytest"]
+notebook = ["ipywidgets"] # interactive SegmentCatalog.browse() picker
+
+[project.scripts]
+gls = "geolangsplat.cli:gls"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["geolangsplat*"]
diff --git a/open_vocabulary_segmentation/geolangsplat/requirements.txt b/open_vocabulary_segmentation/geolangsplat/requirements.txt
new file mode 100644
index 0000000..dc40765
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/requirements.txt
@@ -0,0 +1,10 @@
+# Lightweight dependencies, safe to pip/uv-install into your fvdb environment.
+#
+# NOT listed here (they come from the prebuilt fvdb environment, see README "Setup"):
+# - torch (built/installed with fvdb)
+# - fvdb / fvdb-reality-capture (the fvdb monorepo build; not on PyPI)
+# - sam3 (install from its source checkout; query-time only)
+numpy
+pandas
+pillow
+tyro
diff --git a/open_vocabulary_segmentation/geolangsplat/setup.sh b/open_vocabulary_segmentation/geolangsplat/setup.sh
new file mode 100644
index 0000000..4942b35
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/setup.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+# One-shot setup for GeoLangSplat (e.g. when baking a DLI lab image).
+#
+# Run ONCE at image-build time, from inside the env that already has fvdb + torch +
+# fvdb-reality-capture (the same env the reconstruction / mesh-extraction lab uses).
+# It downloads the SAM3 weights, installs GeoLangSplat, and runs the preflight so the
+# image ships ready -- the lab notebook then only needs to `import` and run `gls doctor`.
+#
+# Configure via env vars (only the URL is site-specific):
+# GEOLANGSPLAT_SAM_URL download URL for the SAM3 checkpoint (required to auto-download)
+# GEOLANGSPLAT_SAM_CKPT where to place / find the checkpoint
+# (default: ./sam3_ckpt/sam3.1_multiplex.pt)
+# GEOLANGSPLAT_SAM_REPO optional git URL for the SAM3 package (pip-installed if `sam3`
+# is not already importable)
+#
+# Usage:
+# GEOLANGSPLAT_SAM_URL=https://.../sam3.1_multiplex.pt ./setup.sh
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CKPT="${GEOLANGSPLAT_SAM_CKPT:-${HERE}/sam3_ckpt/sam3.1_multiplex.pt}"
+
+# --- 1. SAM3 weights -------------------------------------------------------
+# Large + gated, so they must live on disk before the lab. Download once if a URL
+# is given and the file isn't already there.
+if [[ -f "${CKPT}" ]]; then
+ echo "[setup] SAM3 checkpoint already present: ${CKPT}"
+elif [[ -n "${GEOLANGSPLAT_SAM_URL:-}" ]]; then
+ echo "[setup] downloading SAM3 checkpoint -> ${CKPT}"
+ mkdir -p "$(dirname "${CKPT}")"
+ if command -v curl >/dev/null 2>&1; then
+ curl -fL --retry 3 -o "${CKPT}" "${GEOLANGSPLAT_SAM_URL}"
+ elif command -v wget >/dev/null 2>&1; then
+ wget -O "${CKPT}" "${GEOLANGSPLAT_SAM_URL}"
+ else
+ echo "[setup] FATAL: neither curl nor wget available to download the checkpoint" >&2
+ exit 1
+ fi
+else
+ echo "[setup] no checkpoint at ${CKPT} and GEOLANGSPLAT_SAM_URL is unset --"
+ echo "[setup] set GEOLANGSPLAT_SAM_URL to auto-download, or copy the weights into place."
+fi
+export GEOLANGSPLAT_SAM_CKPT="${CKPT}"
+
+# --- 2. SAM3 package -------------------------------------------------------
+# Not on PyPI. Install from source if a repo URL is given and it isn't importable yet.
+if python -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('sam3') else 1)"; then
+ echo "[setup] sam3 package already importable"
+elif [[ -n "${GEOLANGSPLAT_SAM_REPO:-}" ]]; then
+ echo "[setup] installing SAM3 package from ${GEOLANGSPLAT_SAM_REPO}"
+ pip install "git+${GEOLANGSPLAT_SAM_REPO}"
+else
+ echo "[setup] sam3 not importable and GEOLANGSPLAT_SAM_REPO unset -- install SAM3 from source into the image."
+fi
+
+# --- 3. GeoLangSplat (light) ----------------------------------------------
+echo "[setup] installing GeoLangSplat (editable) from ${HERE}"
+pip install -e "${HERE}"
+
+# --- 4. Preflight ----------------------------------------------------------
+echo "[setup] preflight:"
+gls doctor || echo "[setup] doctor reported issues above -- fix them in the image before shipping the lab."
+
+echo "[setup] done. Persist GEOLANGSPLAT_SAM_CKPT=${CKPT} in the image env."
+echo "[setup] In the notebook, the first cell only needs: import geolangsplat + gls doctor (no installs)."
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/conftest.py b/open_vocabulary_segmentation/geolangsplat/tests/conftest.py
new file mode 100644
index 0000000..136f8e6
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/conftest.py
@@ -0,0 +1,62 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Shared test fixtures: CPU-only fakes so the suite needs no GPU/SAM3/fVDB."""
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+
+from geolangsplat.config import GeoLangSplatConfig
+
+
+class FakeModel:
+ """Stand-in for GaussianSplat3d holding only the tensors the writers touch."""
+
+ def __init__(self, n: int = 12, seed: int = 0):
+ g = torch.Generator().manual_seed(seed)
+ self.means = torch.rand(n, 3, generator=g)
+ self.quats = torch.rand(n, 4, generator=g)
+ self.log_scales = torch.rand(n, 3, generator=g)
+ self.logit_opacities = torch.rand(n, 1, generator=g)
+ self.sh0 = torch.rand(n, 1, 3, generator=g)
+ self.shN = torch.rand(n, 15, 3, generator=g)
+
+
+class FakeEngine:
+ """Minimal engine satisfying api.segment's contract, fully deterministic."""
+
+ def __init__(self, cfg, model, n: int = 12, seed: int = 0):
+ self.cfg = cfg
+ self.model = model
+ self.N = n
+ g = torch.Generator().manual_seed(seed)
+ self._scores = torch.rand(n, generator=g)
+ self.denom = torch.ones(n)
+ self.seen = torch.ones(n)
+
+ def query(self, prompt, *, select=None, margin=None, compete=None):
+ thr = self.cfg.select if select is None else select
+ sel = self._scores >= thr
+ return self._scores.clone(), sel, 0.001
+
+ def bake_vocab(self, vocab):
+ g = torch.Generator().manual_seed(123)
+ return torch.rand(self.N, len(vocab), generator=g)
+
+
+@pytest.fixture
+def cfg():
+ return GeoLangSplatConfig(device="cpu")
+
+
+@pytest.fixture
+def fake_model():
+ return FakeModel(n=12)
+
+
+@pytest.fixture
+def fake_engine(cfg, fake_model):
+ return FakeEngine(cfg, fake_model, n=12)
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_api.py b/open_vocabulary_segmentation/geolangsplat/tests/test_api.py
new file mode 100644
index 0000000..e6fc3c1
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_api.py
@@ -0,0 +1,54 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""End-to-end API tests using an injected fake engine (no GPU/SAM3/fVDB)."""
+from __future__ import annotations
+
+import torch
+
+from geolangsplat.api import segment
+
+
+def test_single_prompt_returns_mask(fake_engine):
+ res = segment(None, "house", engine=fake_engine)
+ assert res.label_ids is None
+ assert res.scores.shape == (fake_engine.N,)
+ assert res.selected.dtype == torch.bool
+ assert res.num_selected == int(res.selected.sum())
+ assert "query_seconds" in res.stats
+
+
+def test_multi_prompt_returns_labels(fake_engine):
+ res = segment(None, ["house", "tree", "grass"], engine=fake_engine)
+ assert res.label_ids is not None
+ assert res.scores.shape == (fake_engine.N, 3)
+ assert bool((res.selected == (res.label_ids >= 0)).all())
+ assert "labeled" in res.stats
+
+
+def test_determinism(fake_engine):
+ a = segment(None, "house", engine=fake_engine)
+ b = segment(None, "house", engine=fake_engine)
+ assert torch.equal(a.scores, b.scores)
+ assert torch.equal(a.selected, b.selected)
+
+
+def test_report_output_writes_file(fake_engine, tmp_path):
+ res = segment(None, "house", engine=fake_engine)
+ res.to_report(tmp_path / "out.csv")
+ lines = (tmp_path / "out.csv").read_text().strip().splitlines()
+ assert len(lines) == fake_engine.N + 1 # header + one row per gaussian, in order
+
+
+def test_empty_prompt_raises(fake_engine):
+ import pytest
+
+ with pytest.raises(ValueError):
+ segment(None, [], engine=fake_engine)
+
+
+def test_output_requires_out_path(fake_engine):
+ import pytest
+
+ with pytest.raises(ValueError):
+ segment(None, "house", engine=fake_engine, output="ply_overlay")
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_assess.py b/open_vocabulary_segmentation/geolangsplat/tests/test_assess.py
new file mode 100644
index 0000000..eefbd68
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_assess.py
@@ -0,0 +1,48 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import torch
+
+from geolangsplat.config import GeoLangSplatConfig
+from geolangsplat.engine import GeoLangSplatEngine
+from geolangsplat.cli._common import read_vocab_file
+
+
+def _engine_with(seen, denom, n_views=10):
+ cfg = GeoLangSplatConfig(device="cpu", min_weight=0.03)
+
+ class _M:
+ means = torch.zeros(seen.shape[0], 3)
+
+ eng = GeoLangSplatEngine(_M(), cfg, build=False)
+ eng.seen = seen
+ eng.denom = denom
+ eng.views = list(range(n_views))
+ return eng
+
+
+def test_assess_good_scene():
+ n = 100
+ seen = torch.full((n,), 5.0)
+ denom = torch.full((n,), 1.0)
+ rep = _engine_with(seen, denom).assess()
+ assert rep["verdict"] == "good"
+ assert rep["coverage"] == 1.0
+ assert rep["gaussians"] == n
+
+
+def test_assess_poor_scene():
+ n = 100
+ seen = torch.zeros(n)
+ seen[:20] = 1.0 # only 20% observed, once each
+ denom = torch.zeros(n)
+ denom[:20] = 0.001
+ rep = _engine_with(seen, denom).assess()
+ assert rep["verdict"] == "poor"
+ assert abs(rep["coverage"] - 0.2) < 1e-5
+
+
+def test_read_vocab_file(tmp_path):
+ p = tmp_path / "classes.txt"
+ p.write_text("house\n# a comment\ntree \n\nfire hydrant # inline\n")
+ assert read_vocab_file(p) == ["house", "tree", "fire hydrant"]
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_autoview.py b/open_vocabulary_segmentation/geolangsplat/tests/test_autoview.py
new file mode 100644
index 0000000..59aa415
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_autoview.py
@@ -0,0 +1,108 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Geometry-driven view planning + up-axis estimation (CPU-only, no SAM3/GPU)."""
+from __future__ import annotations
+
+import numpy as np
+import torch
+
+from geolangsplat import autoview, cameras
+
+
+def _ground_scene(seed: int = 0) -> torch.Tensor:
+ """A flat z=0 ground slab with some structure above it (gravity = +z)."""
+ rng = np.random.default_rng(seed)
+ ground = np.c_[rng.uniform(-10, 10, (8000, 2)), np.zeros(8000)]
+ above = np.c_[rng.uniform(-10, 10, (2000, 2)), rng.uniform(0.1, 3.0, 2000)]
+ return torch.tensor(np.r_[ground, above], dtype=torch.float32)
+
+
+def test_estimate_up_finds_plus_z_on_ground_scene():
+ vec, conf = autoview.estimate_up(_ground_scene())
+ assert conf > 0.3
+ assert abs(float(vec[2])) > 0.9 and float(vec[2]) > 0 # points up, away from the slab
+
+
+def _tile_scene(flip: bool = False, seed: int = 0) -> torch.Tensor:
+ """A satellite-tile-like scene: a big dense ground slab, buildings rising above
+ it, and some sub-ground floaters/terrain below (the case that flipped JAX_264).
+ With ``flip`` the whole thing is mirrored in z, so a correct estimator must still
+ return up pointing AWAY from the dense ground (toward the structures)."""
+ rng = np.random.default_rng(seed)
+ ground = np.c_[rng.uniform(-400, 400, (20000, 2)), rng.normal(0, 0.5, 20000)]
+ bldg = np.c_[rng.uniform(-400, 400, (6000, 2)), rng.uniform(2, 100, 6000)]
+ sub = np.c_[rng.uniform(-400, 400, (4000, 2)), rng.uniform(-60, -5, 4000)] # below-ground mass
+ pts = np.r_[ground, bldg, sub].astype(np.float32)
+ if flip:
+ pts[:, 2] = -pts[:, 2]
+ return torch.tensor(pts)
+
+
+def test_estimate_up_sign_robust_to_subground_mass():
+ """Ground at the bottom -> up = +z; mirrored -> up = -z. The estimate must follow
+ the structures (away from the dense ground), not be tipped by below-ground mass."""
+ vec, conf = autoview.estimate_up(_tile_scene(flip=False))
+ assert conf > 0.3 and float(vec[2]) > 0.9
+ vecf, conff = autoview.estimate_up(_tile_scene(flip=True))
+ assert conff > 0.3 and float(vecf[2]) < -0.9
+
+
+def test_estimate_up_sign_stable_across_similar_tiles():
+ """Two near-identical tiles must resolve to the SAME up sign (the JAX_264 vs
+ JAX_175 disagreement that put one orbit underneath the scene)."""
+ z = [float(autoview.estimate_up(_tile_scene(seed=s))[0][2]) for s in (1, 2, 3)]
+ assert all(v > 0.9 for v in z), z
+
+
+def test_resolve_up_honours_explicit_axis():
+ assert np.allclose(autoview.resolve_up("+y", _ground_scene()), [0, 1, 0])
+
+
+def test_resolve_up_falls_back_to_z_on_blob():
+ rng = np.random.default_rng(1)
+ blob = torch.tensor(rng.normal(0, 1, (5000, 3)), dtype=torch.float32)
+ assert np.allclose(autoview.resolve_up("auto", blob), [0, 0, 1])
+
+
+def test_recommend_aerial_for_flat_scene():
+ stats = autoview.measure_geometry(_ground_scene())
+ assert autoview.detect_capture(stats) == "aerial"
+ rec = autoview.recommend_view_config(stats, budget=200)
+ assert rec["capture"] == "aerial"
+ assert rec["n_views_planned"] > 0
+ assert len(rec["tiers"]) == 2 # overview + oblique
+
+
+def test_recommend_object_for_isotropic_scene():
+ rng = np.random.default_rng(2)
+ sphere = torch.tensor(rng.normal(0, 1, (5000, 3)), dtype=torch.float32)
+ stats = autoview.measure_geometry(sphere)
+ assert autoview.detect_capture(stats) == "object"
+ rec = autoview.recommend_view_config(stats, budget=120)
+ assert rec["capture"] == "object"
+ assert rec["view_source"] == "globe" # objects use the auto-framed globe orbit
+
+
+def test_globe_rings_dome_looks_down_and_hits_budget():
+ elevs, az = autoview.globe_rings(200)
+ assert min(elevs) > 0 and max(elevs) > 60 # upper dome: all rings look down
+ assert len(elevs) == len(az)
+ assert all(c >= 3 for c in az)
+ assert 0.6 * 200 <= sum(az) <= 1.4 * 200 # roughly hits the requested budget
+ # lower (more oblique) rings see more scene -> get more azimuths than the cap
+ assert az[0] >= az[-1]
+
+
+def test_inside_out_eye_at_center_and_looks_outward():
+ center = np.array([1.0, 2.0, 3.0])
+ cams = cameras.inside_out_cameras(center, (-25, 0, 25), 6, up="+z")
+ assert len(cams) == 18
+ w2c, _c2w = cams[0]
+ eye = -np.linalg.inv(w2c[:3, :3]) @ w2c[:3, 3]
+ assert np.allclose(eye, center, atol=1e-6)
+
+
+def test_up_vector_named_axes():
+ assert np.allclose(cameras.up_vector("-x"), [-1, 0, 0])
+ assert np.allclose(cameras.up_vector("auto"), [0, 0, 1])
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_cameras.py b/open_vocabulary_segmentation/geolangsplat/tests/test_cameras.py
new file mode 100644
index 0000000..a0349b6
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_cameras.py
@@ -0,0 +1,42 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import numpy as np
+import torch
+
+from geolangsplat.cameras import intrinsics, orbit_cameras, project
+
+
+def test_intrinsics_principal_point_and_focal():
+ K = intrinsics(60.0, 640, 480)
+ assert K.shape == (3, 3)
+ assert K[0, 2] == 320 and K[1, 2] == 240
+ f = 0.5 * 640 / np.tan(np.deg2rad(30.0))
+ assert np.isclose(K[0, 0], f) and np.isclose(K[1, 1], f)
+
+
+def test_orbit_camera_extrinsic_orthonormal():
+ ((w2c, c2w),) = orbit_cameras([0, 0, 0], elev=90.0, radius=10.0, num_azimuth=1)
+ R = w2c[:3, :3]
+ assert np.allclose(R @ R.T, np.eye(3), atol=1e-6)
+ assert np.allclose(c2w @ w2c, np.eye(4), atol=1e-6)
+
+
+def test_orbit_camera_looks_at_center():
+ center = np.array([1.0, 2.0, 0.0])
+ ((w2c, _),) = orbit_cameras(center, elev=70.0, radius=20.0, num_azimuth=1)
+ f, cx, cy = 100.0, 64.0, 64.0
+ u, v, z = project(
+ torch.tensor(center, dtype=torch.float32).view(1, 3),
+ torch.tensor(w2c, dtype=torch.float32),
+ f,
+ cx,
+ cy,
+ )
+ assert float(z) > 0 # center is in front of the camera
+ assert abs(float(u) - cx) < 1e-2 and abs(float(v) - cy) < 1e-2 # projects to image center
+
+
+def test_orbit_num_azimuth_count():
+ cams = orbit_cameras([0, 0, 0], elev=45.0, radius=5.0, num_azimuth=4)
+ assert len(cams) == 4
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_catalog.py b/open_vocabulary_segmentation/geolangsplat/tests/test_catalog.py
new file mode 100644
index 0000000..62a116f
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_catalog.py
@@ -0,0 +1,209 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""CPU tests for the multi-prompt segment catalog (clustering + dedup + table)."""
+from __future__ import annotations
+
+import importlib.util
+
+import pytest
+import torch
+
+import geolangsplat.outputs as out
+from geolangsplat.catalog import COLUMNS, SegmentCatalog, catalog_from_scores
+from geolangsplat.config import GeoLangSplatConfig
+
+
+class _Model:
+ """Minimal GaussianSplat3d stand-in: the tensors the writers / catalog touch."""
+
+ def __init__(self, means: torch.Tensor):
+ n = means.shape[0]
+ g = torch.Generator().manual_seed(0)
+ self.means = means
+ self.quats = torch.rand(n, 4, generator=g)
+ self.log_scales = torch.rand(n, 3, generator=g)
+ self.logit_opacities = torch.rand(n, 1, generator=g)
+ self.sh0 = torch.rand(n, 1, 3, generator=g)
+ self.shN = torch.rand(n, 15, 3, generator=g)
+
+
+class _FakeGS:
+ """Captures the tensors passed to from_tensors (mirrors test_outputs/instances)."""
+
+ last = None
+
+ def __init__(self, **kw):
+ self.kw = kw
+
+ @classmethod
+ def from_tensors(cls, **kw):
+ inst = cls(**kw)
+ _FakeGS.last = inst
+ return inst
+
+ def save_ply(self, path):
+ with open(path, "w") as f:
+ f.write(f"n={self.kw['means'].shape[0]}\n")
+
+
+def _scene(n_a: int = 100, n_b: int = 60, seed: int = 0):
+ """Two well-separated spatial clusters: A near the origin, B ~50 units away."""
+ g = torch.Generator().manual_seed(seed)
+ a = torch.rand(n_a, 3, generator=g)
+ b = torch.rand(n_b, 3, generator=g) + 50.0
+ return torch.cat([a, b], dim=0), n_a, n_b
+
+
+def _scores_car_vehicle_tree(n_a: int, n_b: int) -> torch.Tensor:
+ """[N, 3] columns: car & vehicle both fire on A (overlap), tree fires on B."""
+ n = n_a + n_b
+ s = torch.zeros(n, 3)
+ s[:n_a, 0] = 0.9 # car -> A
+ s[:n_a, 1] = 0.9 # vehicle -> A (should merge with car)
+ s[n_a:, 2] = 0.9 # tree -> B
+ return s
+
+
+def _cfg():
+ # clean3d off so selection is a pure threshold (keeps the test deterministic).
+ return GeoLangSplatConfig(device="cpu", clean3d=False, inst_link_frac=0.02, inst_min_size=25)
+
+
+def _build(cfg=None, iou=None):
+ means, n_a, n_b = _scene()
+ model = _Model(means)
+ scores = _scores_car_vehicle_tree(n_a, n_b)
+ cat = catalog_from_scores(
+ model, ["car", "vehicle", "tree"], cfg or _cfg(), scores, seen=torch.ones(means.shape[0]), iou=iou
+ )
+ return cat, n_a, n_b
+
+
+def test_catalog_merges_overlapping_prompts():
+ cat, n_a, n_b = _build()
+ assert len(cat) == 2 # {car, vehicle} -> one object; tree -> one object
+ big, small = cat[0], cat[1]
+ assert big.n_gaussians == n_a and small.n_gaussians == n_b # largest first
+ assert big.prompts == ["car", "vehicle"] # both prompts recorded, sorted
+ assert big.label in ("car", "vehicle")
+ assert small.label == "tree" and small.prompts == ["tree"]
+
+
+def test_catalog_no_merge_when_iou_high():
+ # An impossible IoU floor keeps car and vehicle as separate objects.
+ cat, n_a, n_b = _build(iou=1.1)
+ assert len(cat) == 3
+ assert sorted(o.label for o in cat) == ["car", "tree", "vehicle"]
+
+
+def test_labels_and_masks_partition_the_scene():
+ cat, n_a, n_b = _build()
+ assert int(cat.all_mask.sum()) == n_a + n_b
+ assert int(cat.mask(0).sum()) == n_a
+ assert int(cat.mask(1).sum()) == n_b
+ # labels are in input order: first n_a -> object 0, rest -> object 1
+ assert int((cat.labels[:n_a] == 0).sum()) == n_a
+ assert int((cat.labels[n_a:] == 1).sum()) == n_b
+ with pytest.raises(KeyError):
+ cat.mask(99)
+
+
+def test_table_columns_and_rows():
+ cat, _, _ = _build()
+ rows = cat.rows()
+ assert len(rows) == 2
+ assert list(rows[0].keys()) == list(COLUMNS)
+ tab = cat.table
+ if importlib.util.find_spec("pandas") is not None:
+ assert list(tab.columns) == list(COLUMNS)
+ assert len(tab) == 2
+ else:
+ assert tab == rows
+
+
+def test_extract_writes_only_that_object(monkeypatch, tmp_path):
+ cat, n_a, _ = _build()
+ monkeypatch.setattr(out, "_gs", lambda: _FakeGS)
+ written = cat.extract(0, tmp_path / "obj0.ply")
+ assert written == n_a
+ assert _FakeGS.last.kw["means"].shape[0] == n_a
+
+
+def test_export_all_writes_folder(monkeypatch, tmp_path):
+ cat, _, _ = _build()
+ monkeypatch.setattr(out, "_gs", lambda: _FakeGS)
+ out_dir = cat.export_all(tmp_path / "cat", labeled_ply=True)
+ assert (out_dir / "catalog.csv").exists()
+ assert (out_dir / "catalog_labeled.ply").exists()
+ plys = list((out_dir / "objects").glob("*.ply"))
+ assert len(plys) == 2
+
+
+def test_save_load_round_trip(tmp_path):
+ cat, _, _ = _build()
+ cat.save(tmp_path / "saved")
+ model = cat._model
+ reloaded = SegmentCatalog.load(tmp_path / "saved", model)
+ assert len(reloaded) == len(cat)
+ assert [o.label for o in reloaded] == [o.label for o in cat]
+ assert [o.n_gaussians for o in reloaded] == [o.n_gaussians for o in cat]
+ assert torch.equal(reloaded.labels, cat.labels)
+
+
+def test_ids_mask_unions_selected_objects():
+ cat, n_a, n_b = _build()
+ assert int(cat._ids_mask([0]).sum()) == n_a
+ assert int(cat._ids_mask([0, 1]).sum()) == n_a + n_b
+ assert int(cat._ids_mask([]).sum()) == 0
+
+
+def test_browse_without_ipywidgets_is_clear(monkeypatch):
+ import builtins
+
+ cat, _, _ = _build()
+ real_import = builtins.__import__
+
+ def _no_ipywidgets(name, *a, **k):
+ if name == "ipywidgets":
+ raise ModuleNotFoundError("No module named 'ipywidgets'")
+ return real_import(name, *a, **k)
+
+ monkeypatch.setattr(builtins, "__import__", _no_ipywidgets)
+ with pytest.raises(RuntimeError, match="ipywidgets"):
+ cat.browse()
+
+
+def test_robust_span_splits_objects_despite_floaters():
+ # Two clusters 4 units apart + one far floater. The naive min/max span would be
+ # ~1e4 -> a huge voxel that merges both; the quantile span tracks the real scene.
+ g = torch.Generator().manual_seed(0)
+ a = torch.rand(100, 3, generator=g)
+ b = torch.rand(100, 3, generator=g) + 5.0
+ floater = torch.full((1, 3), 1e4)
+ means = torch.cat([a, b, floater], dim=0)
+
+ class M:
+ def __init__(s):
+ n = means.shape[0]
+ gg = torch.Generator().manual_seed(1)
+ s.means = means
+ s.quats = torch.rand(n, 4, generator=gg)
+ s.log_scales = torch.rand(n, 3, generator=gg)
+ s.logit_opacities = torch.rand(n, 1, generator=gg)
+ s.sh0 = torch.rand(n, 1, 3, generator=gg)
+ s.shN = torch.rand(n, 15, 3, generator=gg)
+
+ scores = torch.zeros(means.shape[0], 1)
+ scores[:200, 0] = 0.9 # select the two clusters, not the floater
+ cat = catalog_from_scores(M(), ["building"], _cfg(), scores, seen=torch.ones(means.shape[0]))
+ assert len(cat) == 2 # robust span keeps the two buildings apart
+
+
+def test_empty_vocab_selection_yields_empty_catalog():
+ means, _, _ = _scene()
+ model = _Model(means)
+ scores = torch.zeros(means.shape[0], 2) # nothing clears the threshold
+ cat = catalog_from_scores(model, ["car", "tree"], _cfg(), scores, seen=torch.ones(means.shape[0]))
+ assert len(cat) == 0
+ assert int((cat.labels == -1).sum()) == means.shape[0]
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_cli.py b/open_vocabulary_segmentation/geolangsplat/tests/test_cli.py
new file mode 100644
index 0000000..0886b0c
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_cli.py
@@ -0,0 +1,112 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import tyro
+
+from geolangsplat.cli import (
+ Bake,
+ Check,
+ Doctor,
+ Explore,
+ Render,
+ Segment,
+ Serve,
+ Show,
+ Status,
+ Stop,
+)
+
+# Mirror the real `gls()` union so the test fails if a command is dropped from dispatch.
+_UNION = Segment | Bake | Check | Doctor | Serve | Show | Explore | Render | Status | Stop
+
+
+def test_segment_minimal_parses():
+ s = tyro.cli(_UNION, args=["segment", "m.ply", "house", "-r", "satellite"])
+ assert type(s).__name__ == "Segment"
+ assert s.prompt == "house" and s.recipe == "satellite"
+
+
+def test_segment_output_parses():
+ s = tyro.cli(_UNION, args=["segment", "m.ply", "road", "-O", "ply_overlay", "-o", "r.ply", "-t", "0.4"])
+ assert type(s).__name__ == "Segment"
+ assert s.output == "ply_overlay" and s.select == 0.4
+ assert not hasattr(s, "warm") # engine lifecycle lives in serve/stop, not segment
+
+
+def test_serve_background_and_keep_alive_parses():
+ s = tyro.cli(_UNION, args=["serve", "m.ply", "-r", "satellite", "-b", "--keep-alive", "15"])
+ assert type(s).__name__ == "Serve"
+ assert s.recipe == "satellite" and s.background is True and s.keep_alive == 15.0
+
+
+def test_serve_keep_alive_pin_parses():
+ s = tyro.cli(_UNION, args=["serve", "m.ply", "--keep-alive", "0"])
+ assert type(s).__name__ == "Serve" and s.keep_alive == 0.0
+
+
+def test_serve_socket_path_parses(): # the warm-spawn path passes this through
+ s = tyro.cli(_UNION, args=["serve", "m.ply", "--socket-path", "/tmp/x.sock"])
+ assert type(s).__name__ == "Serve" and s.socket_path == "/tmp/x.sock"
+
+
+def test_show_parses():
+ s = tyro.cli(_UNION, args=["show", "out.ply", "-p", "8090"])
+ assert type(s).__name__ == "Show"
+ assert str(s.model_path) == "out.ply" and s.viewer_port == 8090
+
+
+def test_stop_parses():
+ s = tyro.cli(_UNION, args=["stop", "m.ply"])
+ assert type(s).__name__ == "Stop"
+ assert str(s.model_path) == "m.ply"
+
+
+def test_stop_all_parses(): # no model -> stop everything
+ s = tyro.cli(_UNION, args=["stop"])
+ assert type(s).__name__ == "Stop"
+ assert s.model_path is None
+
+
+def test_status_parses():
+ s = tyro.cli(_UNION, args=["status"])
+ assert type(s).__name__ == "Status"
+
+
+def test_segment_low_vram_on_by_default():
+ s = tyro.cli(_UNION, args=["segment", "m.ply", "house"])
+ assert s.low_vram is True # one-shot defaults to bounded VRAM
+ assert s.stream_early_stop == "auto" # capture-gated unless forced
+ assert s.stream_chunk is None # falls through to the config default
+
+
+def test_segment_streaming_flags_parse():
+ s = tyro.cli(
+ _UNION,
+ args=[
+ "segment",
+ "m.ply",
+ "tree",
+ "--no-low-vram",
+ "--stream-early-stop",
+ "off",
+ "--stream-chunk",
+ "16",
+ ],
+ )
+ assert s.low_vram is False
+ assert s.stream_early_stop == "off" and s.stream_chunk == 16
+
+
+def test_segment_streaming_flags_wire_into_config():
+ # build_config must forward the streaming knobs the one-shot path reads.
+ from geolangsplat.cli._common import build_config
+
+ cfg = build_config(low_vram=True, stream_early_stop="on", stream_chunk=4)
+ assert cfg.low_vram is True
+ assert cfg.stream_early_stop == "on" and cfg.stream_chunk == 4
+
+
+def test_doctor_render_explore_parse():
+ assert type(tyro.cli(_UNION, args=["doctor"])).__name__ == "Doctor"
+ assert type(tyro.cli(_UNION, args=["render", "m.ply", "-o", "x.png"])).__name__ == "Render"
+ assert type(tyro.cli(_UNION, args=["explore", "m.ply"])).__name__ == "Explore"
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_config.py b/open_vocabulary_segmentation/geolangsplat/tests/test_config.py
new file mode 100644
index 0000000..9344f19
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_config.py
@@ -0,0 +1,83 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import typing
+
+import pytest
+
+from geolangsplat.config import (
+ RECIPES,
+ GeoLangSplatConfig,
+ RecipeName,
+ apply_recipe,
+ config_field_names,
+ list_recipes,
+)
+from geolangsplat.errors import GeoLangSplatError
+from geolangsplat.cli._common import build_config
+
+
+def test_recipe_name_matches_recipes():
+ # the CLI choice type must stay in sync with the actual recipe registry
+ assert set(typing.get_args(RecipeName)) == set(RECIPES)
+
+
+def test_recipes_exist():
+ names = list_recipes()
+ assert {"aerial", "satellite", "satellite_dense"} <= set(names)
+
+
+def test_competition_off_by_default():
+ cfg = GeoLangSplatConfig()
+ assert cfg.compete is False
+ assert len(cfg.distractors) > 0 # a generic competition set is still available
+
+
+def test_recipes_do_not_force_competition():
+ # recipes supply curated distractors but must not silently turn competition on
+ for name in list_recipes():
+ cfg = GeoLangSplatConfig()
+ apply_recipe(cfg, name)
+ assert cfg.compete is False
+
+
+def test_apply_recipe_fills_defaults():
+ cfg = GeoLangSplatConfig()
+ applied = apply_recipe(cfg, "satellite")
+ assert "select" in applied
+ assert cfg.select == 0.35
+ assert cfg.distractors == ("road", "grass", "tree", "water")
+
+
+def test_explicit_value_wins_over_preset():
+ cfg = GeoLangSplatConfig(select=0.99)
+ applied = apply_recipe(cfg, "satellite")
+ assert "select" not in applied # not overwritten
+ assert cfg.select == 0.99
+
+
+def test_apply_recipe_none_is_noop():
+ cfg = GeoLangSplatConfig()
+ assert apply_recipe(cfg, None) == []
+
+
+def test_unknown_recipe_raises():
+ with pytest.raises(GeoLangSplatError):
+ apply_recipe(GeoLangSplatConfig(), "nope")
+
+
+def test_unknown_recipe_suggests_closest():
+ with pytest.raises(GeoLangSplatError, match="did you mean 'satellite'"):
+ apply_recipe(GeoLangSplatConfig(), "satelite")
+
+
+def test_build_config_ignores_none():
+ cfg = build_config(select=None, margin=0.2, device="cpu")
+ assert cfg.select == GeoLangSplatConfig().select # untouched
+ assert cfg.margin == 0.2
+ assert cfg.device == "cpu"
+
+
+def test_config_field_names_nonempty():
+ names = config_field_names()
+ assert "select" in names and "view_source" in names and "lift" in names
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_instances.py b/open_vocabulary_segmentation/geolangsplat/tests/test_instances.py
new file mode 100644
index 0000000..61d7e97
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_instances.py
@@ -0,0 +1,142 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""CPU tests for the instance split (connected components + result API)."""
+from __future__ import annotations
+
+import numpy as np
+import pytest
+import torch
+
+import geolangsplat.outputs as out
+from geolangsplat.config import GeoLangSplatConfig
+from geolangsplat.instances import build_instances, connected_components
+
+
+class _Model:
+ """Minimal GaussianSplat3d stand-in: the tensors the writers/CC touch."""
+
+ def __init__(self, means: torch.Tensor):
+ n = means.shape[0]
+ g = torch.Generator().manual_seed(0)
+ self.means = means
+ self.quats = torch.rand(n, 4, generator=g)
+ self.log_scales = torch.rand(n, 3, generator=g)
+ self.logit_opacities = torch.rand(n, 1, generator=g)
+ self.sh0 = torch.rand(n, 1, 3, generator=g)
+ self.shN = torch.rand(n, 15, 3, generator=g)
+
+
+class _FakeGS:
+ """Captures the tensors passed to from_tensors (mirrors test_outputs)."""
+
+ last = None
+
+ def __init__(self, **kw):
+ self.kw = kw
+
+ @classmethod
+ def from_tensors(cls, **kw):
+ inst = cls(**kw)
+ _FakeGS.last = inst
+ return inst
+
+ def save_ply(self, path):
+ with open(path, "w") as f:
+ f.write(f"n={self.kw['means'].shape[0]}\n")
+
+
+def _two_clusters(n_a: int = 100, n_b: int = 60, n_speckle: int = 3, seed: int = 0):
+ """means with a big cluster near origin, a smaller far cluster, and a speckle.
+
+ The clusters are separated by a gap far larger than the voxel link size, so
+ connected components must split them; the speckle is below any sane min_size.
+ """
+ g = torch.Generator().manual_seed(seed)
+ a = torch.rand(n_a, 3, generator=g) # ~[0, 1]^3
+ b = torch.rand(n_b, 3, generator=g) + 50.0 # ~[50, 51]^3
+ speck = torch.full((n_speckle, 3), 100.0) # isolated, tiny
+ return torch.cat([a, b, speck], dim=0), n_a, n_b, n_speckle
+
+
+def test_connected_components_splits_and_ranks():
+ means, n_a, n_b, n_sp = _two_clusters()
+ sel = torch.ones(means.shape[0], dtype=torch.bool)
+ cfg = GeoLangSplatConfig(device="cpu", inst_link_frac=0.02, inst_min_size=25)
+
+ comp_ids, infos = connected_components(means, sel, cfg)
+
+ assert len(infos) == 2 # speckle dropped by min_size
+ assert [d["size"] for d in infos] == [n_a, n_b] # largest first
+ assert infos[0]["idx"] == 0 and infos[1]["idx"] == 1
+ # membership: cluster A -> 0, cluster B -> 1, speckle -> -1
+ assert int((comp_ids[:n_a] == 0).sum()) == n_a
+ assert int((comp_ids[n_a : n_a + n_b] == 1).sum()) == n_b
+ assert int((comp_ids[-n_sp:] == -1).sum()) == n_sp
+ # centroid of the big cluster sits near the origin box
+ assert np.all(infos[0]["centroid"] < 2.0)
+
+
+def test_min_size_keeps_speckle_when_low():
+ means, n_a, n_b, n_sp = _two_clusters()
+ sel = torch.ones(means.shape[0], dtype=torch.bool)
+ cfg = GeoLangSplatConfig(device="cpu", inst_link_frac=0.02, inst_min_size=1)
+ _comp, infos = connected_components(means, sel, cfg)
+ assert len(infos) == 3 # speckle now survives
+
+
+def test_link_frac_merges_when_coarse():
+ means, n_a, n_b, _ = _two_clusters()
+ sel = torch.ones(means.shape[0], dtype=torch.bool)
+ cfg = GeoLangSplatConfig(device="cpu")
+ # A voxel large enough to bridge the 50-unit gap collapses both into one blob.
+ _comp, infos = connected_components(means, sel, cfg, link_frac=1.0, min_size=1)
+ assert len(infos) == 1
+ assert infos[0]["size"] == means.shape[0]
+
+
+def test_explicit_span_overrides_naive_range():
+ # Clusters at 0 and 50; a far floater blows up the naive span and merges them,
+ # but passing a robust span keeps the split.
+ means, n_a, n_b, _ = _two_clusters()
+ means = torch.cat([means, torch.full((1, 3), 1e5)], dim=0)
+ sel = torch.zeros(means.shape[0], dtype=torch.bool)
+ sel[: n_a + n_b] = True # select the two clusters, not the floater
+ cfg = GeoLangSplatConfig(device="cpu", inst_link_frac=0.02, inst_min_size=25)
+
+ _c, merged = connected_components(means, sel, cfg) # naive span ~1e5 -> one blob
+ assert len(merged) == 1
+ _c, split = connected_components(means, sel, cfg, span=51.0) # real extent
+ assert len(split) == 2
+
+
+def test_empty_selection_yields_no_instances():
+ means, *_ = _two_clusters()
+ sel = torch.zeros(means.shape[0], dtype=torch.bool)
+ cfg = GeoLangSplatConfig(device="cpu")
+ comp_ids, infos = connected_components(means, sel, cfg)
+ assert infos == []
+ assert int((comp_ids == -1).sum()) == means.shape[0]
+
+
+def test_build_instances_result_and_extract(monkeypatch, tmp_path):
+ means, n_a, n_b, n_sp = _two_clusters()
+ model = _Model(means)
+ sel = torch.ones(means.shape[0], dtype=torch.bool)
+ cfg = GeoLangSplatConfig(device="cpu", inst_link_frac=0.02, inst_min_size=10)
+
+ res = build_instances(model, sel, cfg, "house")
+ assert len(res) == 2
+ assert res[0].size == n_a and res[1].size == n_b
+ assert "house" in repr(res)
+ assert int(res.mask(0).sum()) == n_a
+ assert int(res.all_mask.sum()) == n_a + n_b # speckle excluded
+
+ with pytest.raises(IndexError):
+ res.mask(5)
+
+ # extract one instance -> only its Gaussians are written
+ monkeypatch.setattr(out, "_gs", lambda: _FakeGS)
+ written = res.extract(0, tmp_path / "house0.ply")
+ assert written == n_a
+ assert _FakeGS.last.kw["means"].shape[0] == n_a
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_ipc.py b/open_vocabulary_segmentation/geolangsplat/tests/test_ipc.py
new file mode 100644
index 0000000..6b91482
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_ipc.py
@@ -0,0 +1,116 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import os
+import socket
+
+import geolangsplat.outputs as _out
+from geolangsplat.ipc import (
+ cleanup_stale,
+ daemon_alive,
+ default_socket,
+ handle_request,
+ kill_daemon,
+ pid_alive,
+ pidfile_for,
+ read_pid,
+ recv_obj,
+ remove_runtime_files,
+ send_obj,
+ write_pidfile,
+)
+
+
+class _FakeGS:
+ @classmethod
+ def from_tensors(cls, **kw):
+ inst = cls()
+ inst.kw = kw
+ return inst
+
+ def save_ply(self, path):
+ with open(path, "w") as f:
+ f.write(f"n={self.kw['means'].shape[0]}\n")
+
+
+def test_default_socket_stable_and_per_model():
+ a = default_socket("model.ply")
+ assert a == default_socket("model.ply") # stable
+ assert a != default_socket("other.ply") # per-model
+ assert a.endswith(".sock")
+
+
+def test_send_recv_roundtrip():
+ a, b = socket.socketpair()
+ send_obj(a, {"prompt": "house", "n": 3})
+ got = recv_obj(b)
+ assert got == {"prompt": "house", "n": 3}
+ a.close()
+ b.close()
+
+
+def test_handle_request_mask(fake_engine):
+ resp = handle_request(fake_engine, {"prompt": "house"})
+ assert resp["ok"] is True
+ assert resp["N"] == fake_engine.N
+ assert 0 <= resp["n"] <= fake_engine.N
+ assert resp["path"] is None
+
+
+def test_handle_request_empty_prompt(fake_engine):
+ resp = handle_request(fake_engine, {"prompt": " "})
+ assert resp["ok"] is False
+
+
+def test_handle_request_output_needs_path(fake_engine):
+ resp = handle_request(fake_engine, {"prompt": "house", "output": "ply_overlay"})
+ assert resp["ok"] is False
+
+
+def test_handle_request_writes_overlay(fake_engine, tmp_path, monkeypatch):
+ monkeypatch.setattr(_out, "_gs", lambda: _FakeGS)
+ out = tmp_path / "h.ply"
+ resp = handle_request(fake_engine, {"prompt": "house", "output": "ply_overlay", "out_path": str(out)})
+ assert resp["ok"] is True
+ assert resp["path"] == str(out)
+ assert out.exists()
+
+
+def test_handle_request_applies_select_override(fake_engine):
+ handle_request(fake_engine, {"prompt": "house", "select": 0.99})
+ assert fake_engine.cfg.select == 0.99
+
+
+def test_daemon_alive_false_when_nothing_listening(tmp_path):
+ assert daemon_alive(str(tmp_path / "nope.sock"), timeout=0.5) is False
+
+
+def test_pid_alive_self_and_bogus():
+ assert pid_alive(os.getpid()) is True
+ assert pid_alive(2_147_480_000) is False # almost certainly unused PID
+
+
+def test_pidfile_roundtrip_and_remove(tmp_path):
+ sock = str(tmp_path / "x.sock")
+ write_pidfile(sock, 4242)
+ assert os.path.exists(pidfile_for(sock))
+ assert read_pid(sock) == 4242
+ remove_runtime_files(sock)
+ assert not os.path.exists(pidfile_for(sock))
+ assert read_pid(sock) is None
+
+
+def test_kill_daemon_not_running_is_clean(tmp_path):
+ sock = str(tmp_path / "dead.sock")
+ write_pidfile(sock, 2_147_480_000) # stale pid, no process, no socket bound
+ assert kill_daemon(sock, grace=0.2) == "not running"
+ assert not os.path.exists(pidfile_for(sock))
+
+
+def test_cleanup_stale_removes_when_pid_dead(tmp_path):
+ sock = str(tmp_path / "stale.sock")
+ open(sock, "w").close() # leftover socket file, no listener
+ write_pidfile(sock, 2_147_480_000) # dead pid
+ assert cleanup_stale(sock) is True
+ assert not os.path.exists(sock)
+ assert read_pid(sock) is None
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_outputs.py b/open_vocabulary_segmentation/geolangsplat/tests/test_outputs.py
new file mode 100644
index 0000000..b9bc48d
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_outputs.py
@@ -0,0 +1,76 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import json
+
+import numpy as np
+import torch
+
+import geolangsplat.outputs as out
+
+
+class _FakeGS:
+ """Captures the tensors passed to from_tensors and 'saves' the count."""
+
+ last = None
+
+ def __init__(self, **kw):
+ self.kw = kw
+
+ @classmethod
+ def from_tensors(cls, **kw):
+ inst = cls(**kw)
+ _FakeGS.last = inst
+ return inst
+
+ def save_ply(self, path):
+ with open(path, "w") as f:
+ f.write(f"n={self.kw['means'].shape[0]}\n")
+
+
+def _patch_gs(monkeypatch):
+ monkeypatch.setattr(out, "_gs", lambda: _FakeGS)
+
+
+def test_write_ply_segmented_subsets(monkeypatch, fake_model, tmp_path):
+ _patch_gs(monkeypatch)
+ sel = torch.zeros(12, dtype=torch.bool)
+ sel[:5] = True
+ n = out.write_ply_segmented(fake_model, sel, tmp_path / "seg.ply")
+ assert n == 5
+ assert _FakeGS.last.kw["means"].shape[0] == 5 # only selected written
+
+
+def test_write_ply_overlay_preserves_count(monkeypatch, fake_model, tmp_path):
+ _patch_gs(monkeypatch)
+ sel = torch.zeros(12, dtype=torch.bool)
+ sel[:3] = True
+ out.write_ply_overlay(fake_model, sel, tmp_path / "ov.ply", color=(1, 0, 0), blend=0.5)
+ assert _FakeGS.last.kw["means"].shape[0] == 12 # full splat preserved
+ # selected sh0 changed, unselected unchanged
+ new_sh0 = _FakeGS.last.kw["sh0"]
+ assert not torch.allclose(new_sh0[:3], fake_model.sh0[:3])
+ assert torch.allclose(new_sh0[3:], fake_model.sh0[3:])
+
+
+def test_report_csv_order_and_legend(tmp_path):
+ means = np.arange(12 * 3).reshape(12, 3).astype(np.float32)
+ labels = np.array([0, 1, -1, 0, 1, 2, -1, 0, 1, 2, 0, 1])
+ scores = np.random.RandomState(0).rand(12)
+ out.write_report_csv(means, labels, scores, ["a", "b", "c"], tmp_path / "r.csv")
+ lines = (tmp_path / "r.csv").read_text().strip().splitlines()
+ assert lines[0] == "index,x,y,z,label_id,label,score"
+ assert len(lines) == 13 # header + 12 rows, in order
+ assert lines[1].startswith("0,") and lines[12].startswith("11,")
+ legend = json.loads((tmp_path / "legend.json").read_text())
+ assert legend["0"] == "a" and legend["-1"] == "unlabeled"
+
+
+def test_report_npz_roundtrip(tmp_path):
+ means = np.zeros((4, 3), dtype=np.float32)
+ labels = np.array([0, -1, 1, 0])
+ scores = np.zeros((4, 2), dtype=np.float32)
+ out.write_report_npz(means, labels, scores, ["a", "b"], tmp_path / "r.npz")
+ d = np.load(tmp_path / "r.npz", allow_pickle=True)
+ assert d["label_ids"].tolist() == [0, -1, 1, 0]
+ assert list(d["vocab"]) == ["a", "b"]
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_profile.py b/open_vocabulary_segmentation/geolangsplat/tests/test_profile.py
new file mode 100644
index 0000000..dda2aa3
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_profile.py
@@ -0,0 +1,27 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Unit tests for the step profiler (CPU-only)."""
+from __future__ import annotations
+
+import time
+
+from geolangsplat.profile import Stopwatch
+
+
+def test_stopwatch_records_and_reports():
+ sw = Stopwatch(device=None) # cpu: no cuda sync
+ with sw.span("a"):
+ time.sleep(0.01)
+ sw.add("b", 0.5)
+ names = [n for n, _ in sw.spans]
+ assert names == ["a", "b"]
+ assert sw.total() >= 0.5
+ rep = sw.report("build")
+ assert "build" in rep
+ assert "a" in rep and "b" in rep
+ assert "TOTAL" in rep
+
+
+def test_stopwatch_empty_report_is_blank():
+ assert Stopwatch(device=None).report() == ""
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_robustness.py b/open_vocabulary_segmentation/geolangsplat/tests/test_robustness.py
new file mode 100644
index 0000000..98645d6
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_robustness.py
@@ -0,0 +1,99 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Robustness/edge-case tests: device resolution, missing weights, metadata
+round-trip, amp fallback. All CPU-only (no GPU/SAM3/fvdb needed)."""
+from __future__ import annotations
+
+import pytest
+import torch
+
+import geolangsplat.outputs as out
+from geolangsplat.engine import _load_model, resolve_device
+from geolangsplat.errors import GeoLangSplatError
+from geolangsplat.sam3 import Sam3Scorer, _resolve_amp_dtype
+
+
+# -- device resolution ------------------------------------------------------
+
+
+def test_resolve_device_cpu_ok():
+ assert resolve_device("cpu") == torch.device("cpu")
+
+
+def test_resolve_device_cuda_without_gpu_raises():
+ if torch.cuda.is_available():
+ pytest.skip("CUDA present; cannot test the no-GPU path here")
+ with pytest.raises(GeoLangSplatError):
+ resolve_device("cuda")
+
+
+# -- model loading ----------------------------------------------------------
+
+
+def test_load_model_missing_file_raises():
+ with pytest.raises(GeoLangSplatError):
+ _load_model("/no/such/model.ply", torch.device("cpu"))
+
+
+def test_load_model_passthrough_for_object():
+ sentinel = object()
+ model, meta = _load_model(sentinel, torch.device("cpu"))
+ assert model is sentinel and meta == {}
+
+
+# -- SAM3 guards ------------------------------------------------------------
+
+
+def test_sam3_missing_checkpoint_raises():
+ with pytest.raises(GeoLangSplatError):
+ Sam3Scorer("/no/such/sam3.pt")
+
+
+def test_amp_dtype_fp16_and_default():
+ assert _resolve_amp_dtype("fp16", "cuda") == torch.float16
+ # on cpu (no cuda bf16 query) the default stays bf16
+ assert _resolve_amp_dtype("bf16", "cpu") == torch.bfloat16
+
+
+# -- metadata round-trip ----------------------------------------------------
+
+
+class _FakeGS:
+ last_metadata = "UNSET"
+
+ @classmethod
+ def from_tensors(cls, **kw):
+ inst = cls()
+ inst.kw = kw
+ return inst
+
+ def save_ply(self, path, metadata=None):
+ _FakeGS.last_metadata = metadata
+ with open(path, "w") as f:
+ f.write("ok\n")
+
+
+def test_writer_passes_metadata(monkeypatch, fake_model, tmp_path):
+ monkeypatch.setattr(out, "_gs", lambda: _FakeGS)
+ sel = torch.zeros(12, dtype=torch.bool)
+ sel[:3] = True
+ out.write_ply_overlay(fake_model, sel, tmp_path / "o.ply", metadata={"foo": "bar"})
+ assert _FakeGS.last_metadata == {"foo": "bar"}
+
+
+class _PickyGS(_FakeGS):
+ def save_ply(self, path, metadata=None):
+ if metadata: # backend rejects rich metadata -> writer must retry without it
+ raise ValueError("unsupported metadata")
+ with open(path, "w") as f:
+ f.write("ok\n")
+
+
+def test_writer_falls_back_when_metadata_rejected(monkeypatch, fake_model, tmp_path):
+ monkeypatch.setattr(out, "_gs", lambda: _PickyGS)
+ sel = torch.zeros(12, dtype=torch.bool)
+ sel[:2] = True
+ p = tmp_path / "o.ply"
+ out.write_ply_segmented(fake_model, sel, p, metadata={"nested": {"x": 1}})
+ assert p.exists() # written despite metadata rejection
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_sam3_fusion.py b/open_vocabulary_segmentation/geolangsplat/tests/test_sam3_fusion.py
new file mode 100644
index 0000000..273f6c9
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_sam3_fusion.py
@@ -0,0 +1,106 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Dual-head fusion math (instance + dense semantic) for the SAM3 scorer.
+
+These exercise ``Sam3Scorer``'s head-fusion helpers on synthetic grounding outputs
+so they run on CPU without loading SAM3 weights: we bypass ``__init__`` and set only
+the attributes the fusion path reads.
+"""
+from types import SimpleNamespace
+
+import torch
+
+from geolangsplat.sam3 import Sam3Scorer
+
+
+def _scorer(w=0.5, mode="mean", conf=0.2):
+ s = Sam3Scorer.__new__(Sam3Scorer)
+ s.proc = SimpleNamespace(confidence_threshold=conf)
+ s.dual_head = True
+ s.sem_weight = w
+ s.sem_mode = mode
+ return s
+
+
+def _logits(*per_query):
+ # SAM3 pred_logits is [B, Q, 1] (the trailing class dim is squeezed in decode).
+ return torch.tensor([[[v] for v in per_query]])
+
+
+def _outputs(pred_logits=None, presence=None, masks=None, sem=None):
+ out = {}
+ if pred_logits is not None:
+ out["pred_logits"] = pred_logits
+ if presence is not None:
+ out["presence_logit_dec"] = presence
+ if masks is not None:
+ out["pred_masks"] = masks
+ if sem is not None:
+ out["semantic_seg"] = sem
+ return out
+
+
+def test_semantic_head_sigmoid_and_resize():
+ s = _scorer()
+ sem = torch.full((1, 1, 4, 4), 1.5)
+ out = s._semantic_from_outputs(_outputs(sem=sem), 8, 8)
+ assert out.shape == (8, 8)
+ assert torch.allclose(out, torch.sigmoid(torch.tensor(1.5)).expand(8, 8), atol=1e-5)
+
+
+def test_instance_head_presence_gate_and_keep():
+ # presence ~sigmoid(2)=0.88; query0 survives sam_conf, query1 is dropped.
+ s = _scorer(conf=0.5)
+ presence = torch.tensor([2.0])
+ pred_logits = _logits(2.0, -2.0)
+ masks = torch.full((1, 2, 4, 4), 5.0)
+ out = s._instance_from_outputs(_outputs(pred_logits, presence, masks), 4, 4)
+ prob = torch.sigmoid(torch.tensor(2.0)) ** 2 # logit * presence
+ expected = prob * torch.sigmoid(torch.tensor(5.0)) # mask prob
+ assert out.shape == (4, 4)
+ assert torch.allclose(out, expected.expand(4, 4), atol=1e-3)
+
+
+def test_instance_head_empty_when_all_filtered():
+ # presence ~0 -> every detection falls below sam_conf -> no instance.
+ s = _scorer(conf=0.5)
+ o = _outputs(_logits(5.0, 5.0), torch.tensor([-5.0]), torch.full((1, 2, 4, 4), 5.0))
+ assert s._instance_from_outputs(o, 4, 4) is None
+
+
+def test_dual_head_semantic_only_recall():
+ # The recall case: instance head is filtered out, dense semantic head still fires.
+ s = _scorer(w=0.5, mode="mean", conf=0.5)
+ o = _outputs(
+ _logits(5.0, 5.0),
+ torch.tensor([-5.0]),
+ torch.full((1, 2, 4, 4), 5.0),
+ torch.full((1, 1, 4, 4), 2.0),
+ )
+ out = s._fuse_heads(o, 4, 4)
+ expected = 0.5 * torch.sigmoid(torch.tensor(2.0))
+ assert torch.allclose(out, expected.expand(4, 4), atol=1e-4)
+
+
+def test_dual_head_mean_and_max_dispatch():
+ s = _scorer(w=0.3, mode="mean", conf=0.2)
+ o = _outputs(
+ _logits(5.0),
+ torch.tensor([5.0]),
+ torch.full((1, 1, 4, 4), 3.0),
+ torch.full((1, 1, 4, 4), -1.0),
+ )
+ inst = s._instance_from_outputs(o, 4, 4)
+ sem = s._semantic_from_outputs(o, 4, 4)
+ mean_out = s._fuse_heads(o, 4, 4)
+ assert torch.allclose(mean_out, 0.7 * inst + 0.3 * sem, atol=1e-5)
+ s.sem_mode = "max"
+ max_out = s._fuse_heads(o, 4, 4)
+ assert torch.allclose(max_out, torch.maximum(0.7 * inst, 0.3 * sem), atol=1e-5)
+
+
+def test_dual_head_both_empty_returns_none():
+ s = _scorer(conf=0.5)
+ o = _outputs(_logits(-9.0), torch.tensor([-9.0]), torch.zeros(1, 1, 4, 4))
+ assert s._fuse_heads(o, 4, 4) is None # no semantic_seg key, instance filtered
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_select.py b/open_vocabulary_segmentation/geolangsplat/tests/test_select.py
new file mode 100644
index 0000000..d3044eb
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_select.py
@@ -0,0 +1,147 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+import torch
+
+from geolangsplat.config import GeoLangSplatConfig
+from geolangsplat.select import (
+ assign_labels,
+ competitor_idx,
+ select_query,
+ smooth_scores,
+ spatial_cleanup,
+)
+
+
+def test_competitor_idx_drops_hypernym_and_self():
+ names = ["vehicle", "building", "grass", "car"]
+ keep = competitor_idx("car", names)
+ # "vehicle" (hypernym of car) and "car" (self) dropped; building/grass kept.
+ assert set(keep) == {1, 2}
+
+
+def test_assign_labels_gates():
+ # 3 gaussians, 2 classes
+ scores = torch.tensor([[0.9, 0.1], [0.3, 0.28], [0.05, 0.02]])
+ denom = torch.tensor([1.0, 1.0, 1.0])
+ labels, stats = assign_labels(scores, denom, min_weight=0.03, tau=0.15, delta=0.05)
+ assert labels[0].item() == 0 # confident + unambiguous
+ assert labels[1].item() == -1 # ambiguous (0.3-0.28 < delta)
+ assert labels[2].item() == -1 # below tau
+ assert stats["labeled"] == 1
+
+
+def test_assign_labels_min_weight_gate():
+ scores = torch.tensor([[0.9, 0.1]])
+ denom = torch.tensor([0.001]) # below min_weight
+ labels, _ = assign_labels(scores, denom, min_weight=0.03, tau=0.15, delta=0.05)
+ assert labels[0].item() == -1
+
+
+def test_select_query_competition_off_does_not_suppress():
+ cfg = GeoLangSplatConfig(device="cpu") # compete defaults False
+ qscore = torch.tensor([0.8, 0.5, 0.2])
+ seen = torch.ones(3)
+ denom = torch.ones(3)
+ dist = torch.tensor([[0.9], [0.9], [0.9]]) # strong distractor everywhere
+ sel = select_query(
+ qscore,
+ seen,
+ cfg,
+ query="house",
+ select=0.4,
+ denom=denom,
+ dist_scores=dist,
+ dist_names=["grass"],
+ )
+ # competition is off -> distractor is ignored; only the threshold applies
+ assert sel.tolist() == [True, True, False]
+
+
+def test_select_query_threshold_and_competition():
+ cfg = GeoLangSplatConfig(device="cpu")
+ qscore = torch.tensor([0.8, 0.5, 0.2])
+ seen = torch.ones(3)
+ denom = torch.ones(3)
+ dist = torch.tensor([[0.1], [0.6], [0.0]]) # one distractor column
+ sel = select_query(
+ qscore,
+ seen,
+ cfg,
+ query="house",
+ select=0.4,
+ margin=0.08,
+ compete=True,
+ denom=denom,
+ dist_scores=dist,
+ dist_names=["grass"],
+ )
+ # g0: 0.8>=0.4 and 0.8>=0.1+0.08 -> True
+ # g1: 0.5>=0.4 but 0.5 < 0.6+0.08 -> False (loses to distractor)
+ # g2: 0.2<0.4 -> False
+ assert sel.tolist() == [True, False, False]
+
+
+def test_select_query_relative_rescues_weak_prompt():
+ # All scores far below the absolute select (0.30) -> fixed mode returns empty,
+ # but relative mode keeps the peak region (the recall fix).
+ cfg = GeoLangSplatConfig(device="cpu", select=0.30)
+ qscore = torch.tensor([0.10, 0.06, 0.02])
+ seen = torch.ones(3)
+ denom = torch.ones(3)
+ fixed = select_query(qscore, seen, cfg, denom=denom)
+ assert fixed.sum().item() == 0 # weak prompt -> nothing (the 0.0-IoU failure)
+ rel = select_query(qscore, seen, cfg, denom=denom, select_mode="relative", select_rel=0.5)
+ # thr = 0.5 * 0.10 = 0.05 -> keep g0 (0.10) and g1 (0.06), drop g2 (0.02)
+ assert rel.tolist() == [True, True, False]
+
+
+def test_select_query_min_keep_guard():
+ # Fixed mode, everything below threshold, but min_keep guarantees the top-1.
+ cfg = GeoLangSplatConfig(device="cpu", select=0.30)
+ qscore = torch.tensor([0.10, 0.06, 0.02])
+ seen = torch.ones(3)
+ denom = torch.ones(3)
+ sel = select_query(qscore, seen, cfg, denom=denom, min_keep=1)
+ assert sel.tolist() == [True, False, False]
+
+
+def test_smooth_scores_fills_and_denoises():
+ # Two voxels: a tight cluster near origin (mixed scores) and a lone far point.
+ cfg = GeoLangSplatConfig(device="cpu", smooth=True, smooth_beta=1.0, smooth_vox_frac=0.2, smooth_weighted=False)
+ means = torch.tensor([[0, 0, 0], [0.01, 0, 0], [0, 0.01, 0], [5.0, 5.0, 5.0]], dtype=torch.float32)
+ q = torch.tensor([0.9, 0.1, 0.2, 0.8])
+ out = smooth_scores(means, q, None, cfg)
+ # beta=1 -> each score becomes its voxel mean. Cluster mean = (0.9+0.1+0.2)/3 = 0.4.
+ assert torch.allclose(out[:3], torch.full((3,), 0.4), atol=1e-5)
+ assert torch.allclose(out[3], torch.tensor(0.8), atol=1e-5) # lone point unchanged
+
+
+def test_smooth_scores_off_is_identity():
+ cfg = GeoLangSplatConfig(device="cpu") # smooth defaults False
+ means = torch.randn(10, 3)
+ q = torch.rand(10)
+ assert torch.equal(smooth_scores(means, q, None, cfg), q)
+
+
+def test_select_query_consensus_gate():
+ # g0 strong + well-supported; g1 strong score but only 1 supporting view -> gated out.
+ cfg = GeoLangSplatConfig(device="cpu", select=0.30, consensus=True, consensus_frac=0.5, consensus_min=2)
+ qscore = torch.tensor([0.8, 0.8, 0.1])
+ seen = torch.tensor([10.0, 10.0, 10.0])
+ denom = torch.ones(3)
+ support = torch.tensor([8.0, 1.0, 0.0]) # need = max(2, 0.5*10) = 5
+ sel = select_query(qscore, seen, cfg, denom=denom, support=support)
+ assert sel.tolist() == [True, False, False]
+
+
+def test_spatial_cleanup_removes_isolated():
+ cfg = GeoLangSplatConfig(device="cpu", clean3d=True, voxel_frac=0.2, min_pts=3)
+ # cluster of 4 near origin + 1 far floater
+ means = torch.tensor(
+ [[0, 0, 0], [0.01, 0, 0], [0, 0.01, 0], [0.01, 0.01, 0], [5.0, 5.0, 5.0]],
+ dtype=torch.float32,
+ )
+ sel = torch.ones(5, dtype=torch.bool)
+ out = spatial_cleanup(means, sel, cfg)
+ assert out[:4].all() and not out[4]
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_stream.py b/open_vocabulary_segmentation/geolangsplat/tests/test_stream.py
new file mode 100644
index 0000000..2851656
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_stream.py
@@ -0,0 +1,271 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Tests for the low-VRAM streaming lift.
+
+Two properties matter: (1) streaming the whole view set produces the SAME per-Gaussian
+scores as the cached path (it's the same math, just evicted per view), and (2) the
+early-stop fires once enough angularly-diverse views agree.
+"""
+import numpy as np
+import torch
+from PIL import Image
+
+from geolangsplat.config import GeoLangSplatConfig
+from geolangsplat.lift import (
+ _azimuth_coverage,
+ aggregate_alpha,
+ aggregate_alpha_views,
+ build_alpha_cache,
+ stream_scores,
+)
+from geolangsplat.views import View
+
+
+class _Jag:
+ """Stand-in for fVDB's JaggedTensor (only the fields view_contrib reads)."""
+
+ def __init__(self, jdata, joffsets=None):
+ self.jdata = jdata
+ self.joffsets = joffsets
+
+
+class StreamFakeModel:
+ """Fake splat with deterministic per-view contributions (keyed by w2c[0,3])."""
+
+ def __init__(self, n=10, nviews=6, npix=16, top_k=2, seed=0):
+ g = torch.Generator().manual_seed(seed)
+ self.means = torch.rand(n, 3, generator=g)
+ self.n = n
+ self.npix = npix
+ self.top_k = top_k
+ self._store = {
+ i: (torch.randint(0, n, (npix * top_k,), generator=g), torch.rand(npix * top_k, generator=g))
+ for i in range(nviews)
+ }
+
+ def render_contributing_gaussian_ids(
+ self, *, world_to_camera_matrices, projection_matrices, image_width, image_height, near, far, top_k_contributors
+ ):
+ i = int(round(float(world_to_camera_matrices[0, 0, 3])))
+ ids, w = self._store[i]
+ joff = torch.arange(0, ids.numel() + 1, top_k_contributors)
+ return _Jag(ids.clone(), joff), _Jag(w.clone())
+
+
+class StreamFakeScorer:
+ amp_dtype = torch.float32
+
+ def __init__(self, score=0.8):
+ self.score = score
+
+ def encode(self, pils, height, width, batch_encode=False, batch_size=8):
+ return [{"i": k} for k in range(len(pils))]
+
+ def forward_text(self, prompt):
+ return None
+
+ def scoremap(self, prompt, state, text_outputs=None, out_hw=None):
+ rh, rw = out_hw
+ return torch.full((rh, rw), self.score)
+
+
+class PromptStreamScorer(StreamFakeScorer):
+ """Scorer whose per-pixel score depends on the prompt (so competition discriminates)."""
+
+ def __init__(self, scores: dict):
+ self.scores = scores
+
+ def scoremap(self, prompt, state, text_outputs=None, out_hw=None):
+ rh, rw = out_hw
+ return torch.full((rh, rw), float(self.scores.get(prompt, 0.0)))
+
+
+def _views(nviews=6, side=4):
+ out = []
+ for i in range(nviews):
+ w2c = torch.eye(4)
+ w2c[0, 3] = float(i) # view-index tag the fake model reads
+ w2c[1, 3] = float(i % 2) # vary position so azimuths differ
+ K = torch.tensor([[2.0, 0, side / 2], [0, 2.0, side / 2], [0, 0, 1]])
+ pil = Image.fromarray(np.zeros((side, side, 3), dtype=np.uint8))
+ out.append(View(pil=pil, w2c=w2c, K=K, height=side, width=side))
+ return out
+
+
+def _cfg():
+ c = GeoLangSplatConfig(device="cpu")
+ c.top_k = 2
+ c.peak = 0.0
+ return c
+
+
+def _early_stop(view_source, capture, mode="auto"):
+ """Exercise GeoLangSplatEngine._stream_early_stop without building an engine."""
+ from geolangsplat.engine import GeoLangSplatEngine
+
+ eng = GeoLangSplatEngine.__new__(GeoLangSplatEngine)
+ eng.cfg = GeoLangSplatConfig(view_source=view_source, stream_early_stop=mode)
+ eng.auto_report = {"capture": capture} if capture else None
+ return eng._stream_early_stop()
+
+
+def test_early_stop_is_capture_gated():
+ # Compact subject (object dome) -> early-stop ON; spread-out captures -> OFF so
+ # streaming sees every view and recall matches the full build.
+ assert _early_stop("globe", None) is True
+ assert _early_stop("render", "object") is True
+ assert _early_stop("render", "aerial") is False
+ assert _early_stop("images", "aerial") is False
+ # Explicit override wins over the auto gate either way.
+ assert _early_stop("render", "aerial", mode="on") is True
+ assert _early_stop("globe", "object", mode="off") is False
+
+
+def test_azimuth_coverage_spans_the_ring():
+ assert _azimuth_coverage([0.0, 180.0]) == 180.0 # opposite sides
+ assert _azimuth_coverage([0.0, 10.0, 20.0]) == 20.0 # clustered -> small
+ assert _azimuth_coverage([5.0]) == 0.0 # one view -> none
+
+
+def test_streaming_matches_cached_lift():
+ cfg = _cfg()
+ model = StreamFakeModel()
+ scorer = StreamFakeScorer(score=0.8)
+ views = _views()
+ dev = torch.device("cpu")
+
+ cache = build_alpha_cache(model, views, cfg, dev)
+ states = scorer.encode([v.pil for v in views], views[0].height, views[0].width)
+ cached = aggregate_alpha(scorer, states, cache, "thing", cfg)
+
+ scores, denom, seen, stats = stream_scores(model, scorer, views, cfg, ["thing"], dev)
+ # Cached path stores weights as fp16 (a_w.half()); streaming keeps full precision,
+ # so they match to ~1e-3 (streaming is the more precise of the two).
+ assert torch.allclose(scores[0], cached, atol=3e-3)
+ assert stats["views_used"] == len(views) and not stats["early_stopped"]
+ # every covered gaussian gets the constant score; unseen ones are 0.
+ assert torch.allclose(scores[0][denom > 0], torch.full_like(scores[0][denom > 0], 0.8), atol=3e-3)
+
+
+def test_streaming_chunking_is_invariant():
+ """Chunked batched encode must not change the result vs. per-view streaming."""
+ model = StreamFakeModel(nviews=6)
+ scorer = StreamFakeScorer(score=0.8)
+ views = _views(nviews=6)
+ dev = torch.device("cpu")
+
+ cfg1 = _cfg()
+ cfg1.stream_chunk = 1
+ one, *_ = stream_scores(model, scorer, views, cfg1, ["thing"], dev)
+
+ cfg4 = _cfg()
+ cfg4.stream_chunk = 4
+ four, *_ = stream_scores(model, scorer, views, cfg4, ["thing"], dev)
+
+ assert torch.allclose(one[0], four[0], atol=1e-6)
+
+
+def test_streaming_competition_suppresses_losing_query():
+ """Low-VRAM path scores the query + distractors in one stream, then competes:
+ a Gaussian a distractor wins is dropped from the query's selection -- exactly the
+ pool-leaks-into-building case, fixed without the warm cache."""
+ from geolangsplat.select import select_query
+
+ cfg = _cfg()
+ cfg.min_weight = 0.0 # isolate the competition gate from floater culling
+ model = StreamFakeModel(nviews=6)
+ scorer = PromptStreamScorer({"car": 0.9, "building": 0.5})
+ views = _views(nviews=6)
+ dev = torch.device("cpu")
+
+ scores, denom, seen, _stats = stream_scores(model, scorer, views, cfg, ["car", "building"], dev)
+ covered = denom > 0
+ assert int(covered.sum()) > 0
+
+ # 'car' (0.9) beats distractor 'building' (0.5) everywhere -> kept.
+ sel_car = select_query(
+ scores[0],
+ seen,
+ cfg,
+ query="car",
+ compete=True,
+ denom=denom,
+ dist_scores=scores[1:].t().contiguous(),
+ dist_names=["building"],
+ )
+ assert bool(sel_car[covered].all())
+
+ # 'building' (0.5) clears the 0.30 threshold on its own ...
+ sel_bld_solo = select_query(
+ scores[1],
+ seen,
+ cfg,
+ query="building",
+ compete=False,
+ denom=denom,
+ )
+ assert int(sel_bld_solo.sum()) > 0
+ # ... but loses to 'car' (0.9) under competition -> fully suppressed.
+ sel_bld = select_query(
+ scores[1],
+ seen,
+ cfg,
+ query="building",
+ compete=True,
+ denom=denom,
+ dist_scores=scores[0:1].t().contiguous(),
+ dist_names=["car"],
+ )
+ assert int(sel_bld.sum()) == 0
+
+
+def test_fast_views_match_full_over_all_views():
+ """Subset aggregation over ALL views reproduces the full cached lift, and a
+ half-view subset still gives the constant score on covered Gaussians with a
+ smaller (subset) denominator -- the fast query path's contract."""
+ cfg = _cfg()
+ model = StreamFakeModel()
+ scorer = StreamFakeScorer(score=0.8)
+ views = _views()
+ dev = torch.device("cpu")
+ cache = build_alpha_cache(model, views, cfg, dev)
+ states = scorer.encode([v.pil for v in views], views[0].height, views[0].width)
+
+ full = aggregate_alpha(scorer, states, cache, "thing", cfg)
+ allv, denom_all, seen_all = aggregate_alpha_views(scorer, states, cache, "thing", cfg, list(range(len(views))))
+ assert torch.allclose(allv, full, atol=3e-3)
+
+ half = list(range(0, len(views), 2))
+ sub, denom_sub, seen_sub = aggregate_alpha_views(scorer, states, cache, "thing", cfg, half)
+ cov = denom_sub > 0
+ assert torch.allclose(sub[cov], torch.full_like(sub[cov], 0.8), atol=3e-3)
+ assert float(denom_sub.sum()) < float(denom_all.sum()) # fewer views -> less weight
+
+
+def test_subset_indices_are_spread():
+ from geolangsplat.engine import GeoLangSplatEngine
+
+ eng = GeoLangSplatEngine.__new__(GeoLangSplatEngine)
+ eng.views = list(range(100))
+ idx = eng._subset_indices(10)
+ assert len(idx) == 10 and idx == sorted(idx) and idx[0] == 0 and idx[-1] == 99
+ assert eng._subset_indices(0) == list(range(100)) # 0 -> all
+ assert eng._subset_indices(500) == list(range(100)) # cap at n
+
+
+def test_streaming_early_stops_when_views_agree():
+ cfg = _cfg()
+ cfg.agree_k = 3
+ cfg.stream_chunk = 1 # let early-stop fire view-by-view, not per big chunk
+ cfg.min_azimuth_spread = 0.0 # only require the count + convergence here
+ cfg.converge_frac = 1.0 # constant scores converge immediately
+ model = StreamFakeModel(nviews=6)
+ scorer = StreamFakeScorer(score=0.9)
+ views = _views(nviews=6)
+
+ _scores, _denom, _seen, stats = stream_scores(
+ model, scorer, views, cfg, ["thing"], torch.device("cpu"), early_stop=True
+ )
+ assert stats["early_stopped"]
+ assert stats["views_used"] < len(views)
diff --git a/open_vocabulary_segmentation/geolangsplat/tests/test_views.py b/open_vocabulary_segmentation/geolangsplat/tests/test_views.py
new file mode 100644
index 0000000..16ca8c3
--- /dev/null
+++ b/open_vocabulary_segmentation/geolangsplat/tests/test_views.py
@@ -0,0 +1,100 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Tests for the image-view loader's metadata<->COLMAP frame handling.
+
+The critical correctness property: a splat saved during training is recentered by
+a ``normalization_transform``, so its cameras live in a normalized frame while the
+COLMAP poses live in the original (e.g. UTM) frame. We must match each metadata
+camera to its source photo *through* that transform.
+"""
+import numpy as np
+import torch
+from PIL import Image
+
+from geolangsplat.views import (
+ dome_radius_scale,
+ dome_view_metrics,
+ match_metadata_to_colmap,
+ view_sharpness,
+)
+
+
+def _c2w(center: np.ndarray) -> np.ndarray:
+ m = np.eye(4)
+ m[:3, 3] = center
+ return m
+
+
+def test_match_recovers_permutation_through_normalization():
+ rng = np.random.default_rng(0)
+ # Original-frame COLMAP camera centers (large geospatial offset + scatter).
+ base = np.array([6.3e6, 1.9e4, 0.0])
+ col_centers = base[None, :] + rng.normal(scale=50.0, size=(8, 3))
+ col_c2w = np.stack([_c2w(c) for c in col_centers], axis=0)
+
+ # A normalization transform that recenters to the origin (rotation + translation).
+ theta = 0.5
+ R = np.array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]])
+ nt = np.eye(4)
+ nt[:3, :3] = R
+ nt[:3, 3] = -R @ base
+
+ # Metadata cameras = the same cameras expressed in the normalized frame, shuffled.
+ norm_c2w = nt[None] @ col_c2w
+ perm = rng.permutation(8)
+ meta_c2w = norm_c2w[perm]
+
+ match = match_metadata_to_colmap(meta_c2w, col_c2w, nt)
+ assert np.array_equal(match, perm)
+
+
+def test_identity_transform_is_nearest_position():
+ # With no normalization, matching is plain nearest-neighbor by position.
+ centers = np.array([[0.0, 0, 0], [10, 0, 0], [0, 10, 0]])
+ col_c2w = np.stack([_c2w(c) for c in centers], axis=0)
+ meta_c2w = np.stack([_c2w(c + 0.1) for c in centers[::-1]], axis=0)
+ match = match_metadata_to_colmap(meta_c2w, col_c2w, np.eye(4))
+ assert np.array_equal(match, np.array([2, 1, 0]))
+
+
+def test_dome_radius_scale_pulls_low_rings_closer():
+ # Lowest ring sits at `low` (closer); top ring at 1.0; monotone in between.
+ lo, hi, low = 18.0, 88.0, 0.72
+ assert dome_radius_scale(lo, lo, hi, low) == low
+ assert dome_radius_scale(hi, lo, hi, low) == 1.0
+ mid = dome_radius_scale((lo + hi) / 2, lo, hi, low)
+ assert low < mid < 1.0
+ # Degenerate single-ring set: no scaling.
+ assert dome_radius_scale(45.0, 45.0, 45.0, low) == 1.0
+
+
+def test_view_sharpness_ranks_detail_above_blur():
+ rng = np.random.default_rng(0)
+ # High-frequency noise = lots of detail; flat gray = none.
+ noisy = Image.fromarray(rng.integers(0, 256, size=(64, 64, 3), dtype=np.uint8))
+ flat = Image.fromarray(np.full((64, 64, 3), 128, dtype=np.uint8))
+ assert view_sharpness(noisy) > view_sharpness(flat)
+ assert view_sharpness(flat) == 0.0
+
+
+def test_dome_view_metrics_coverage_and_occlusion():
+ H = W = 64
+ radius = 10.0
+ pil = Image.fromarray(np.zeros((H, W, 3), dtype=np.uint8))
+
+ # Good view: center fully covered by the splat, all at the subject distance.
+ alpha = torch.ones(H, W)
+ depth = torch.full((H, W), radius)
+ cov, occ, _ = dome_view_metrics(pil, depth, alpha, radius)
+ assert cov > 0.9 and occ == 0.0
+
+ # See-through view: nothing covered in the center -> low coverage.
+ cov, occ, _ = dome_view_metrics(pil, depth, torch.zeros(H, W), radius)
+ assert cov == 0.0
+
+ # Blocked view: covered, but the whole center sits right in front of the camera
+ # (a near floater) -> high occlusion.
+ near = torch.full((H, W), 0.2 * radius)
+ cov, occ, _ = dome_view_metrics(pil, near, torch.ones(H, W), radius)
+ assert cov > 0.9 and occ > 0.9
+
+
+ 
+