From b45d46c92c8a9b7deefeadaea0a5f5eacef072ae Mon Sep 17 00:00:00 2001 From: akrishnakuma Date: Tue, 30 Jun 2026 13:08:55 -0700 Subject: [PATCH] Add GeoLangSplat: training-free open-vocabulary 3D segmentation for Gaussian splats GeoLangSplat segments a Gaussian splat from a text prompt with no per-scene training: it runs SAM 3 over views of the splat and lifts the 2D scores into 3D with fVDB's alpha-weighted back-projection, returning one label per Gaussian in input .ply order. A single engine handles aerial, satellite, and object/interior captures; an `auto` recipe reads the scene geometry and synthesizes its own views. * segment() / `gls segment` -- prompt in, segmented .ply / overlay / report out * segment catalog -- run a vocabulary, cluster each prompt into objects, browse and extract them as per-object .ply (notebook-friendly; `gls catalog`) * one-shot low-VRAM streaming path plus an optional warm engine (`gls serve`) * CPU-only test suite (SAM 3 and fVDB mocked), black-formatted, Apache-2.0 Lives under open_vocabulary_segmentation/ alongside langsplatv2. Signed-off-by: akrishnakuma --- .../geolangsplat/.gitignore | 56 ++ .../geolangsplat/README.md | 310 +++++++++ .../geolangsplat/completions/gls.bash | 59 ++ .../geolangsplat/examples/.gitkeep | 0 .../examples/building0_safetypark_cutout.png | Bin 0 -> 440113 bytes .../examples/building1_safetypark_cutout.png | Bin 0 -> 364468 bytes .../examples/catalog_safetypark_topdown.png | Bin 0 -> 1254273 bytes .../geolangsplat/geolangsplat/__init__.py | 78 +++ .../geolangsplat/geolangsplat/__main__.py | 8 + .../geolangsplat/geolangsplat/api.py | 213 ++++++ .../geolangsplat/geolangsplat/autoview.py | 395 +++++++++++ .../geolangsplat/geolangsplat/cameras.py | 174 +++++ .../geolangsplat/geolangsplat/catalog.py | 583 +++++++++++++++++ .../geolangsplat/geolangsplat/cli/__init__.py | 64 ++ .../geolangsplat/geolangsplat/cli/_bake.py | 113 ++++ .../geolangsplat/geolangsplat/cli/_catalog.py | 111 ++++ .../geolangsplat/geolangsplat/cli/_check.py | 93 +++ .../geolangsplat/geolangsplat/cli/_common.py | 70 ++ .../geolangsplat/geolangsplat/cli/_doctor.py | 105 +++ .../geolangsplat/geolangsplat/cli/_explore.py | 114 ++++ .../geolangsplat/geolangsplat/cli/_render.py | 202 ++++++ .../geolangsplat/geolangsplat/cli/_segment.py | 195 ++++++ .../geolangsplat/geolangsplat/cli/_serve.py | 260 ++++++++ .../geolangsplat/geolangsplat/cli/_show.py | 94 +++ .../geolangsplat/geolangsplat/cli/_status.py | 60 ++ .../geolangsplat/geolangsplat/cli/_stop.py | 49 ++ .../geolangsplat/geolangsplat/config.py | 306 +++++++++ .../geolangsplat/geolangsplat/engine.py | 612 ++++++++++++++++++ .../geolangsplat/geolangsplat/errors.py | 14 + .../geolangsplat/geolangsplat/instances.py | 404 ++++++++++++ .../geolangsplat/geolangsplat/ipc.py | 335 ++++++++++ .../geolangsplat/geolangsplat/lift.py | 465 +++++++++++++ .../geolangsplat/geolangsplat/outputs.py | 161 +++++ .../geolangsplat/geolangsplat/profile.py | 62 ++ .../geolangsplat/geolangsplat/sam3.py | 340 ++++++++++ .../geolangsplat/geolangsplat/select.py | 212 ++++++ .../geolangsplat/viewer/__init__.py | 9 + .../geolangsplat/viewer/server.py | 365 +++++++++++ .../geolangsplat/geolangsplat/views.py | 596 +++++++++++++++++ .../geolangsplat/pyproject.toml | 35 + .../geolangsplat/requirements.txt | 10 + .../geolangsplat/setup.sh | 68 ++ .../geolangsplat/tests/conftest.py | 62 ++ .../geolangsplat/tests/test_api.py | 54 ++ .../geolangsplat/tests/test_assess.py | 48 ++ .../geolangsplat/tests/test_autoview.py | 108 ++++ .../geolangsplat/tests/test_cameras.py | 42 ++ .../geolangsplat/tests/test_catalog.py | 209 ++++++ .../geolangsplat/tests/test_cli.py | 112 ++++ .../geolangsplat/tests/test_config.py | 83 +++ .../geolangsplat/tests/test_instances.py | 142 ++++ .../geolangsplat/tests/test_ipc.py | 116 ++++ .../geolangsplat/tests/test_outputs.py | 76 +++ .../geolangsplat/tests/test_profile.py | 27 + .../geolangsplat/tests/test_robustness.py | 99 +++ .../geolangsplat/tests/test_sam3_fusion.py | 106 +++ .../geolangsplat/tests/test_select.py | 147 +++++ .../geolangsplat/tests/test_stream.py | 271 ++++++++ .../geolangsplat/tests/test_views.py | 100 +++ 59 files changed, 9202 insertions(+) create mode 100644 open_vocabulary_segmentation/geolangsplat/.gitignore create mode 100644 open_vocabulary_segmentation/geolangsplat/README.md create mode 100644 open_vocabulary_segmentation/geolangsplat/completions/gls.bash create mode 100644 open_vocabulary_segmentation/geolangsplat/examples/.gitkeep create mode 100644 open_vocabulary_segmentation/geolangsplat/examples/building0_safetypark_cutout.png create mode 100644 open_vocabulary_segmentation/geolangsplat/examples/building1_safetypark_cutout.png create mode 100644 open_vocabulary_segmentation/geolangsplat/examples/catalog_safetypark_topdown.png create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/__init__.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/__main__.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/api.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/autoview.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cameras.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/catalog.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/__init__.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_bake.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_catalog.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_check.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_common.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_doctor.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_explore.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_render.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_segment.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_serve.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_show.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_status.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/cli/_stop.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/config.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/engine.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/errors.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/instances.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/ipc.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/lift.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/outputs.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/profile.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/sam3.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/select.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/__init__.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/viewer/server.py create mode 100644 open_vocabulary_segmentation/geolangsplat/geolangsplat/views.py create mode 100644 open_vocabulary_segmentation/geolangsplat/pyproject.toml create mode 100644 open_vocabulary_segmentation/geolangsplat/requirements.txt create mode 100644 open_vocabulary_segmentation/geolangsplat/setup.sh create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/conftest.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_api.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_assess.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_autoview.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_cameras.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_catalog.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_cli.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_config.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_instances.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_ipc.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_outputs.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_profile.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_robustness.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_sam3_fusion.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_select.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_stream.py create mode 100644 open_vocabulary_segmentation/geolangsplat/tests/test_views.py diff --git a/open_vocabulary_segmentation/geolangsplat/.gitignore b/open_vocabulary_segmentation/geolangsplat/.gitignore new file mode 100644 index 0000000..f76fb49 --- /dev/null +++ b/open_vocabulary_segmentation/geolangsplat/.gitignore @@ -0,0 +1,56 @@ +# Python +__pycache__/ +*.pyc +*.egg-info/ +build/ +dist/ +.pytest_cache/ +.mypy_cache/ + +# Local query/segmentation outputs and large data artifacts (never commit splats) +*.ply +*.npz +*.csv +*.png +*.zip +*.glscache +outputs/ +runs/ + +# ...but do ship the README example figures +!examples/ +!examples/*.png + +# Local benchmark / evaluation run directories +eval_out/ +fast_bench/ +id_bench/ +robust_out/ +routed_bench/ +frgs_logs/ + +# Local design notes (not shipped) +ARCHITECTURE.md +SLIDES.md +INTEGRATION.md +BENCHMARKS.md + +# Deferred identity field (seed/click grouping) -- lands in a later change +geolangsplat/identity.py +tests/test_identity.py + +# Local benchmark / evaluation code and scripts (not part of this example) +geolangsplat/eval/ +scripts/bench_inference.py +scripts/bench_fast_query.py +scripts/bench_identity.py +scripts/bench_robustness.py +scripts/bench_scenes_aerial_sat.json +scripts/eval_scenes.py +scripts/eval_benchmark.py +scripts/run_lerf_eval.sh +scripts/try_instances.py +scripts/safety_park_demo.sh +tests/test_eval_align.py +tests/test_eval_lerf_ovs.py +tests/test_eval_metrics.py diff --git a/open_vocabulary_segmentation/geolangsplat/README.md b/open_vocabulary_segmentation/geolangsplat/README.md new file mode 100644 index 0000000..470ef16 --- /dev/null +++ b/open_vocabulary_segmentation/geolangsplat/README.md @@ -0,0 +1,310 @@ +# GeoLangSplat + +**Training-free, open-vocabulary 3D segmentation for Gaussian splats.** Give it a text +prompt (`"house"`, `"road"`, `"tree"`, …) and a splat; it returns which Gaussians belong to +that concept — as a segmented `.ply`, a highlight overlay, or a per-Gaussian report. One +engine serves **aerial** (oblique drone), **satellite** (near-nadir), and **object/interior** +(360° inward) captures; the default `auto` recipe reads the scene geometry and synthesizes its +own views, so the common case is `gls segment scene.ply "thing"` with no flags. + +

+ 18 building instances segmented on SafetyPark +

+

+ one building lifted out as its own .ply +   + another building lifted out as its own .ply +

+

gls catalog scene.ply building  -->  every building becomes its own instance, then lifts out as a .ply

+ +--- + +## How it works + +``` +text prompt + | + v +SAM3 segments each view --> alpha-weighted back-projection --> per-Gaussian scores + ^ | +views of the splat competition + cleanup <--+ +(auto-synthesized, or your real photos) | + v + segmented .ply / overlay .ply / per-Gaussian report +``` + +- **Training-free.** No per-scene optimization, no CLIP. We run SAM3 over views of the splat + and lift its per-pixel scores into 3D --> one continuous score per Gaussian. A brand-new + scene is usable in seconds; quality just tracks SAM3 and the reconstruction. +- **Views are the quality knob.** The model can only label what SAM3 sees. The `auto` recipe + reads the scene and picks the views for you --> flat/top-down captures get a nadir + oblique + ladder, rooms and objects get a gravity-up dome. Or point it at the scene's real photos with + `--view-source images --sfm`. +- **One code path, two speeds.** One-shot (default) streams the lift one chunk at a time + (encode --> score --> project --> evict), so peak VRAM stays flat no matter how big the scene + is. Keep an engine warm with `gls serve` and repeat queries come back in ~1-2 s. + +--- + +## Installation + +GeoLangSplat runs inside an existing **fVDB** environment (the fvdb monorepo build, with +`torch` + `fvdb-reality-capture`; not on PyPI). It is light --> it adds only `numpy`, `pillow`, +`tyro`, and the `gls` command. + +1. **Install into your fVDB env:** + + ```bash + source /path/to/fvdb-env/bin/activate + pip install -e . # registers the `gls` command + ``` + +2. **Add SAM3** — the 2D segmenter used by `segment`/`bake`. It is not on PyPI: install it from + source so that `import sam3` works. It loads lazily, so it is *not* needed to import the + package, run the tests (SAM3 is mocked), or run `gls check`. + +3. **Point at a checkpoint and run the preflight:** + + ```bash + export GEOLANGSPLAT_SAM_CKPT=/path/to/sam3.1_multiplex.pt + gls doctor # checks GPU/bf16, fvdb, SAM3, checkpoint + ``` + +For a one-shot provision (e.g. baking a lab image), `setup.sh` downloads the checkpoint +(`GEOLANGSPLAT_SAM_URL=...`), installs the package, and runs `gls doctor` so the environment +ships ready — the notebook/first run then needs no installs. + +--- + +## Python / notebook + +```python +from geolangsplat import segment, GeoLangSplatConfig + +# single prompt -> per-Gaussian selection, written to an overlay .ply +res = segment( + "scene.ply", "building", + config=GeoLangSplatConfig(low_vram=True), # bound VRAM (~6-7 GB) + output="ply_overlay", out_path="building.ply", +) +print(res.num_selected, "/", res.scores.shape[0]) + +# multiple prompts -> multi-class labelling (argmax + confidence/ambiguity gates) +res = segment("scene.ply", ["building", "tree", "grass", "road"], recipe="satellite") +res.to_report("labels.csv") # index,x,y,z,label_id,label,score (+ legend.json) +``` + +`segment(...)` returns a `SegmentResult` with per-Gaussian `scores`, a boolean `selected` +mask, and (multi-prompt) `label_ids`, all in **input `.ply` order**. Pass a `GaussianSplat3d` +or a path; every `GeoLangSplatConfig` field is settable (explicit fields beat a recipe). Reuse +`GeoLangSplatEngine` to build once and query many times. + +### Use as a post-processing step + +GeoLangSplat is a **self-contained post-processing stage** over any reconstruction: in goes a +Gaussian splat (a `.ply` or in-memory `fvdb.GaussianSplat3d`, plus an optional COLMAP scene), +out come per-Gaussian labels in input order. With no per-scene training and no required upstream +state, it drops in after a reconstruction step (e.g. fvdb-reality-capture) — consume +`result.selected` / `result.label_ids` directly or persist `to_ply_*` / `to_report`. + +Display inline in a notebook without an interactive viewer: + +```python +from IPython.display import Image +import subprocess; subprocess.run(["gls", "render", "building.ply", "-o", "building.png"]) +Image("building.png") +``` + +The overlay `.ply` also opens in the `fvdb.viz` viewer (`gls show building.ply`). + +### Segment catalog (notebook object database) + +`build_catalog` runs a whole vocabulary in one pass and turns the result into a small, +browsable database of individual objects: each prompt is segmented, split into spatial +objects (3D connected components), and objects hit by several prompts (e.g. `car` and +`vehicle`) are merged and given a stable id. The catalog is a table you browse and pull +single objects from — handy for picking a specific segment to export downstream. + +```python +from geolangsplat import build_catalog + +cat = build_catalog("scene.ply", ["building", "car", "tree", "road"], recipe="satellite") +cat # rich HTML table inline (id, label, n_gaussians, size, prompts) +cat.browse() # interactive picker: preview + select objects -> export .ply +cat.table # the same data as a pandas DataFrame (filter / sort) +cat.show() # top-down render with each object's id drawn on it +cat.extract(3, "obj3.ply") # pull one object out as its own .ply +cat.export_all("scene_catalog/") # objects/_