From 2bbbad520f14f44861185d589af47c2936e22937 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 31 Aug 2026 11:04:29 -0400 Subject: [PATCH 1/5] =?UTF-8?q?feat(#gen3d):=20TRELLIS.2=20image-to-3D=20b?= =?UTF-8?q?ackend=20=E2=80=94=20native=20game-ready=20pipeline,=20no=20nvd?= =?UTF-8?q?iffrast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Microsoft TRELLIS.2 as the third generate3d backend and the DEFAULT whenever its runtime is installed. Two runtime flavors, discovered at run time: trellis.cpp (C++/GGML — runs on Apple Silicon Metal via our upstreamed port; models: HF fernandotonon/QtMeshEditor-trellis2-gguf) and the Python CUDA sidecar (ai/trellis2/, inference ONLY). QtMeshEditor natively owns ALL asset processing — deliberately WITHOUT NVIDIA nvdiffrast/nvdiffrec (research -only license; CI-guarded by scripts/check-trellis2-restricted-deps.sh + Trellis2Guard_test): - Trellis2Interchange: QTM3D interchange (mesh + sparse voxel PBR attrs) + trellis.cpp --dump-post reader; full-res source preserved for re-baking - Trellis2Bake: game-ready pass (voxel-scale weld, debris cull, winding unification with small-island orientation vote, optional Taubin pre-smooth, capped QEM ladder) + xatlas UV + parallel multi-channel bake (basecolor RGBA/roughness/metallic/tangent-space detail normal) from the alpha-aware sparse-volume sampler; bakeDetailNormal shared with the TripoSR path - Trellis2Predictor: runtime discovery, presets fast/balanced/high, implicit 150k/300k bake cap, progress mapping, QTMESH_TRELLIS2_IMPORT re-bake hook - game-ready pass (weld/cull/simplify + detail-normal re-bake) now available for ALL backends via --target-tris / target_tris / GUI Mesh dropdown — fixes 'decimated Tripo output turns into a blob' - surfaces: CLI --backend trellis2/--preset/--seed, MCP backend/preset/seed/ target_tris args, GUI backend picker + Quality/Mesh/Texture rows - license audit: docs/trellis2-dependencies.md; user guide docs/TRELLIS2.md; RMBG-2.0 (CC-BY-NC) never loads — own U²-Net matte; unified CLI/GUI QSettings identity so runtime discovery works in both Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 2 + .gitignore | 2 + CLAUDE.md | 3 + THIRD_PARTY_AI_MODELS.md | 43 + ai/trellis2/README.md | 79 + ai/trellis2/THIRD_PARTY_LICENSES.md | 30 + ai/trellis2/generate.py | 431 +++++ ai/trellis2/install.py | 266 +++ ai/trellis2/qtm3d.py | 135 ++ ai/trellis2/requirements.txt | 27 + docs/TRELLIS2.md | 132 ++ docs/trellis2-dependencies.md | 161 ++ qml/PropertiesPanel.qml | 182 +- scripts/check-trellis2-restricted-deps.sh | 58 + src/CLIPipeline.cpp | 102 +- src/CMakeLists.txt | 3 + src/ImageTo3D/BackgroundRemover.cpp | 31 +- src/ImageTo3D/BackgroundRemover.h | 8 + ...LIPipeline_cmdgenerate3d_coverage_test.cpp | 44 + src/ImageTo3D/MeshGenBuilder.cpp | 41 +- src/ImageTo3D/MeshGenController.cpp | 87 +- src/ImageTo3D/MeshGenController.h | 7 + src/ImageTo3D/MeshGenPredictor.cpp | 133 +- src/ImageTo3D/MeshGenPredictor.h | 50 +- src/ImageTo3D/Trellis2Bake.cpp | 1705 +++++++++++++++++ src/ImageTo3D/Trellis2Bake.h | 198 ++ src/ImageTo3D/Trellis2Bake_test.cpp | 385 ++++ src/ImageTo3D/Trellis2Guard_test.cpp | 152 ++ src/ImageTo3D/Trellis2Interchange.cpp | 429 +++++ src/ImageTo3D/Trellis2Interchange.h | 77 + src/ImageTo3D/Trellis2Interchange_test.cpp | 173 ++ src/ImageTo3D/Trellis2Predictor.cpp | 612 ++++++ src/ImageTo3D/Trellis2Predictor.h | 116 ++ src/ImageTo3D/Trellis2Predictor_test.cpp | 85 + src/MCPServer.cpp | 83 +- src/main.cpp | 7 + tests/CMakeLists.txt | 3 + 37 files changed, 5993 insertions(+), 89 deletions(-) create mode 100644 ai/trellis2/README.md create mode 100644 ai/trellis2/THIRD_PARTY_LICENSES.md create mode 100644 ai/trellis2/generate.py create mode 100644 ai/trellis2/install.py create mode 100644 ai/trellis2/qtm3d.py create mode 100644 ai/trellis2/requirements.txt create mode 100644 docs/TRELLIS2.md create mode 100644 docs/trellis2-dependencies.md create mode 100755 scripts/check-trellis2-restricted-deps.sh create mode 100644 src/ImageTo3D/Trellis2Bake.cpp create mode 100644 src/ImageTo3D/Trellis2Bake.h create mode 100644 src/ImageTo3D/Trellis2Bake_test.cpp create mode 100644 src/ImageTo3D/Trellis2Guard_test.cpp create mode 100644 src/ImageTo3D/Trellis2Interchange.cpp create mode 100644 src/ImageTo3D/Trellis2Interchange.h create mode 100644 src/ImageTo3D/Trellis2Interchange_test.cpp create mode 100644 src/ImageTo3D/Trellis2Predictor.cpp create mode 100644 src/ImageTo3D/Trellis2Predictor.h create mode 100644 src/ImageTo3D/Trellis2Predictor_test.cpp diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0bb2ed68f..c9e436761 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -56,6 +56,8 @@ jobs: run: ./scripts/sync-doc-versions-from-cmake.sh --check - name: File association packaging files present run: chmod +x ./scripts/verify-file-associations.sh && ./scripts/verify-file-associations.sh + - name: TRELLIS.2 restricted-dependency gate (no nvdiffrast/nvdiffrec) + run: chmod +x ./scripts/check-trellis2-restricted-deps.sh && ./scripts/check-trellis2-restricted-deps.sh #################################################################### # Asset Scan (runs first, before all builds) diff --git a/.gitignore b/.gitignore index 9a56429a7..19f6ea08c 100755 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ __pycache__/ !docs/MOCAP_SPIKE.md .mocap_work/ .venv-mocap/ +!docs/TRELLIS2.md +!docs/trellis2-dependencies.md diff --git a/CLAUDE.md b/CLAUDE.md index 0166af908..e6271a2cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,6 +155,9 @@ qtmesh generate3d image.png --texture-size 2048 -o out.glb # quality pass (ON b qtmesh generate3d image.png --no-smooth --no-refine --no-bake-texture -o out.glb # raw marching-cubes output with per-vertex color (pre-quality-pass behavior) qtmesh generate3d image.png --upscale-texture -o out.glb # + Real-ESRGAN 2x on the baked diffuse (sharper color; upscale model downloads on demand) qtmesh generate3d image.png --no-pbr -o out.glb # skip the PBR stage (#404 normal+roughness synthesized from the baked diffuse and bound into the material — ON by default; the polished-surface look; writes *_normal/_roughness.png sidecars) +qtmesh generate3d photo.png -o out.glb --backend trellis2 --preset balanced --seed 42 # TRELLIS.2 backend (Microsoft, MIT code+weights): the highest-quality tier and the DEFAULT whenever its sidecar runtime is installed (ai/trellis2/install.py; Linux + NVIDIA GPU ≥24GB VRAM). Python does INFERENCE ONLY; QtMeshEditor natively does alpha matte (own U²-Net — upstream's RMBG-2.0 is CC-BY-NC and never loads), game-ready weld/debris-cull/simplify, xatlas UV + multi-channel PBR bake (basecolor RGBA/roughness/metallic/normal via Trellis2Bake — deliberately WITHOUT NVIDIA nvdiffrast/nvdiffrec, which are research-only-licensed and CI-gated out; audit: docs/trellis2-dependencies.md). Full-res source preserved as _source.qtm3d +qtmesh generate3d photo.png -o out.glb --backend trellis2 --preset high --target-tris 25000 --texture-size 4096 # TRELLIS.2 game-ready presets: --target-tris 10000/25000/50000 (0 = original density); presets fast=512 / balanced=1024_cascade / high=1536_cascade; QTMESH_TRELLIS2_MOCK=1 exercises the whole pipeline without a GPU +qtmesh generate3d photo.png --target-tris 25000 -o out.glb # game-ready pass, ALL backends: weld + debris-cull + meshopt-simplify toward the target, then re-bake the diffuse on the simplified mesh (TripoSR field bake is density-independent) + bake the dense source's relief into a tangent-space detail normal map sharing the same atlas (Trellis2Bake::bakeDetailNormal). This is the fix for 'decimated Tripo output turns into a blob / skins badly' — simplify hard, keep detail in textures. 0 = original density qtmesh generate3d image.png --backend triposg --flow-steps 25 --guidance 7 -o out.glb # TripoSG backend (1.5B rectified-flow DiT, MIT): higher-fidelity GEOMETRY, slower; models download on first use; --guidance 0 disables CFG. TripoSG is GEOMETRY-ONLY (no colour decoder) — a colour bake queries TripoSR's image-conditioned colour field on the same image + projects the input photo onto the visible front (the back is inferred, so it's approximate; a "Generate texture (AI)" option in the GUI does a front-photo + SD-generated-back multi-view bake for a better back). int8 tier is DROPPED for TripoSG (fp32 only — quantized geometry degrades to blobs, no ARM speed win) qtmesh segment model.fbx # AI part segmentation (#410/#818): per-part vertex/face counts; category auto-detected (body/vegetation/vehicle/building) qtmesh segment model.fbx --json # full vertex/face → label arrays + per-part summary + resolved category (stable schema) diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index cc53c3569..af1dfdd3d 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -127,6 +127,49 @@ the binary). Attribution + licenses for the models and their training data: `scripts/upload-triposr-models.sh`). First use downloads them; if ever absent the feature reports a clean "not yet hosted" state (no crash) — the RigNet precedent. +## TRELLIS.2 — image-to-3D generation, sidecar backend (default when installed) + +- **Model:** Microsoft TRELLIS.2 — sparse-voxel "O-Voxel" flexible-dual-grid + representation with volumetric PBR attributes (base color / metallic / + roughness / alpha), 4B-parameter flow stack. NOT an ONNX consumer: it runs as + an out-of-process **Python sidecar** (`ai/trellis2/`, Linux + NVIDIA CUDA, + ≥24 GB VRAM recommended) because the custom sparse CUDA kernels + (FlexGEMM/o-voxel/CuMesh) have no ONNX lowering. First runtime-Python + component in the project; nothing is bundled — the user installs the isolated + environment with `ai/trellis2/install.py`. +- **Source:** Microsoft — *"Native and Compact Structured Latents for 3D + Generation"* (arXiv 2512.14692). https://github.com/microsoft/TRELLIS.2 — + code **MIT**, pinned `75fbf0183001ed9876c8dbb35de6b68552ee08bd`. Weights: + https://huggingface.co/microsoft/TRELLIS.2-4B — **MIT**, rev `af44b45f…`, + ≈18.9 GB, downloaded on first generation under the user's HF account (plus + the MIT `microsoft/TRELLIS-image-large` sparse-structure decoder). + Companion libs JeffreyXiang/CuMesh + FlexGEMM — **MIT**, pinned in install.py. +- **License boundary (the deciding work of this integration):** upstream's + texture bake + preview renderers use NVIDIA **nvdiffrast/nvdiffrec** (NVIDIA + Source Code License — research/evaluation only) — both are **excluded + entirely** (not installed/imported/invoked; `install.py` patches the MIT file + `o_voxel/__init__.py` so the package imports without them; CI gate + `scripts/check-trellis2-restricted-deps.sh` + `Trellis2GuardTest`). Their + functionality is QtMeshEditor-native code: `src/ImageTo3D/Trellis2Bake.{h,cpp}` + (xatlas unwrap + UV-space barycentric rasterizer + Ericson closest-point + + trilinear sparse-volume sampling + meshoptimizer game-ready pipeline). + The upstream default background remover `briaai/RMBG-2.0` is **CC BY-NC** and + is never downloaded — the loader is stubbed and the alpha matte comes from + the project's own U²-Net (`BackgroundRemover` keepAlpha). The conditioning + encoder is **DINOv3** (`facebook/dinov3-vitl16-pretrain-lvd1689m`) under + Meta's custom **DINOv3 License** — commercial use permitted, gated download, + "Built with DINOv3" attribution; NOT MIT, so the stack must never be + described as "entirely MIT". Full audit table + pins: + `docs/trellis2-dependencies.md`; user guide: `docs/TRELLIS2.md`. +- Sidecar contract: `generate.py` emits raw vertices/faces + the sparse + attribute volume as a **QTM3D interchange** file + (`src/ImageTo3D/Trellis2Interchange.{h,cpp}`); `Trellis2Predictor` drives the + process (JSON-line progress, cancellation, runtime discovery via + `QTMESH_TRELLIS2_ENV`/`ai/trellis2Env`, mock mode for GPU-less e2e tests) and + is the **default backend** (`MeshGenPredictor::defaultBackend()`) whenever + the runtime resolves; every surface reports a clean "runtime not installed" + message otherwise (no crash). + ## U²-Net — background removal for image-to-3D (epic #764) - **Model:** U²-Net salient-object detection (`u2net.onnx`) — the default diff --git a/ai/trellis2/README.md b/ai/trellis2/README.md new file mode 100644 index 000000000..7ddaf2dfa --- /dev/null +++ b/ai/trellis2/README.md @@ -0,0 +1,79 @@ +# QtMeshEditor TRELLIS.2 sidecar + +Out-of-process Python runtime for the `trellis2` image-to-3D backend +(`qtmesh generate3d --backend trellis2`, the "TRELLIS.2 — High Quality" option in the +GUI's *AI: Image → 3D* panel). + +## What it does — and what it deliberately does NOT do + +`generate.py` runs **Microsoft TRELLIS.2** (MIT, pinned revision) inference only: + +``` +RGBA image (alpha matte made by QtMeshEditor's own U²-Net) + ↓ generate.py DINOv3 cond → sparse structure → shape SLat → tex SLat +raw geometry + sparse PBR attribute volume (base color / metallic / roughness / alpha) + ↓ QTM3D interchange file (qtm3d.py ↔ src/ImageTo3D/Trellis2Interchange.*) +QtMeshEditor C++: cleanup → weld → simplify (game-ready presets) → xatlas UV + → texture/PBR bake (own rasterizer) → normals/tangents → Ogre / GLB / FBX +``` + +The upstream reference implementation uses **NVIDIA nvdiffrast + nvdiffrec** for UV-space +rasterization, texture baking and PBR previews. Those libraries are under the NVIDIA Source +Code License (research/evaluation only) and are **prohibited here**: they are not in +`requirements.txt`, `install.py` never installs them, `generate.py` never imports them (and +warns — `--strict`: fails — if they are unexpectedly present), and `install.py` patches the +MIT file `o_voxel/__init__.py` so the o-voxel package imports without them. The equivalent +functionality is QtMeshEditor's own C++ code (`src/ImageTo3D/Trellis2Bake.*`). Full audit: +`docs/trellis2-dependencies.md`. + +The upstream default background remover (`briaai/RMBG-2.0`, named in the shipped +`pipeline.json`) is **CC BY-NC** and is likewise never downloaded or loaded — the loader is +stubbed and inputs must already carry an alpha matte. + +## Requirements + +- Linux, NVIDIA GPU (**≥ 24 GB VRAM** recommended; `low_vram` staggering is on), CUDA 12.4 +- Python ≥ 3.10, git, a compiler toolchain (the o-voxel/CuMesh/FlexGEMM CUDA extensions are + built from pinned sources) +- A Hugging Face account that has **accepted Meta's DINOv3 License** (the + `facebook/dinov3-vitl16-pretrain-lvd1689m` encoder is gated): `huggingface-cli login` +- ~19 GB for TRELLIS.2-4B weights (MIT), downloaded on first generation + +## Install + +```bash +python3 ai/trellis2/install.py # → /QtMeshEditor/trellis2 +# or choose a location: +python3 ai/trellis2/install.py --dest /opt/qtmesh-trellis2 +``` + +QtMeshEditor auto-detects the default location. For a custom one, set the env var +`QTMESH_TRELLIS2_ENV=` or QSettings `ai/trellis2Env`. + +## Direct use / troubleshooting + +```bash +ENV=~/.local/share/QtMeshEditor/trellis2 +$ENV/env/bin/python $ENV/generate.py --check # environment probe +$ENV/env/bin/python $ENV/generate.py --report-deps --check # dependency report +$ENV/env/bin/python $ENV/generate.py --input subject.png --output out.qtm3d \ + --preset balanced --seed 42 +$ENV/env/bin/python $ENV/generate.py --input any.png --output out.qtm3d --mock + # plumbing test, no GPU +``` + +Presets: `fast` = TRELLIS.2 `512`, `balanced` = `1024_cascade` (upstream default), +`high` = `1536_cascade`. + +Progress/status is emitted as JSON lines on stdout (QtMeshEditor parses these); tqdm and +debug logs go to stderr. + +## Licenses + +See `THIRD_PARTY_LICENSES.md` (this directory) and `docs/trellis2-dependencies.md` for the +authoritative table. Summary: TRELLIS.2 code + weights MIT; CuMesh/FlexGEMM MIT; PyTorch / +flash-attn BSD-3; transformers Apache-2.0; DINOv3 encoder under Meta's **DINOv3 License** +(commercial use permitted, gated download, "Built with DINOv3" attribution); easydict +LGPL-3.0 (pure-Python, flagged); nvdiffrast/nvdiffrec/RMBG-2.0 excluded. + +*Built with DINOv3.* diff --git a/ai/trellis2/THIRD_PARTY_LICENSES.md b/ai/trellis2/THIRD_PARTY_LICENSES.md new file mode 100644 index 000000000..ac630994e --- /dev/null +++ b/ai/trellis2/THIRD_PARTY_LICENSES.md @@ -0,0 +1,30 @@ +# TRELLIS.2 sidecar — third-party licenses + +This environment is installed on the user's machine by `install.py`; QtMeshEditor does not +redistribute any of it. Authoritative audit: `docs/trellis2-dependencies.md` (pinned +revisions, red/yellow flags, enforcement). + +| Component | License | Notes | +|---|---|---| +| Microsoft TRELLIS.2 (code, incl. in-repo `o-voxel`; pinned `75fbf018…`) | MIT | `o_voxel/__init__.py` locally patched (lazy `postprocess`) so the package imports without nvdiffrast | +| `microsoft/TRELLIS.2-4B` weights (HF rev `af44b45f…`) | MIT | downloaded on first use | +| `microsoft/TRELLIS-image-large` sparse-structure decoder (HF rev `25e0d31f…`) | MIT | referenced by upstream `pipeline.json` | +| JeffreyXiang/CuMesh (pinned `12289e10…`) | MIT | built from source — never `pip install cumesh` (unrelated unlicensed PyPI package) | +| JeffreyXiang/FlexGEMM (pinned `6dd94a85…`) | MIT | sparse conv + `grid_sample_3d` | +| Eigen (vendored inside o-voxel) | MPL-2.0 | | +| PyTorch 2.6.0 / torchvision 0.21.0 | BSD-3-Clause | CUDA build | +| flash-attn 2.7.3 (or xformers) | BSD-3-Clause | attention backend | +| transformers, huggingface_hub, safetensors | Apache-2.0 | | +| numpy | BSD-3-Clause | | +| Pillow | HPND/MIT-CMU | | +| easydict 1.13 | **LGPL-3.0** | pure-Python, imported unmodified from this user-installed env; the only copyleft item — flagged in the audit | +| ninja, packaging | Apache-2.0 / BSD | build-time | +| `facebook/dinov3-vitl16-pretrain-lvd1689m` | **DINOv3 License** (Meta, custom) | commercial use permitted; gated download under the user's HF account; license text: . **Built with DINOv3.** | + +## Intentionally excluded + +| Component | License | Why excluded | +|---|---|---| +| nvdiffrast | NVIDIA Source Code License (1-Way Commercial) — research/evaluation only | replaced by QtMeshEditor's own C++ rasterizer/baker (`src/ImageTo3D/Trellis2Bake.*`) | +| nvdiffrec (`nvdiffrec_render`) | NVIDIA Source Code License for nvdiffrec — research/evaluation only | preview lighting replaced by QtMeshEditor's Ogre/RTSS + HDR/IBL renderer | +| `briaai/RMBG-2.0` background remover | CC BY-NC 4.0 | never downloaded/loaded; background removal is done by QtMeshEditor's U²-Net (Apache-2.0) | diff --git a/ai/trellis2/generate.py b/ai/trellis2/generate.py new file mode 100644 index 000000000..0bcab4038 --- /dev/null +++ b/ai/trellis2/generate.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""QtMeshEditor TRELLIS.2 inference sidecar. + +Runs Microsoft TRELLIS.2 (MIT, pinned revision - see runtime.json written by +install.py) image-to-3D inference and exports the RAW generation - vertices, +faces and the sparse PBR attribute volume (base color / metallic / roughness / +alpha) - to a QTM3D interchange file. Everything downstream (mesh cleanup, +simplification, UV unwrapping, texture/PBR baking, preview rendering, GLB/FBX +export) is done by QtMeshEditor's own C++ code. + +LICENSE BOUNDARY (do not weaken - see docs/trellis2-dependencies.md): + * nvdiffrast and nvdiffrec are PROHIBITED. This process must run in an + environment where importing the prohibited nvdiffrast fails because it is not + installed. They are never imported here, and o_voxel/__init__.py is + patched by install.py so the o-voxel package no longer imports them + either. A startup check warns if either is unexpectedly present + (--strict turns the warning into a hard error). + * briaai/RMBG-2.0 (the upstream default background remover named in + TRELLIS.2-4B/pipeline.json) is CC BY-NC and is NEVER downloaded or + loaded: the rembg loader is stubbed out before pipeline construction, + and the input image must already carry an alpha matte (QtMeshEditor + produces it with its own Apache-2.0 U^2-Net remover). + +Progress protocol: one JSON object per line on stdout - + {"event":"stage","stage":""} stage transitions + {"event":"progress","stage":s,"done":d,"total":t} coarse progress + {"event":"deps", ...} dependency report + {"event":"done", ...stats} success (last line) + {"event":"error","message":m} failure (last line) +Human/debug output (tqdm etc.) goes to stderr only. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import platform +import signal +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import qtm3d # noqa: E402 (local interchange writer) + +PROHIBITED_MODULES = ("nvdiffrast", "nvdiffrec", "nvdiffrec_render") + +PRESET_PIPELINE_TYPE = { + "fast": "512", + "balanced": "1024_cascade", + "high": "1536_cascade", +} + + +def emit(obj: dict) -> None: + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + +def stage(name: str) -> None: + emit({"event": "stage", "stage": name}) + + +def log(msg: str) -> None: + sys.stderr.write(f"[trellis2] {msg}\n") + sys.stderr.flush() + + +def fail(message: str, code: int = 1) -> "NoReturn": # noqa: F821 + emit({"event": "error", "message": message}) + sys.exit(code) + + +def check_prohibited(strict: bool) -> list[str]: + """Detect prohibited NVIDIA research-only packages in the environment. + + They must not be installed at all; this integration never imports them. + Presence is a packaging mistake worth surfacing (Phase 12). + """ + present = [m for m in PROHIBITED_MODULES if importlib.util.find_spec(m) is not None] + for m in present: + log(f"WARNING: prohibited module '{m}' is installed in this environment. " + "QtMeshEditor's TRELLIS.2 integration does not use it, but its license " + "(NVIDIA Source Code License, research/evaluation only) makes it unsafe " + "for commercial environments - uninstall it.") + if present and strict: + fail("prohibited modules present in --strict mode: " + ", ".join(present)) + return present + + +def runtime_info() -> dict: + info_path = os.path.join(HERE, "runtime.json") + # install.py writes runtime.json next to the installed copy of this script; + # when running from the source tree, look in the env dir instead. + if not os.path.exists(info_path): + env_dir = os.environ.get("QTMESH_TRELLIS2_ENV", "") + if env_dir: + info_path = os.path.join(env_dir, "runtime.json") + if os.path.exists(info_path): + try: + with open(info_path) as f: + return json.load(f) + except Exception: + pass + return {} + + +def dependency_report(present_prohibited: list[str]) -> dict: + """Phase 15: report which backend dependencies are actually loaded.""" + rep = { + "event": "deps", + "python": platform.python_version(), + "platform": platform.platform(), + } + rt = runtime_info() + rep["trellis2Revision"] = rt.get("trellis2Revision", "unknown") + rep["cumeshRevision"] = rt.get("cumeshRevision", "unknown") + rep["flexgemmRevision"] = rt.get("flexgemmRevision", "unknown") + for mod, key in (("torch", "torch"), ("transformers", "transformers"), + ("flash_attn", "flashAttn"), ("o_voxel", "oVoxel"), + ("cumesh", "cumesh"), ("flex_gemm", "flexGemm")): + try: + spec = importlib.util.find_spec(mod) + except (ModuleNotFoundError, ValueError): + spec = None + rep[key] = "installed" if spec is not None else "NOT INSTALLED" + try: + import torch # noqa: WPS433 + rep["torch"] = torch.__version__ + rep["cuda"] = torch.version.cuda or "none" + rep["cudaAvailable"] = bool(torch.cuda.is_available()) + if torch.cuda.is_available(): + rep["gpu"] = torch.cuda.get_device_name(0) + rep["vramGiB"] = round( + torch.cuda.get_device_properties(0).total_memory / (1024 ** 3), 1) + except Exception as exc: # torch missing/broken + rep["cudaAvailable"] = False + rep["torchError"] = str(exc) + for m in PROHIBITED_MODULES: + rep[m] = "PRESENT (prohibited!)" if m in present_prohibited \ + else "NOT INSTALLED / NOT USED" + rep["qtmeshTextureBaker"] = "enabled (C++ Trellis2Bake)" + rep["qtmeshRasterizer"] = "enabled (C++ Trellis2Bake)" + return rep + + +def load_rgba(path: str): + from PIL import Image + import numpy as np + + img = Image.open(path) + img.load() + if img.mode != "RGBA": + img = img.convert("RGBA") + alpha = np.asarray(img)[:, :, 3] + has_matte = bool((alpha < 255).any()) + return img, has_matte + + +def make_mock_result(seed: int): + """Synthetic generation for plumbing tests: a UV sphere with procedural + PBR attributes and a matching sparse attribute volume. No torch, no GPU, + no TRELLIS.2 needed - exercises the exact interchange/bake/export path.""" + import numpy as np + + rng = np.random.default_rng(seed) + rings, segs = 48, 64 + verts = [] + for r in range(rings + 1): + theta = np.pi * r / rings + for s in range(segs): + phi = 2 * np.pi * s / segs + verts.append(( + 0.4 * np.sin(theta) * np.cos(phi), + 0.4 * np.cos(theta), + 0.4 * np.sin(theta) * np.sin(phi), + )) + verts = np.asarray(verts, dtype=np.float32) + faces = [] + for r in range(rings): + for s in range(segs): + a = r * segs + s + b = r * segs + (s + 1) % segs + c = (r + 1) * segs + s + d = (r + 1) * segs + (s + 1) % segs + if r != 0: + faces.append((a, b, c)) + if r != rings - 1: + faces.append((b, d, c)) + faces = np.asarray(faces, dtype=np.uint32) + + resolution = 64 + voxel_size = 1.0 / resolution + origin = np.array([-0.5, -0.5, -0.5], dtype=np.float32) + # occupied voxels = shell around the sphere surface + ijk = np.unique(((verts - origin) / voxel_size).astype(np.int64), axis=0) + neigh = np.array([(dx, dy, dz) for dx in (-1, 0, 1) for dy in (-1, 0, 1) + for dz in (-1, 0, 1)], dtype=np.int64) + ijk = np.unique((ijk[:, None, :] + neigh[None, :, :]).reshape(-1, 3), axis=0) + ijk = ijk.clip(0, resolution - 1) + ijk = np.unique(ijk, axis=0) + centers = origin + (ijk.astype(np.float32) + 0.5) * voxel_size + # procedural attrs: hue bands by height, metallic top half, rough bottom + h = (centers[:, 1] + 0.5) + attrs = np.zeros((ijk.shape[0], 6), dtype=np.uint8) + attrs[:, 0] = (255 * np.clip(np.abs(np.sin(6.0 * h)), 0, 1)).astype(np.uint8) + attrs[:, 1] = (255 * h).clip(0, 255).astype(np.uint8) + attrs[:, 2] = (255 * (1.0 - h)).clip(0, 255).astype(np.uint8) + attrs[:, 3] = np.where(h > 0.5, 230, 10).astype(np.uint8) # metallic + attrs[:, 4] = (255 * (0.2 + 0.6 * (1.0 - h))).astype(np.uint8) # roughness + attrs[:, 5] = 255 # alpha + del rng + + vcol = np.zeros((verts.shape[0], 4), dtype=np.uint8) + vh = (verts[:, 1] + 0.5) + vcol[:, 0] = (255 * np.clip(np.abs(np.sin(6.0 * vh)), 0, 1)).astype(np.uint8) + vcol[:, 1] = (255 * vh).clip(0, 255).astype(np.uint8) + vcol[:, 2] = (255 * (1.0 - vh)).clip(0, 255).astype(np.uint8) + vcol[:, 3] = 255 + return verts, faces, ijk.astype(np.uint16), attrs, vcol, resolution, voxel_size, origin + + +def run_trellis2(args): + """The real inference path. Imports of torch/trellis2 happen here so + --mock/--check work in torch-free environments.""" + stage("load_model") + os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + import numpy as np + import torch + + if not torch.cuda.is_available(): + fail("CUDA GPU not available. TRELLIS.2 requires an NVIDIA GPU with " + ">=24 GB VRAM (upstream requirement). QtMeshEditor's TripoSR/" + "TripoSG backends remain available without one.") + + # ---- RMBG-2.0 (CC BY-NC) bypass: stub the rembg loader BEFORE pipeline + # construction so the non-commercial model is never downloaded/loaded. + import trellis2.pipelines.rembg as _rembg + + class _NoRembg: # noqa: D401 + def __init__(self, *a, **k): + pass + + def to(self, *a, **k): + pass + + def cuda(self): + pass + + def cpu(self): + pass + + def __call__(self, image): + raise RuntimeError( + "background removal is handled by QtMeshEditor (U^2-Net); the " + "input image must already carry an alpha matte. The upstream " + "default (briaai/RMBG-2.0) is CC BY-NC and is not used.") + + _rembg.BiRefNet = _NoRembg + + from trellis2.pipelines import Trellis2ImageTo3DPipeline + + model = args.model or "microsoft/TRELLIS.2-4B" + pipeline = Trellis2ImageTo3DPipeline.from_pretrained(model) + pipeline.cuda() # low_vram mode staggers the individual models + + stage("preprocess") + image, has_matte = load_rgba(args.input) + if not has_matte and not args.allow_opaque: + fail("input image has no alpha matte; QtMeshEditor should remove the " + "background first (or pass --allow-opaque to proceed - the whole " + "frame will be treated as foreground).") + + pipeline_type = args.pipeline_type or PRESET_PIPELINE_TYPE[args.preset] + sampler_params = {} + if args.steps: + sampler_params = {"steps": int(args.steps)} + + stage("generate") + t0 = time.time() + meshes = pipeline.run( + image, + seed=args.seed, + preprocess_image=True, # model-free on RGBA-with-alpha input + pipeline_type=pipeline_type, + max_num_tokens=args.max_num_tokens, + sparse_structure_sampler_params=sampler_params, + shape_slat_sampler_params=sampler_params, + tex_slat_sampler_params=sampler_params, + ) + gen_seconds = time.time() - t0 + mesh = meshes[0] + + stage("extract") + verts = mesh.vertices.detach().float().cpu().numpy().astype(np.float32) + faces = mesh.faces.detach().cpu().numpy().astype(np.uint32) + coords = mesh.coords.detach().cpu().numpy() + attrs01 = (mesh.attrs.detach().float().cpu().numpy()).clip(0.0, 1.0) + attrs = (attrs01 * 255.0 + 0.5).astype(np.uint8) + resolution = int(round(1.0 / mesh.voxel_size)) + origin = mesh.origin.detach().float().cpu().numpy().astype(np.float32) + + stage("attributes") + try: + vattr01 = mesh.query_vertex_attrs().detach().float().cpu().numpy().clip(0, 1) + vcol = np.empty((verts.shape[0], 4), dtype=np.uint8) + vcol[:, 0:3] = (vattr01[:, 0:3] * 255.0 + 0.5).astype(np.uint8) + vcol[:, 3] = (vattr01[:, 5] * 255.0 + 0.5).astype(np.uint8) + except Exception as exc: # non-fatal: baking samples the volume anyway + log(f"per-vertex attribute query failed (non-fatal): {exc}") + vcol = None + + coord_dtype = np.uint16 if coords.max(initial=0) < 65536 else np.uint32 + return (verts, faces, coords.astype(coord_dtype), attrs, vcol, + resolution, float(mesh.voxel_size), origin, pipeline_type, + gen_seconds) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--input", help="input image (RGBA with alpha matte)") + ap.add_argument("--output", help="output .qtm3d interchange path") + ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--preset", choices=sorted(PRESET_PIPELINE_TYPE), + default="balanced") + ap.add_argument("--pipeline-type", + choices=["512", "1024", "1024_cascade", "1536_cascade"], + help="override the preset's TRELLIS.2 pipeline type") + ap.add_argument("--steps", type=int, default=0, + help="sampler steps override for all three stages " + "(0 = upstream defaults)") + ap.add_argument("--max-num-tokens", type=int, default=49152) + ap.add_argument("--model", default="", + help="HF repo id or local path (default microsoft/TRELLIS.2-4B)") + ap.add_argument("--allow-opaque", action="store_true", + help="proceed even when the input has no alpha matte") + ap.add_argument("--mock", action="store_true", + help="write a synthetic result without loading TRELLIS.2 " + "(plumbing/e2e tests; no GPU or torch needed)") + ap.add_argument("--check", action="store_true", + help="verify the environment and exit (no generation)") + ap.add_argument("--report-deps", action="store_true", + help="print the runtime dependency report (Phase 15)") + ap.add_argument("--strict", action="store_true", + help="fail (instead of warn) if prohibited modules are " + "installed in the environment") + args = ap.parse_args() + + signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) + + present = check_prohibited(args.strict) + if args.report_deps or args.check: + emit(dependency_report(present)) + + if args.check: + problems = [] + for mod in ("torch", "trellis2", "o_voxel", "cumesh", "flex_gemm"): + if importlib.util.find_spec(mod) is None: + problems.append(f"missing module: {mod}") + # o_voxel must import WITHOUT nvdiffrast (install.py patches it) + if not problems: + try: + import o_voxel # noqa: F401 + except Exception as exc: + problems.append(f"import o_voxel failed: {exc}") + if problems: + fail("environment check failed: " + "; ".join(problems)) + emit({"event": "done", "check": "ok"}) + return + + if not args.input or not args.output: + fail("--input and --output are required", 2) + if not os.path.exists(args.input): + fail(f"input image not found: {args.input}", 2) + + try: + if args.mock: + stage("generate") + (verts, faces, coords, attrs, vcol, + resolution, voxel_size, origin) = make_mock_result(args.seed) + pipeline_type, gen_seconds = "mock", 0.0 + else: + (verts, faces, coords, attrs, vcol, resolution, voxel_size, + origin, pipeline_type, gen_seconds) = run_trellis2(args) + + stage("write") + arrays = { + "positions": verts, + "indices": faces, + "voxel_coords": coords, + "voxel_attrs": attrs, + } + if vcol is not None: + arrays["vertex_colors"] = vcol + rt = runtime_info() + qtm3d.write(args.output, arrays, meta={ + "seed": args.seed, + "preset": args.preset, + "pipelineType": pipeline_type, + "resolution": resolution, + "voxelSize": voxel_size, + "origin": [float(x) for x in origin], + "sourceImage": os.path.basename(args.input), + "generationSeconds": round(gen_seconds, 2), + "trellis2Revision": rt.get("trellis2Revision", "unknown"), + "mock": bool(args.mock), + }) + emit({ + "event": "done", + "output": args.output, + "vertexCount": int(verts.shape[0]), + "triangleCount": int(faces.shape[0]), + "voxelCount": int(coords.shape[0]), + "resolution": resolution, + "generationSeconds": round(gen_seconds, 2), + }) + except SystemExit: + raise + except KeyboardInterrupt: + fail("cancelled", 130) + except Exception as exc: # surface a single structured error line + import traceback + traceback.print_exc(file=sys.stderr) + fail(f"{type(exc).__name__}: {exc}") + + +if __name__ == "__main__": + main() diff --git a/ai/trellis2/install.py b/ai/trellis2/install.py new file mode 100644 index 000000000..3c531f1de --- /dev/null +++ b/ai/trellis2/install.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""QtMeshEditor TRELLIS.2 runtime installer. + +Creates an ISOLATED Python environment for the TRELLIS.2 backend (Phase 12 - +nothing here touches QtMeshEditor's own dependencies), pinned to the audited +revisions in docs/trellis2-dependencies.md: + + TRELLIS.2 microsoft/TRELLIS.2 75fbf0183001ed9876c8dbb35de6b68552ee08bd (MIT) + CuMesh JeffreyXiang/CuMesh 12289e1062f0603f2f0d0771b02e1395d247f26f (MIT) + FlexGEMM JeffreyXiang/FlexGEMM 6dd94a859c26ee8246888502eada3dd8ad85532e (MIT) + +PROHIBITED (never installed - NVIDIA Source Code License, research-only): + nvdiffrast and nvdiffrec are prohibited. The upstream setup.sh installs + them for its own GLB texture baking and preview renderers; QtMeshEditor + replaces that functionality with its own C++ rasterizer/baker + (src/ImageTo3D/Trellis2Bake.*). To make the o-voxel package importable + WITHOUT the prohibited nvdiffrast, this installer patches the MIT file + o_voxel/__init__.py to import its `postprocess` module lazily (that + module imports the prohibited nvdiffrast at top level but is never used + by QtMeshEditor). + +Layout under --dest (default: /trellis2): + env/ the virtualenv + TRELLIS.2/ the pinned upstream checkout (trellis2 + o-voxel packages) + CuMesh/ pinned checkout (built + installed into env) + FlexGEMM/ pinned checkout (built + installed into env) + runtime.json marker read by QtMeshEditor to detect the runtime + +Requires: Linux, Python >= 3.10, git, an NVIDIA GPU (>= 24 GB VRAM +recommended), CUDA toolkit 12.4 for building the extensions. + +Model weights are NOT fetched here; TRELLIS.2-4B (MIT) downloads on first +generation via huggingface_hub. The DINOv3 image encoder +(facebook/dinov3-vitl16-pretrain-lvd1689m) is GATED - accept Meta's DINOv3 +License on Hugging Face and `huggingface-cli login` before first use. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import venv + +TRELLIS2_REPO = "https://github.com/microsoft/TRELLIS.2.git" +TRELLIS2_REV = "75fbf0183001ed9876c8dbb35de6b68552ee08bd" +CUMESH_REPO = "https://github.com/JeffreyXiang/CuMesh.git" +CUMESH_REV = "12289e1062f0603f2f0d0771b02e1395d247f26f" +FLEXGEMM_REPO = "https://github.com/JeffreyXiang/FlexGEMM.git" +FLEXGEMM_REV = "6dd94a859c26ee8246888502eada3dd8ad85532e" + +TORCH_SPEC = ["torch==2.6.0", "torchvision==0.21.0"] +TORCH_INDEX = "https://download.pytorch.org/whl/cu124" +FLASH_ATTN_SPEC = "flash-attn==2.7.3" + +# sha256 of the pristine o-voxel/o_voxel/__init__.py at TRELLIS2_REV. +OVOXEL_INIT_SHA256 = "ca30e1545d11f3e862b7d11b31af6200427c91482f11d8b543982d692cd8588b" +OVOXEL_INIT_PATCHED = '''"""o_voxel package init - PATCHED by QtMeshEditor (ai/trellis2/install.py). + +Upstream imports `postprocess` eagerly, and o_voxel/postprocess.py imports the +prohibited nvdiffrast at module top. nvdiffrast is under the NVIDIA Source Code +License (research/evaluation only) and is prohibited in QtMeshEditor's +TRELLIS.2 integration, so `postprocess` is made lazy: the core conversion / +IO / rasterize / serialize APIs work without the prohibited nvdiffrast, and +`o_voxel.postprocess` still resolves for users who have it. This patch +modifies MIT-licensed Microsoft code only - no NVIDIA source is involved. +""" +from . import ( + convert, + io, + rasterize, + serialize +) + + +def __getattr__(name): + if name == 'postprocess': + import importlib + return importlib.import_module('.postprocess', __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +''' + + +def log(msg: str) -> None: + print(f"[trellis2-install] {msg}", flush=True) + + +def run(cmd: list[str], **kw) -> None: + log("$ " + " ".join(cmd)) + subprocess.run(cmd, check=True, **kw) + + +def default_dest() -> str: + if os.environ.get("QTMESH_TRELLIS2_ENV"): + return os.environ["QTMESH_TRELLIS2_ENV"] + if sys.platform.startswith("linux"): + base = os.environ.get("XDG_DATA_HOME", + os.path.expanduser("~/.local/share")) + return os.path.join(base, "QtMeshEditor", "trellis2") + if sys.platform == "darwin": + return os.path.expanduser( + "~/Library/Application Support/QtMeshEditor/trellis2") + return os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), + "QtMeshEditor", "trellis2") + + +def clone_pinned(repo: str, rev: str, dest: str) -> None: + if os.path.isdir(os.path.join(dest, ".git")): + head = subprocess.run(["git", "-C", dest, "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() + if head == rev: + log(f"{dest}: already at {rev[:12]}") + return + run(["git", "-C", dest, "fetch", "origin", rev]) + run(["git", "-C", dest, "checkout", "--detach", rev]) + return + os.makedirs(dest, exist_ok=True) + run(["git", "init", "-q", dest]) + run(["git", "-C", dest, "remote", "add", "origin", repo]) + run(["git", "-C", dest, "fetch", "--depth", "1", "origin", rev]) + run(["git", "-C", dest, "checkout", "--detach", "FETCH_HEAD"]) + + +def patch_ovoxel_init(trellis_dir: str) -> None: + path = os.path.join(trellis_dir, "o-voxel", "o_voxel", "__init__.py") + with open(path, "rb") as f: + data = f.read() + digest = hashlib.sha256(data).hexdigest() + if b"PATCHED by QtMeshEditor" in data: + log("o_voxel/__init__.py already patched") + return + if digest != OVOXEL_INIT_SHA256: + raise SystemExit( + f"o_voxel/__init__.py has unexpected content (sha256 {digest}); " + "the pinned revision must have changed - re-audit before patching " + "(docs/trellis2-dependencies.md).") + with open(path, "w", encoding="utf-8") as f: + f.write(OVOXEL_INIT_PATCHED) + log("patched o_voxel/__init__.py (lazy postprocess - " + "the prohibited nvdiffrast is no longer imported)") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--dest", default=default_dest(), + help="install root (default: %(default)s)") + ap.add_argument("--attn", choices=["flash-attn", "xformers"], + default="flash-attn", + help="attention backend to install") + ap.add_argument("--skip-torch", action="store_true", + help="assume torch/torchvision already in the env") + ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 1), + help="parallel build jobs for the CUDA extensions") + args = ap.parse_args() + + if not sys.platform.startswith("linux"): + log("WARNING: TRELLIS.2 upstream supports Linux + NVIDIA GPUs only; " + "continuing, but generation will not work on this platform.") + if sys.version_info < (3, 10): + raise SystemExit("Python >= 3.10 required") + if shutil.which("git") is None: + raise SystemExit("git is required") + if shutil.which("nvidia-smi") is None: + log("WARNING: nvidia-smi not found - no NVIDIA GPU detected. " + "TRELLIS.2 needs an NVIDIA GPU (>= 24 GB VRAM recommended).") + + dest = os.path.abspath(args.dest) + env_dir = os.path.join(dest, "env") + os.makedirs(dest, exist_ok=True) + log(f"installing into {dest}") + + # 1. venv --------------------------------------------------------------- + py = os.path.join(env_dir, "bin", "python") + if not os.path.exists(py): + log("creating virtualenv") + venv.EnvBuilder(with_pip=True, upgrade_deps=True).create(env_dir) + pip = [py, "-m", "pip", "install", "--no-input"] + + # 2. torch + base requirements ------------------------------------------ + if not args.skip_torch: + run(pip + ["--index-url", TORCH_INDEX] + TORCH_SPEC) + req = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "requirements.txt") + run(pip + ["-r", req]) + if args.attn == "flash-attn": + # needs torch importable at build time + run(pip + ["--no-build-isolation", FLASH_ATTN_SPEC]) + else: + run(pip + ["xformers"]) + log("remember to run generation with ATTN_BACKEND=xformers") + + # 3. pinned checkouts ----------------------------------------------------- + trellis_dir = os.path.join(dest, "TRELLIS.2") + clone_pinned(TRELLIS2_REPO, TRELLIS2_REV, trellis_dir) + patch_ovoxel_init(trellis_dir) + cumesh_dir = os.path.join(dest, "CuMesh") + clone_pinned(CUMESH_REPO, CUMESH_REV, cumesh_dir) + flexgemm_dir = os.path.join(dest, "FlexGEMM") + clone_pinned(FLEXGEMM_REPO, FLEXGEMM_REV, flexgemm_dir) + + # NOTE: never `pip install cumesh` - the PyPI project of that name is an + # unrelated, unlicensed package (see docs/trellis2-dependencies.md). + build_env = dict(os.environ, MAX_JOBS=str(args.jobs)) + run(pip + ["--no-build-isolation", os.path.join(trellis_dir, "o-voxel")], + env=build_env) + run(pip + ["--no-build-isolation", cumesh_dir], env=build_env) + run(pip + ["--no-build-isolation", flexgemm_dir], env=build_env) + + # 4. make `trellis2` importable from the pinned checkout ------------------ + site = subprocess.run( + [py, "-c", "import sysconfig;print(sysconfig.get_paths()['purelib'])"], + capture_output=True, text=True, check=True).stdout.strip() + with open(os.path.join(site, "qtmesh_trellis2.pth"), "w") as f: + f.write(trellis_dir + "\n") + + # 5. prohibited-module check (Phase 12/13) -------------------------------- + probe = subprocess.run( + [py, "-c", + "import importlib.util as u;" + "print(int(u.find_spec('nvdiffrast') is not None)," + "int(u.find_spec('nvdiffrec') is not None)," + "int(u.find_spec('nvdiffrec_render') is not None))"], + capture_output=True, text=True, check=True).stdout.split() + if any(p != "0" for p in probe): + log("WARNING: a prohibited NVIDIA research-only package (nvdiffrast/" + "nvdiffrec) is present in the environment. QtMeshEditor never " + "uses it; uninstall it to keep the environment license-clean.") + + # 6. verify the core imports WITHOUT nvdiffrast --------------------------- + log("verifying imports (torch-free modules)") + run([py, "-c", "import o_voxel, o_voxel.convert; print('o_voxel ok')"]) + run([py, "-c", "import trellis2; print('trellis2 ok')"]) + + # 7. copy the sidecar scripts + write the runtime marker ------------------ + here = os.path.dirname(os.path.abspath(__file__)) + for name in ("generate.py", "qtm3d.py", "requirements.txt", + "THIRD_PARTY_LICENSES.md"): + src = os.path.join(here, name) + if os.path.exists(src) and os.path.abspath(here) != dest: + shutil.copy2(src, os.path.join(dest, name)) + with open(os.path.join(dest, "runtime.json"), "w") as f: + json.dump({ + "schema": "qtmesh-trellis2-runtime-v1", + "python": os.path.join(env_dir, "bin", "python"), + "generate": os.path.join(dest, "generate.py"), + "trellis2Revision": TRELLIS2_REV, + "cumeshRevision": CUMESH_REV, + "flexgemmRevision": FLEXGEMM_REV, + "attnBackend": args.attn, + "platform": platform.platform(), + }, f, indent=2) + log("done. QtMeshEditor will auto-detect this runtime " + "(or set QTMESH_TRELLIS2_ENV / QSettings ai/trellis2Env to " + f"{dest}).") + log("First generation downloads microsoft/TRELLIS.2-4B (MIT, ~19 GB) and " + "the GATED facebook/dinov3-vitl16 encoder - accept Meta's DINOv3 " + "License on Hugging Face and `huggingface-cli login` first.") + + +if __name__ == "__main__": + main() diff --git a/ai/trellis2/qtm3d.py b/ai/trellis2/qtm3d.py new file mode 100644 index 000000000..98ee63218 --- /dev/null +++ b/ai/trellis2/qtm3d.py @@ -0,0 +1,135 @@ +"""QTM3D interchange writer/reader. + +The QtMeshEditor <-> TRELLIS.2 sidecar interchange container. One little-endian +binary file carrying named typed arrays plus a JSON manifest. Deliberately +trivial so the C++ reader (src/ImageTo3D/Trellis2Interchange.{h,cpp}) needs no +zip/npz/hdf5 dependency. + +Layout: + bytes 0..7 magic b"QTMESH3D" + u32 version (currently 1) + u32 json manifest byte length J + J bytes UTF-8 JSON manifest + padding zeros up to the next 16-byte boundary (relative to file start) + blobs raw little-endian array data; each array's "offset" in the + manifest is relative to the START OF THE BLOB SECTION (the + 16-byte boundary after the JSON), and every blob starts on a + 16-byte boundary. + +Manifest schema (JSON object): + { + "generator": "qtmesh-trellis2", + "formatVersion": 1, + "meta": { ... free-form generation metadata ... }, + "arrays": { + "": {"dtype": "f32|f16|u8|u16|u32|i32", + "shape": [..], "offset": N, "byteLength": N} + } + } + +Array conventions used by the TRELLIS.2 backend: + positions f32 [N,3] mesh vertices (TRELLIS space, aabb [-0.5,0.5]^3) + indices u32 [M,3] triangle indices + voxel_coords u16 [L,3] occupied sparse-voxel integer coordinates + voxel_attrs u8 [L,6] base_color.rgb, metallic, roughness, alpha (0..255) + vertex_colors u8 [N,4] per-vertex base_color.rgb + alpha (optional) +meta keys: resolution (int grid res), voxelSize (float), origin ([3] floats), + seed, preset, pipelineType, trellis2Revision, sourceImage. + +This file must never import (or require) the prohibited nvdiffrast/nvdiffrec +NVIDIA-licensed packages - they are prohibited across QtMeshEditor's TRELLIS.2 +integration (see docs/trellis2-dependencies.md). +""" + +from __future__ import annotations + +import json +import struct + +import numpy as np + +MAGIC = b"QTMESH3D" +VERSION = 1 + +_DTYPES = { + "f32": np.dtype(" str: + dt = arr.dtype.newbyteorder("<") + name = _DTYPE_NAMES.get(np.dtype(dt)) + if name is None: + raise ValueError(f"unsupported dtype for QTM3D: {arr.dtype}") + return name + + +def _align16(n: int) -> int: + return (n + 15) & ~15 + + +def write(path: str, arrays: dict, meta: dict | None = None) -> None: + """Write a QTM3D file. `arrays` maps name -> numpy array.""" + entries = {} + blobs = [] + offset = 0 + for name, arr in arrays.items(): + arr = np.ascontiguousarray(arr) + data = arr.astype(arr.dtype.newbyteorder("<"), copy=False).tobytes() + entries[name] = { + "dtype": _dtype_name(arr), + "shape": list(arr.shape), + "offset": offset, + "byteLength": len(data), + } + blobs.append((offset, data)) + offset = _align16(offset + len(data)) + + manifest = { + "generator": "qtmesh-trellis2", + "formatVersion": VERSION, + "meta": meta or {}, + "arrays": entries, + } + mjson = json.dumps(manifest, separators=(",", ":")).encode("utf-8") + + header_len = len(MAGIC) + 8 + len(mjson) + blob_base = _align16(header_len) + with open(path, "wb") as f: + f.write(MAGIC) + f.write(struct.pack(" pos: + f.write(b"\x00" * (off - pos)) + pos = off + f.write(data) + pos += len(data) + + +def read(path: str) -> tuple[dict, dict]: + """Read a QTM3D file -> (arrays dict of numpy arrays, meta dict).""" + with open(path, "rb") as f: + blob = f.read() + if blob[:8] != MAGIC: + raise ValueError("not a QTM3D file") + version, jlen = struct.unpack_from("=1.26,<3 +pillow>=10 +# DINOv3ViTModel support landed in transformers 4.56 +transformers>=4.56,<5 +huggingface_hub>=0.27 +safetensors>=0.4 +# LGPL-3.0 (pure-Python, imported unmodified) - required by o_voxel.rasterize; +# the only copyleft item in the runtime set, flagged in the audit doc. +easydict==1.13 +ninja +packaging diff --git a/docs/TRELLIS2.md b/docs/TRELLIS2.md new file mode 100644 index 000000000..38849df0f --- /dev/null +++ b/docs/TRELLIS2.md @@ -0,0 +1,132 @@ +# TRELLIS.2 image-to-3D backend + +TRELLIS.2 (Microsoft, MIT code + MIT weights) is QtMeshEditor's highest-quality +image-to-3D backend, and the **default** one whenever its runtime is installed. It sits +next to the local ONNX backends: + +| Backend | Runs | Quality | Output | +|---|---|---|---| +| **TRELLIS.2** | local Python sidecar, Linux + NVIDIA GPU (≥24 GB VRAM rec.) | highest | full PBR (base color + metallic + roughness + alpha), game-ready presets | +| TripoSR | in-process ONNX, any machine | fast tier | diffuse (+#404 synthesized PBR) | +| TripoSG | in-process ONNX, any machine | best local geometry | geometry (+AI texture pass) | + +## Architecture — who does what + +**TRELLIS.2 generates; QtMeshEditor makes the asset.** + +``` +input image ── U²-Net alpha matte (QtMeshEditor, Apache-2.0) ──► RGBA + ──► ai/trellis2/generate.py (inference ONLY: DINOv3 cond → sparse structure + → shape SLat → tex SLat → raw mesh + sparse PBR attribute volume) + ──► QTM3D interchange file (Trellis2Interchange) + ──► QtMeshEditor C++ (Trellis2Bake): + weld → remove debris components → simplify (game-ready presets) + → xatlas UV unwrap → rasterize charts → project each texel to the + closest point on the full-res source surface → trilinearly sample + the attribute volume → bake base color (RGBA) + roughness + + metallic + tangent-space normal map → dilate seams + ──► Ogre scene / GLB / FBX / any Assimp-supported export +``` + +The full-resolution generation is preserved as a `*_source.qtm3d` sidecar (next to CLI/MCP +exports; under `/generated_sources/` for GUI runs) so textures and LODs can be +re-baked later without re-running inference. + +### The NVIDIA exclusion + +The upstream reference implementation uses **nvdiffrast** (UV rasterization/texture bake, +preview rendering) and **nvdiffrec** (environment-light PBR previews). Both are under the +NVIDIA Source Code License — *research or evaluation use only* — which is incompatible +with QtMeshEditor's commercial redistribution, so this integration **excludes them +entirely**: never installed, never imported, never invoked; the sidecar warns if they are +unexpectedly present, `scripts/check-trellis2-restricted-deps.sh` + `Trellis2GuardTest` +gate CI, and everything they did is replaced by QtMeshEditor's own code +(`src/ImageTo3D/Trellis2Bake.{h,cpp}` — xatlas, meshoptimizer, a conventional barycentric +UV-space rasterizer, Ericson closest-point queries and trilinear sparse-volume sampling; +previews come from the existing Ogre/RTSS + HDR/IBL renderer). Two more license traps are +bypassed: the upstream default background remover `briaai/RMBG-2.0` is **CC BY-NC** and is +never downloaded (QtMeshEditor's own U²-Net produces the alpha matte), and `pip install +cumesh` would fetch an unrelated unlicensed PyPI package (CuMesh is built from the pinned +MIT checkout instead). + +**Accurate license statement:** TRELLIS.2 code and TRELLIS.2-4B weights are MIT; the +required DINOv3 image encoder is under Meta's **DINOv3 License** (commercial use permitted, +gated download under the user's own HF account, "Built with DINOv3" attribution); the +easydict transitive dependency is LGPL-3.0 (pure-Python). It would be wrong to call the +whole stack "entirely MIT". Full audit: [`docs/trellis2-dependencies.md`](trellis2-dependencies.md). + +## Runtime flavors + +The backend has **two interchangeable runtimes**; `Trellis2Predictor` prefers trellis.cpp +when both are present: + +| Flavor | Stack | Platforms | Discovery | +|---|---|---|---| +| **trellis.cpp** (preferred) | C++/GGML ([fork](https://github.com/fernandotonon/trellis.cpp), branch `macos-metal-support`) — CUDA / Vulkan / **Metal** / CPU, no Python, GGUF weights | Linux, Windows, **macOS/Apple Silicon** | env `QTMESH_TRELLIS2_CLI` → QSettings `ai/trellis2Cli` → `trellis-cli` on PATH; models: `QTMESH_TRELLIS2_CLI_MODELS` / `ai/trellis2CliModels` / `/models` (needs the 512-pipeline GGUFs from [`ilintar/trellis2-gguf`](https://huggingface.co/ilintar/trellis2-gguf)) | +| Python sidecar | upstream-exact PyTorch/CUDA (`ai/trellis2/`) | Linux + NVIDIA | as below | + +QtMeshEditor invokes `trellis-cli --dump-post` (added in the fork): trellis.cpp emits the +RAW decoded mesh + sparse PBR volume and exits before its own remesh/UV/bake — QtMeshEditor +keeps the game-ready + native-bake pipeline either way. Presets map to `--res` (fast=512, +balanced=1024, high=1536); with only the 512 GGUFs installed the backend drops to the 512 +pipeline with a warning. `--mock` runs always route to the Python sidecar. + +## Install (Linux + NVIDIA GPU) + +```bash +python3 ai/trellis2/install.py # → ~/.local/share/QtMeshEditor/trellis2 +huggingface-cli login # DINOv3 is gated — accept Meta's terms on HF first +``` + +QtMeshEditor auto-detects the default location; override with `QTMESH_TRELLIS2_ENV` (or +QSettings `ai/trellis2Env`), and the interpreter with `QTMESH_TRELLIS2_PYTHON` +(`ai/trellis2Python`). The ~19 GB TRELLIS.2-4B weights download on the first generation. +Everything is isolated in that directory — nothing touches QtMeshEditor's own dependencies. + +Health checks: + +```bash +$ENV/env/bin/python $ENV/generate.py --check # environment probe +$ENV/env/bin/python $ENV/generate.py --check --report-deps # Phase 15 dependency report +``` + +The report prints the loaded stack, ending with +`nvdiffrast: NOT INSTALLED / NOT USED` / `nvdiffrec: NOT INSTALLED / NOT USED`. + +## Use + +**GUI:** Object mode → Mode Tools → *AI: Image → 3D*. The Backend combo lists +*TRELLIS.2 (high quality)* first and preselects it when the runtime is present. Options: +Quality (*Fast* = 512 / *Balanced* = 1024 cascade / *High* = 1536 cascade), Mesh +(*Original*, *Game Low ~10k*, *Game Medium ~25k*, *Game High ~50k* triangles), Texture +(1024/2048/4096), plus the shared Remove-background / Bake / PBR / Upscale toggles. + +**CLI:** + +```bash +qtmesh generate3d photo.png -o out.glb # trellis2 when installed, else triposr +qtmesh generate3d photo.png -o out.glb --backend trellis2 \ + --preset high --target-tris 25000 --texture-size 4096 --seed 7 +``` + +**MCP:** `generate_mesh_from_image` with `backend: "trellis2"` (`seed`, `preset`, +`target_tris` args; the response carries `backend` and `sourcePath`). + +## Errors you may see + +- *"TRELLIS.2 runtime not installed"* — run `ai/trellis2/install.py` or point + `QTMESH_TRELLIS2_ENV` at it. +- *"CUDA GPU not available"* — the sidecar needs an NVIDIA GPU; use TripoSR/TripoSG locally. +- Hugging Face 401/403 on DINOv3 — accept the DINOv3 License on HF and `huggingface-cli login`. +- *"input image has no alpha matte"* — enable Remove background (or supply an RGBA image). +- Python stack traces stay on stderr (developer log); surfaces show the single structured + error message. + +## Testing without a GPU + +The sidecar's `--mock` mode (or env `QTMESH_TRELLIS2_MOCK=1`) generates a synthetic +sphere + attribute volume with no torch/CUDA, exercising the exact interchange → +game-ready → bake → export path end-to-end. Unit tests: `Trellis2Interchange_test.cpp`, +`Trellis2Bake_test.cpp`, `Trellis2Predictor_test.cpp`, `Trellis2Guard_test.cpp`. + +*Built with DINOv3.* diff --git a/docs/trellis2-dependencies.md b/docs/trellis2-dependencies.md new file mode 100644 index 000000000..a68cae3ef --- /dev/null +++ b/docs/trellis2-dependencies.md @@ -0,0 +1,161 @@ +# TRELLIS.2 integration — dependency & license audit + +Status: **authoritative record** for the `qtmesh generate3d --backend trellis2` integration. +Audited: 2026-08-30, against the pinned upstream revisions below. Re-audit whenever a pin moves. + +QtMeshEditor's TRELLIS.2 integration is designed around one hard constraint: + +> **NVIDIA nvdiffrast and nvdiffrec are excluded — not installed, not imported, not +> dynamically loaded, not bundled, not invoked indirectly, and not ported/translated.** +> Both are under the *NVIDIA Source Code License*, which limits use to +> "research or evaluation purposes only" for everyone but NVIDIA. That is incompatible +> with QtMeshEditor's permissive commercial redistribution (Homebrew / Snap / WinGet / +> Docker / Marketplace). + +Everything those libraries do for the upstream reference implementation (UV-space +rasterization, texture baking, PBR preview rendering) is performed by **QtMeshEditor's own +C++ code** (`src/ImageTo3D/Trellis2Bake.{h,cpp}`, xatlas + meshoptimizer + a conventional +scanline/barycentric rasterizer and trilinear sparse-volume sampler — standard, publicly +documented graphics algorithms; no NVIDIA source was read, copied, or translated for it). + +--- + +## 1. Pinned upstream revisions + +| Component | Source | Pinned revision | License | +|---|---|---|---| +| TRELLIS.2 code (incl. in-repo `o-voxel` package) | | `75fbf0183001ed9876c8dbb35de6b68552ee08bd` (main, 2026-06-05) | **MIT** | +| TRELLIS.2-4B weights (9 safetensors, ≈18.9 GB) | | `af44b45f2e35a493886929c6d786e563ec68364d` | **MIT** (model card `license: mit`) | +| TRELLIS 1 sparse-structure decoder (`ss_dec_conv3d_16l8_fp16`, pulled by `pipeline.json`) | | `25e0d31ffbebe4b5a97464dd851910efc3002d96` | **MIT** | +| CuMesh | | pinned in `ai/trellis2/install.py` | **MIT** | +| FlexGEMM | | pinned in `ai/trellis2/install.py` | **MIT** | +| DINOv3 image encoder (`facebook/dinov3-vitl16-pretrain-lvd1689m`) | | HF-gated; downloaded by the user's own HF account | **DINOv3 License** (Meta, custom — see §4) | + +> ⚠️ `cumesh` on PyPI (0.1.0, author "Congjie He") is an **unrelated project with no license**. +> Never `pip install cumesh` — `install.py` builds JeffreyXiang/CuMesh from source at the pin. + +## 2. Runtime dependency table + +"Redistributed" = shipped inside QtMeshEditor binaries/packages. **Nothing in this table is +redistributed** — the whole Python environment is user-installed into +`/trellis2/` by `ai/trellis2/install.py` (the same "downloads on first use" stance +as every other AI model in this project). "Required" = required at runtime for the TRELLIS.2 +backend specifically (all other QtMeshEditor features work without any of this). + +| Dependency | Purpose | Version/pin | License | Redistributed | Downloaded separately | Required at runtime | Commercial use | +|---|---|---|---|---|---|---|---| +| TRELLIS.2 (`trellis2` pkg) | generation pipeline | `75fbf018…` | MIT | no | yes (git) | yes | ✅ | +| `o-voxel` (in-repo) | flexible-dual-grid → mesh extraction, sparse attr volume | same repo pin | MIT (repo LICENSE; vendors Eigen, MPL-2.0) | no | yes | yes | ✅ | +| CuMesh | GPU mesh cleanup (`fill_holes` in `decode_latent`), simplify | install.py pin | MIT | no | yes (git) | yes | ✅ | +| FlexGEMM | sparse conv backend + `grid_sample_3d` | install.py pin | MIT | no | yes (git) | yes | ✅ | +| PyTorch + torchvision (CUDA) | tensor runtime | 2.6.0 / 0.21.0 cu124 | BSD-3-Clause | no | yes (pip) | yes | ✅ | +| flash-attn | default attention backend | 2.7.3 | BSD-3-Clause | no | yes (pip) | yes (or xformers) | ✅ | +| xformers | alternative attention backend | optional | BSD-3-Clause | no | optional | no | ✅ | +| transformers | loads DINOv3 | setup.sh floating; install.py pins | Apache-2.0 | no | yes | yes | ✅ | +| huggingface_hub | weight downloads | (transformers dep) | Apache-2.0 | no | yes | yes | ✅ | +| numpy, pillow | array/image IO | — | BSD / HPND | no | yes | yes | ✅ | +| easydict | config dicts (used by `o_voxel.rasterize`) | 1.13 | **LGPL-3.0** ⚠️ | no | yes | yes (transitive import) | ✅ (pure-Python, imported unmodified from a user-installed env; flagged as the only copyleft item — see §5) | +| trimesh | upstream GLB assembly | not installed | MIT | no | no | **no** (QtMeshEditor writes the asset) | ✅ | +| utils3d | upstream renderers/datasets only | not installed | MIT | no | no | **no** | ✅ | +| opencv / imageio / imageio-ffmpeg / gradio / kornia / timm / lpips / pandas / tensorboard | upstream demos, training, video previews | not installed | various permissive (+LGPL ffmpeg binary) | no | no | **no** | n/a | +| spconv | alternative sparse-conv backend | not installed | Apache-2.0 | no | no | no (default is FlexGEMM) | ✅ | +| kaolin, diffusers, rembg (PyPI) | **not used anywhere** by TRELLIS.2 | — | — | no | no | no | n/a | +| **nvdiffrast** | upstream UV rasterization in `o_voxel.postprocess.to_glb` + preview renderers | **EXCLUDED** | NVIDIA Source Code License (1-Way Commercial) — *"non-commercially… research or evaluation purposes only"* | **never** | **never** | **no — replaced by `Trellis2Bake`** | ❌ | +| **nvdiffrec** (`nvdiffrec_render` fork) | upstream env-map lighting for PBR previews (`pbr_mesh_renderer`) | **EXCLUDED** | NVIDIA Source Code License for nvdiffrec (same restriction) | **never** | **never** | **no — QtMeshEditor's HDR/IBL renderer covers previews** | ❌ | + +### Model weights + +| Weights | Purpose | License | Bundled? | Commercial use | +|---|---|---|---|---| +| `microsoft/TRELLIS.2-4B` (rev `af44b45f…`) | flow models + shape/tex VAEs | MIT | no — HF download on install | ✅ | +| `microsoft/TRELLIS-image-large` `ss_dec_conv3d_16l8_fp16` only (rev `25e0d31f…`) | sparse-structure decoder referenced by `pipeline.json` | MIT | no — HF download | ✅ | +| `facebook/dinov3-vitl16-pretrain-lvd1689m` | image conditioning encoder | **DINOv3 License** (custom Meta) | no — **gated** HF download by the user | ✅ with conditions (§4) | +| `briaai/RMBG-2.0` | upstream default background remover (named in `pipeline.json`) | **CC BY-NC 4.0** ❌ | **never downloaded or loaded** — bypassed (§3) | ❌ non-commercial | +| `ZhengPeng7/BiRefNet` | MIT alternative background remover | MIT | not used (QtMeshEditor does its own bg removal) | ✅ | +| U²-Net (`u2net.onnx`, already shipped-on-demand by QtMeshEditor #764) | the background removal actually used | Apache-2.0 code, permissive weights | existing on-demand download | ✅ | + +## 3. Where nvdiffrast/nvdiffrec live upstream, and how each use is avoided + +Complete grep of the pinned revision (`grep -rn 'nvdiffrast\|nvdiffrec' --include=*.py`): + +| Upstream site | What it does | How QtMeshEditor avoids it | +|---|---|---| +| `o-voxel/o_voxel/postprocess.py` (top-level `import nvdiffrast.torch`) | `to_glb()`: rasterize UV atlas → texel 3D positions → BVH snap to hi-res mesh → trilinear volume sample → trimesh GLB | **Never called.** `generate.py` exports raw vertices/faces + the sparse attribute volume to the QTM3D interchange; UV unwrap, rasterization, sampling and baking happen in C++ (`Trellis2Bake`). `install.py` patches the MIT file `o_voxel/__init__.py` to import `postprocess` lazily, so `import o_voxel` no longer requires nvdiffrast to be installed at all. | +| `trellis2/renderers/mesh_renderer.py`, `pbr_mesh_renderer.py` (lazy imports; `pbr_mesh_renderer` also imports `nvdiffrec_render.light`) | turntable/PBR preview videos | **Never imported** — `trellis2/renderers/__init__.py` is lazy (`__getattr__`); `generate.py` never touches renderers. Previews come from QtMeshEditor's own Ogre/RTSS + HDR/IBL pipeline. | +| `trellis2/pipelines/trellis2_texturing.py` (top-level import) | the separate "texture an existing mesh" pipeline | **Never imported** — `trellis2/pipelines/__init__.py` is lazy; only `Trellis2ImageTo3DPipeline` is loaded. | +| `example.py` / `app.py` | demo scripts (comment: 2^24-vertex nvdiffrast limit) | not used. | + +The core generation chain — image → DINOv3 cond → sparse-structure flow → shape SLat flow → +`shape_slat_decoder` → `o_voxel.convert.flexible_dual_grid_to_mesh` (Eigen QEF dual grid, +**not** FlexiCubes; zero NVIDIA code, verified by grep for `flexicubes` and NVIDIA copyright +headers) → tex SLat flow → `tex_slat_decoder` → `MeshWithVoxel` — **contains no nvdiffrast or +nvdiffrec call**. `generate.py` refuses to start if either module is importable in strict +mode, and reports their absence in its dependency report (Phase 15). + +### RMBG-2.0 (non-commercial) bypass + +`TRELLIS.2-4B/pipeline.json` sets the rembg model to `briaai/RMBG-2.0` (CC BY-NC 4.0), and +the upstream `BiRefNet.__init__` downloads it **eagerly at pipeline construction**. The +QtMeshEditor sidecar therefore: + +1. replaces `trellis2.pipelines.rembg.BiRefNet` with an inert stub **before** + `from_pretrained` runs, so the weights are never downloaded or loaded; +2. always feeds the pipeline an **RGBA image whose alpha was produced by QtMeshEditor's own + U²-Net background remover** (Apache-2.0, already part of #764) — upstream's + `preprocess_image()` uses a supplied alpha channel directly and never calls the rembg + model on such input. + +## 4. DINOv3 — the one non-MIT required model (be precise about this) + +The shipped TRELLIS.2-4B `pipeline.json` conditions on +`facebook/dinov3-vitl16-pretrain-lvd1689m` loaded via `transformers.DINOv3ViTModel`. The +checkpoint was trained against this embedding space; **swapping in CLIP/DINOv2 would break +generation**, so it is not replaced. + +- License: **DINOv3 License** (Meta, custom): commercial use **permitted**; redistribution + permitted **with conditions** (include the license, display **"Built with DINOv3"**, + derivatives inherit the license); acceptable-use policy and export-control restrictions + apply; the HF repo is **gated** (user must accept terms and use their own HF token). +- QtMeshEditor **does not redistribute** DINOv3. `install.py`/first run download it under the + *user's* HF account after they accept Meta's terms. The "Built with DINOv3" notice appears + in `ai/trellis2/THIRD_PARTY_LICENSES.md` and `docs/TRELLIS2.md`. +- Consequence: it is **incorrect** to describe the TRELLIS.2 integration as "entirely MIT". + The accurate statement is: *TRELLIS.2 code and weights are MIT; the required DINOv3 image + encoder is under Meta's DINOv3 License (commercial use permitted, gated download, + attribution required); NVIDIA nvdiffrast/nvdiffrec are excluded entirely.* + +## 5. Flagged items (yellow) + +- **easydict (LGPL-3.0)** — pure-Python, imported unmodified from the user-installed + environment (the standard LGPL-compliant usage pattern for interpreted code); the only + copyleft item in the runtime set. It is required because `o_voxel.rasterize` imports it at + module level. Tracked follow-up: upstream a lazy import or vendor a ~20-line permissive + replacement into the o_voxel patch if this ever becomes a distribution concern. +- **imageio-ffmpeg** — not installed (video previews are not used); noted only because + upstream's `setup.sh` installs it. +- **DINOv3** — see §4. + +## 6. Prohibited-dependency enforcement + +- `ai/trellis2/requirements.txt` and `install.py` never reference nvdiffrast/nvdiffrec + (except in prohibition comments). +- `generate.py` startup check: warns (and in `--strict` mode refuses to run) if + `nvdiffrast` or `nvdiffrec`/`nvdiffrec_render` is importable in the environment; its + `--report-deps` output prints `nvdiffrast: NOT INSTALLED / NOT USED`. +- `scripts/check-trellis2-restricted-deps.sh` — CI grep gate over `ai/trellis2/` and the + C++ integration sources; fails on any non-allowlisted mention. +- `Trellis2GuardTest` (Google Test, runs in the normal Linux CI test job) re-checks the same + invariants from the built test binary. + +Allowed mentions of the two names: license documentation (this file, +`ai/trellis2/THIRD_PARTY_LICENSES.md`, `docs/TRELLIS2.md`, `THIRD_PARTY_AI_MODELS.md`), +prohibition comments/guards in `ai/trellis2/*.py`, the CI check script, and the guard test. + +## 7. GPU / platform requirements (upstream) + +Linux + NVIDIA GPU with **≥ 24 GB VRAM** (verified on A100/H100 upstream; `low_vram: true` +staggers models CPU↔GPU), CUDA 12.4, PyTorch 2.6.0+cu124, flash-attn (or xformers). No CPU, +no Apple Silicon path — on machines without a suitable GPU the backend reports "runtime not +available" and QtMeshEditor's TripoSR/TripoSG ONNX backends remain the local option. +Using CUDA/PyTorch is fine license-wise (BSD-3); the exclusion above concerns only the +research-only NVIDIA libraries, not ordinary GPU runtimes. diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index a31b812b7..6f0299e63 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -1915,6 +1915,9 @@ Rectangle { // higher = more detail but much slower. Labels flag the trade-off. Row { spacing: 6 + // Marching-cubes grid — local ONNX backends only (TRELLIS.2's + // resolution comes from its Quality preset instead). + visible: !mgBackendCombo.t2Selected Text { text: "Resolution" color: PropertiesPanelController.textColor @@ -1935,10 +1938,12 @@ Rectangle { } } - // Backend: TripoSR (fast, textured) vs TripoSG (rectified flow — - // higher-fidelity geometry, geometry-only, slower; models download - // on first use). Declared BEFORE the Model row so the tier picker - // can react to it. + // Backend: TRELLIS.2 (Microsoft, out-of-process sidecar — the + // highest-quality tier and the DEFAULT whenever its runtime is + // installed), TripoSR (fast, textured, local ONNX) or TripoSG + // (rectified flow — higher-fidelity local geometry, geometry-only, + // slower). Declared BEFORE the Model row so the tier picker can + // react to it. Index map: 0 = TRELLIS.2, 1 = TripoSR, 2 = TripoSG. Row { spacing: 6 Text { @@ -1951,16 +1956,110 @@ Rectangle { id: mgBackendCombo width: 190 enabled: !MeshGenController.busy - model: ["TripoSR (fast, textured)", "TripoSG (best geometry)"] - currentIndex: 0 + // Runtime state read once at instantiation (an install + // while the app runs is picked up on the next session / + // section reopen). + readonly property bool t2Ready: MeshGenController.trellis2Available() + readonly property bool t2Selected: currentIndex === 0 + readonly property bool sgSelected: currentIndex === 2 + model: ["TRELLIS.2 (high quality" + (t2Ready ? ")" : ", needs runtime)"), + "TripoSR (fast, textured)", + "TripoSG (best geometry)"] + currentIndex: t2Ready ? 0 : 1 // Switching to TripoSG snaps the tier picker to fp32 (its // only geometry tier); the int8 option is meaningless there. onCurrentIndexChanged: { - if (currentIndex === 1) + if (currentIndex === 2) mgQualityCombo.currentIndex = 0 } } } + // Runtime hint when TRELLIS.2 is selected but not installed. + Text { + visible: mgBackendCombo.t2Selected && !mgBackendCombo.t2Ready + text: " ⚠ " + MeshGenController.trellis2RuntimeHint() + color: PropertiesPanelController.textColor + font.pixelSize: 10 + wrapMode: Text.WordWrap + width: parent.width - 16 + } + // Which runtime flavor is active (trellis.cpp / Python sidecar). + Text { + visible: mgBackendCombo.t2Selected && mgBackendCombo.t2Ready + text: " " + MeshGenController.trellis2RuntimeHint() + + " Textures + PBR maps are baked natively by QtMeshEditor." + color: PropertiesPanelController.textColor + font.pixelSize: 10 + wrapMode: Text.WordWrap + width: parent.width - 16 + } + + // ---- TRELLIS.2 options (only for the TRELLIS.2 backend) ---------- + Row { + spacing: 6 + visible: mgBackendCombo.t2Selected + Text { + text: "Quality" + color: PropertiesPanelController.textColor + font.pixelSize: 11 + anchors.verticalCenter: parent.verticalCenter + } + InspectorComboBox { + id: mgT2Preset + width: 190 + enabled: !MeshGenController.busy + model: ["Fast (512)", "Balanced (default)", "High (1536, more VRAM)"] + currentIndex: 1 + readonly property var presetValues: ["fast", "balanced", "high"] + property string presetValue: presetValues[currentIndex] + } + } + Row { + spacing: 6 + // ALL backends: game-ready weld + debris-cull + simplify (the + // raw generations are marching-cubes/voxel-dense — decimating + // them blind blobs out and skins badly; this path simplifies + // hard and re-bakes detail as diffuse + normal maps instead). + Text { + text: "Mesh" + color: PropertiesPanelController.textColor + font.pixelSize: 11 + anchors.verticalCenter: parent.verticalCenter + } + InspectorComboBox { + id: mgT2Mesh + width: 190 + enabled: !MeshGenController.busy + // Targets are approximate — border locking can stop earlier. + // "Maximum" still caps dense TRELLIS sources at ~150–300k + // for the bake (an uncapped raw dual-grid mesh is + // un-unwrappable); the raw source is preserved either way. + model: ["Maximum detail (auto cap)", "Game Low (~10k tris)", + "Game Medium (~25k tris)", "Game High (~50k tris)"] + currentIndex: 2 + readonly property var triValues: [0, 10000, 25000, 50000] + property int triValue: triValues[currentIndex] + } + } + Row { + spacing: 6 + visible: mgBackendCombo.t2Selected + Text { + text: "Texture" + color: PropertiesPanelController.textColor + font.pixelSize: 11 + anchors.verticalCenter: parent.verticalCenter + } + InspectorComboBox { + id: mgT2Tex + width: 190 + enabled: !MeshGenController.busy + model: ["1024 px", "2048 px (default)", "4096 px"] + currentIndex: 1 + readonly property var sizeValues: [1024, 2048, 4096] + property int sizeValue: sizeValues[currentIndex] + } + } // Model quality/size tier. TripoSR: fp32 (best/largest) vs int8 // (smallest), 1:1 with the MeshGenController quality int. TripoSG: @@ -1969,7 +2068,10 @@ Rectangle { // real option and locks. Row { spacing: 6 - property bool isSG: mgBackendCombo.currentIndex === 1 + property bool isSG: mgBackendCombo.sgSelected + // The TRELLIS.2 sidecar manages its own weights — the local + // tier picker doesn't apply, so the row hides entirely. + visible: !mgBackendCombo.t2Selected Text { text: "Model" color: PropertiesPanelController.textColor @@ -2003,26 +2105,34 @@ Rectangle { id: mgSmooth text: "Smooth mesh (Taubin)" checked: true + // Marching-cubes polish — local ONNX backends only (TRELLIS.2's + // dual-grid extraction has no stair-stepping to smooth). + visible: !mgBackendCombo.t2Selected } InspectorCheck { id: mgRefine text: "Refine surface (re-project)" checked: true + visible: !mgBackendCombo.t2Selected } // ---- Colour / texture stages, in execution order ---------------- // TripoSG (geometry-only) gets its colour SOLELY from the AI // texture pass — so when TripoSG is the backend, that checkbox // leads and the plain "Bake diffuse" (TripoSR field colour) is // hidden. TripoSR keeps the classic bake → PBR → upscale chain. - property bool sgSelected: mgBackendCombo.currentIndex === 1 + property bool sgSelected: mgBackendCombo.sgSelected + property bool t2Selected: mgBackendCombo.t2Selected // AI texture (multi-view depth-ControlNet): front from the input // photo, back/sides SD-generated from the shape, then projected. // For TripoSG this is the only colour source; for TripoSR it's an - // optional higher-quality alternative to the field bake. + // optional higher-quality alternative to the field bake. TRELLIS.2 + // doesn't need it — it generates real PBR attributes that + // QtMeshEditor bakes natively. InspectorCheck { id: mgAiTexture text: "Generate texture (AI, front photo + generated back)" + visible: !parent.t2Selected checked: parent.sgSelected enabled: !MeshGenController.busy && MaterialEditorQML.stableDiffusionEnabled @@ -2074,24 +2184,33 @@ Rectangle { text: "Generate 3D" clickEnabled: !MeshGenController.busy && MeshGenController.selectedImagePath.length > 0 + && (!mgBackendCombo.t2Selected || mgBackendCombo.t2Ready) onClicked: { - var sg = mgBackendCombo.currentIndex === 1 // TripoSG + var t2 = mgBackendCombo.currentIndex === 0 // TRELLIS.2 + var sg = mgBackendCombo.currentIndex === 2 // TripoSG var steps = [{ key: "prep", label: "Prepare models" }] // The worker only posts a "background" stage on the TripoSR - // path; TripoSG removes the bg inside its predict() dispatch - // without a discrete progress event, so a "background" row - // there would never resolve and look stuck. - if (mgRemoveBg.checked && !sg) + // path; TripoSG/TRELLIS.2 remove the bg inside their + // predict() dispatch without a discrete progress event, so + // a "background" row there would never resolve. + if (mgRemoveBg.checked && !sg && !t2) steps.push({ key: "background", label: "Remove background" }) - steps.push({ key: "encode", label: "Encode image" }) + steps.push({ key: "encode", + label: t2 ? "Prepare subject + load model" + : "Encode image" }) if (sg) steps.push({ key: "denoise", label: "Denoise (flow steps)" }) - steps.push({ key: "decode", label: "Reconstruct 3D" }) - if (mgRefine.checked) + if (t2) + steps.push({ key: "denoise", label: "Generate (TRELLIS.2)" }) + steps.push({ key: "decode", + label: t2 ? "Transfer + optimize mesh" + : "Reconstruct 3D" }) + if (mgRefine.checked && !t2) steps.push({ key: "refine", label: "Refine surface" }) // AI texture requested + available? (TripoSG's ONLY colour - // source; optional extra for TripoSR.) - var aiTex = mgAiTexture.checked + // source; optional extra for TripoSR. TRELLIS.2 never needs + // it — its PBR attributes are baked natively.) + var aiTex = mgAiTexture.checked && !t2 && MaterialEditorQML.stableDiffusionEnabled // Colour stages. For TripoSG we do NOT bake colour at build @@ -2130,16 +2249,23 @@ Rectangle { mgRoot.mgActiveProgress = -1 mgRoot.mgAiPending = aiTex // gate onCompleted's AI kickoff + var genOptions = { + "smooth": mgSmooth.checked, + "refine": mgRefine.checked, + "bake_texture": buildBake, + "generate_pbr": buildPbr, + "upscale_texture": mgUpscale.checked && buildBake, + "backend": t2 ? "trellis2" : (sg ? "triposg" : "triposr"), + // Game-ready simplification target (all backends). + "target_tris": mgT2Mesh.triValue + } + if (t2) { + genOptions["preset"] = mgT2Preset.presetValue + genOptions["texture_size"] = mgT2Tex.sizeValue + } MeshGenController.generateSelected( mgResCombo.resValue, mgRemoveBg.checked, mgQualityCombo.currentIndex, - { - "smooth": mgSmooth.checked, - "refine": mgRefine.checked, - "bake_texture": buildBake, - "generate_pbr": buildPbr, - "upscale_texture": mgUpscale.checked && buildBake, - "backend": sg ? "triposg" : "triposr" - }) + genOptions) } } diff --git a/scripts/check-trellis2-restricted-deps.sh b/scripts/check-trellis2-restricted-deps.sh new file mode 100755 index 000000000..017882149 --- /dev/null +++ b/scripts/check-trellis2-restricted-deps.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Phase 13 CI gate: the TRELLIS.2 integration must never (re)introduce the +# prohibited NVIDIA research-only libraries (nvdiffrast / nvdiffrec — NVIDIA +# Source Code License, "research or evaluation purposes only"; see +# docs/trellis2-dependencies.md). Their names are allowed ONLY in license +# documentation, prohibition comments/guards, this script, and the guard test. +# +# Usage: ./scripts/check-trellis2-restricted-deps.sh (from the repo root) +set -euo pipefail + +cd "$(dirname "$0")/.." +fail=0 + +note() { echo "check-trellis2-restricted-deps: $*"; } + +# 1. No import/require form ANYWHERE in the sidecar or the C++ integration. +# (import nvdiffrast / from nvdiffrast import / import nvdiffrec…) +# The guard test itself is an allowed location — it QUOTES the import forms +# in order to detect them (same carve-out the spec gives this script). +if grep -RInE '(^|[^a-zA-Z_])(import|from)[[:space:]]+nvdiff(rast|rec)' \ + --exclude='Trellis2Guard_test.cpp' \ + ai/trellis2 src/ImageTo3D 2>/dev/null; then + note "FAIL: an import of a prohibited NVIDIA library was introduced." + fail=1 +fi + +# 2. Dependency manifests must not list them (or the PyPI `cumesh` trap — an +# unrelated, unlicensed package; CuMesh is built from the pinned checkout). +if grep -RInE '^[[:space:]]*(nvdiffrast|nvdiffrec|cumesh)([=<>![:space:];[]|$)' \ + ai/trellis2/requirements.txt 2>/dev/null; then + note "FAIL: a prohibited/trap package appears in requirements.txt." + fail=1 +fi + +# 3. The C++ replacement layer may mention the names only in comments. +if grep -InE 'nvdiff(rast|rec)' \ + src/ImageTo3D/Trellis2Predictor.cpp \ + src/ImageTo3D/Trellis2Bake.cpp \ + src/ImageTo3D/Trellis2Interchange.cpp \ + src/ImageTo3D/Trellis2Predictor.h \ + src/ImageTo3D/Trellis2Bake.h \ + src/ImageTo3D/Trellis2Interchange.h 2>/dev/null \ + | grep -vE ':[0-9]+:[[:space:]]*(//|\*)' ; then + note "FAIL: non-comment reference to a prohibited NVIDIA library in the C++ integration." + fail=1 +fi + +# 4. install.py must not clone/install them either (git URLs, pip specs). +if grep -InE 'nvdiffrast\.git|nvdiffrec\.git|pip.*nvdiff' ai/trellis2/install.py 2>/dev/null; then + note "FAIL: install.py fetches a prohibited NVIDIA library." + fail=1 +fi + +if [ "$fail" -ne 0 ]; then + note "prohibited-dependency check FAILED" + exit 1 +fi +note "OK — nvdiffrast/nvdiffrec are absent from the TRELLIS.2 integration." diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 75db3629d..c79e10e57 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -53,6 +53,7 @@ #include "FaceRig/FaceRigLandmarks.h" #include "ImageTo3D/MeshGenPredictor.h" #include "ImageTo3D/TripoSGPredictor.h" +#include "ImageTo3D/Trellis2Predictor.h" #include "ImageTo3D/MeshGenBuilder.h" #include "MeshSegmenter.h" #include "SubMeshOps.h" @@ -10466,8 +10467,12 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) int textureSize = 1024; int flowSteps = 25; // TripoSG rectified-flow steps float guidance = 7.0f; // TripoSG CFG scale (0 disables CFG) + unsigned seed = 42; // TRELLIS.2 generation seed + QString preset = QStringLiteral("balanced"); // TRELLIS.2 fast|balanced|high + int targetTris = 0; // TRELLIS.2 game-ready simplification target MeshGenPredictor::Quality quality = MeshGenPredictor::Quality::Fp32; MeshGenPredictor::Backend backend = MeshGenPredictor::Backend::TripoSR; + bool backendSet = false; // explicit --backend beats the auto default for (int i = 1; i < argc; ++i) { const QString arg = QString::fromLocal8Bit(argv[i]); @@ -10482,13 +10487,42 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) if (arg == "--no-pbr") { generatePbr = false; continue; } if (arg == "--backend") { if (i + 1 >= argc) { - err() << "Error: --backend requires triposr or triposg." << Qt::endl; + err() << "Error: --backend requires trellis2, triposr or triposg." << Qt::endl; return 2; } const QString b = QString::fromLocal8Bit(argv[++i]).toLower(); if (b == "triposr") backend = MeshGenPredictor::Backend::TripoSR; else if (b == "triposg") backend = MeshGenPredictor::Backend::TripoSG; - else { err() << "Error: --backend must be triposr or triposg." << Qt::endl; return 2; } + else if (b == "trellis2" || b == "trellis.2" || b == "trellis") + backend = MeshGenPredictor::Backend::Trellis2; + else { err() << "Error: --backend must be trellis2, triposr or triposg." << Qt::endl; return 2; } + backendSet = true; + continue; + } + if (arg == "--seed") { + if (i + 1 >= argc) { err() << "Error: --seed requires a number." << Qt::endl; return 2; } + bool ok = false; + seed = QString::fromLocal8Bit(argv[++i]).toUInt(&ok); + if (!ok) { err() << "Error: --seed must be a non-negative integer." << Qt::endl; return 2; } + continue; + } + if (arg == "--preset") { + if (i + 1 >= argc) { err() << "Error: --preset requires fast, balanced or high." << Qt::endl; return 2; } + preset = QString::fromLocal8Bit(argv[++i]).toLower(); + if (preset != "fast" && preset != "balanced" && preset != "high") { + err() << "Error: --preset must be fast, balanced or high." << Qt::endl; + return 2; + } + continue; + } + if (arg == "--target-tris") { + if (i + 1 >= argc) { err() << "Error: --target-tris requires a number." << Qt::endl; return 2; } + bool ok = false; + targetTris = QString::fromLocal8Bit(argv[++i]).toInt(&ok); + if (!ok || targetTris < 0 || targetTris > 10000000) { + err() << "Error: --target-tris must be in [0, 10000000] (0 = original)." << Qt::endl; + return 2; + } continue; } if (arg == "--flow-steps") { @@ -10577,8 +10611,13 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) "[--no-color] [--remove-bg] [--quality fp32|int8] " "[--no-smooth] [--no-refine] [--no-bake-texture] [--texture-size 1024] " "[--upscale-texture] [--no-pbr] " - "[--backend triposr|triposg] [--flow-steps 25] [--guidance 7.0]" + "[--backend trellis2|triposr|triposg] [--flow-steps 25] [--guidance 7.0] " + "[--seed 42] [--preset fast|balanced|high] [--target-tris N]" << Qt::endl; + err() << " Default backend: trellis2 when its runtime is installed " + "(ai/trellis2/install.py), else triposr. --seed/--preset " + "apply to trellis2; --target-tris (game-ready simplify + " + "detail-normal bake) applies to every backend." << Qt::endl; return 2; } QFileInfo fi(inputPath); @@ -10589,12 +10628,27 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) if (outputPath.isEmpty()) outputPath = fi.absolutePath() + "/" + fi.completeBaseName() + ".glb"; + // Resolve the default backend when none was requested: TRELLIS.2 when its + // sidecar runtime is installed on this machine, else TripoSR. + if (!backendSet) { + backend = MeshGenPredictor::defaultBackend(); + if (backend == MeshGenPredictor::Backend::Trellis2) + err() << "Using backend: trellis2 (runtime detected; pass " + "--backend triposr|triposg to override)." << Qt::endl; + } + const bool useTrellis2 = (backend == MeshGenPredictor::Backend::Trellis2); + #ifndef ENABLE_ONNX - Q_UNUSED(resolution); Q_UNUSED(vertexColor); Q_UNUSED(noModel); - err() << "Error: this build was compiled without AI image-to-3D generation " - "(rebuild with -DENABLE_ONNX=ON)." << Qt::endl; - return 1; -#else + // The TRELLIS.2 sidecar backend has no ONNX dependency; only the local + // TripoSR/TripoSG paths need the ONNX build. + if (!useTrellis2) { + Q_UNUSED(resolution); Q_UNUSED(vertexColor); Q_UNUSED(noModel); + err() << "Error: this build was compiled without AI image-to-3D generation " + "(rebuild with -DENABLE_ONNX=ON, or install the TRELLIS.2 runtime " + "and use --backend trellis2)." << Qt::endl; + return 1; + } +#endif if (noModel) { err() << "Error: --no-model given but TripoSR has no non-model fallback " "(unlike segmentation/in-betweening). Remove --no-model." << Qt::endl; @@ -10606,14 +10660,21 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) } const bool useSG = (backend == MeshGenPredictor::Backend::TripoSG); + const QString backendName = useTrellis2 ? QStringLiteral("trellis2") + : (useSG ? QStringLiteral("triposg") : QStringLiteral("triposr")); SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.image_to_3d"), QString("generate3d .%1 res=%2 color=%3 backend=%4") .arg(fi.suffix()).arg(resolution).arg(vertexColor) - .arg(useSG ? QStringLiteral("triposg") : QStringLiteral("triposr"))); + .arg(backendName)); // Download the chosen backend's models on first use (blocks; clear // message when not hosted). - if (useSG) { + if (useTrellis2) { + if (!Trellis2Predictor::runtimeAvailable()) { + err() << "Error: " << Trellis2Predictor::runtimeDescription() << Qt::endl; + return 1; + } + } else if (useSG) { if (quality == MeshGenPredictor::Quality::Int8) err() << "Note: --quality int8 is not available for the triposg " "backend (the quantized DiT degrades geometry); using fp32." @@ -10665,6 +10726,18 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) opts.backend = backend; opts.flowSteps = flowSteps; opts.guidanceScale = guidance; + opts.seed = seed; + opts.trellis2Preset = preset; + opts.targetTriangles = targetTris; + opts.bakeNormalMap = generatePbr && bake; + if (useTrellis2) { + opts.removeBackground = true; // trellis2 needs an alpha matte; the + // predictor skips it when the input + // already carries one + // Phase 9: keep the raw full-res generation next to the export. + opts.trellis2SourceKeepDir = QFileInfo(outputPath).absolutePath(); + opts.trellis2SourceKeepBaseName = QFileInfo(outputPath).completeBaseName(); + } MeshGenPredictor::Result res = MeshGenPredictor::predict( image, MeshGenPredictor::encoderModelPath(quality), MeshGenPredictor::decoderModelPath(), opts); @@ -10720,15 +10793,18 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) return 1; } - cliWrite(QString("Generated 3D mesh: %1 verts, %2 tris%3\nWrote: %4\n") + cliWrite(QString("Generated 3D mesh: %1 verts, %2 tris%3\nWrote: %4\n%5") .arg(res.vertexCount).arg(res.triangleCount) .arg(!res.uvs.empty() ? QStringLiteral(" (+baked %1px diffuse texture)").arg(res.texture.width()) : (res.colors.empty() ? QString() : QStringLiteral(" (+vertex color)"))) - .arg(QFileInfo(outputPath).fileName())); + .arg(QFileInfo(outputPath).fileName()) + .arg(res.sourceInterchangePath.isEmpty() + ? QString() + : QStringLiteral("Kept full-res source: %1\n") + .arg(QFileInfo(res.sourceInterchangePath).fileName()))); return 0; -#endif } int CLIPipeline::cmdSegment(int argc, char* argv[]) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b0f5867ee..f2264c178 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -210,6 +210,9 @@ ImageTo3D/MeshRefine.cpp ImageTo3D/MeshGenBaker.cpp ImageTo3D/MeshGenPredictor.cpp ImageTo3D/TripoSGPredictor.cpp +ImageTo3D/Trellis2Interchange.cpp +ImageTo3D/Trellis2Bake.cpp +ImageTo3D/Trellis2Predictor.cpp ImageTo3D/MeshGenBuilder.cpp ImageTo3D/MeshGenController.cpp ImageTo3D/BackgroundRemover.cpp diff --git a/src/ImageTo3D/BackgroundRemover.cpp b/src/ImageTo3D/BackgroundRemover.cpp index eec2f5daa..96ece3049 100644 --- a/src/ImageTo3D/BackgroundRemover.cpp +++ b/src/ImageTo3D/BackgroundRemover.cpp @@ -255,20 +255,37 @@ BackgroundRemover::Result BackgroundRemover::removeBackground(const QImage& imag // Top-left of the subject square within the padded output. const int sqOffX = (outSz - fgW) / 2, sqOffY = (outSz - fgH) / 2; - QImage composed(outSz, outSz, QImage::Format_RGB888); - composed.fill(qRgb(opts.bgR, opts.bgG, opts.bgB)); + // keepAlpha (TRELLIS.2): carry the matte in a real alpha channel + // instead of blending it into the RGB — the consumer premultiplies + // itself and this is what keeps the upstream (non-commercial) rembg + // model from ever being needed. + QImage composed(outSz, outSz, + opts.keepAlpha ? QImage::Format_RGBA8888 + : QImage::Format_RGB888); + if (opts.keepAlpha) + composed.fill(QColor(opts.bgR, opts.bgG, opts.bgB, 0)); + else + composed.fill(qRgb(opts.bgR, opts.bgG, opts.bgB)); + const int bpp = opts.keepAlpha ? 4 : 3; for (int oy = 0; oy < outSz; ++oy) { uchar* dp = composed.scanLine(oy); const int sy = by0 + (oy - sqOffY); for (int ox = 0; ox < outSz; ++ox) { const int sx = bx0 + (ox - sqOffX); - if (sx < 0 || sy < 0 || sx >= W || sy >= H) continue; // stays gray + if (sx < 0 || sy < 0 || sx >= W || sy >= H) continue; // stays bg const float a = alphaAt(sx, sy); if (a <= 0.0f) continue; - uchar* px = dp + ox * 3; - px[0] = uchar(srcAt(sx, sy, 0) * a + opts.bgR * (1 - a) + 0.5f); - px[1] = uchar(srcAt(sx, sy, 1) * a + opts.bgG * (1 - a) + 0.5f); - px[2] = uchar(srcAt(sx, sy, 2) * a + opts.bgB * (1 - a) + 0.5f); + uchar* px = dp + ox * bpp; + if (opts.keepAlpha) { + px[0] = uchar(srcAt(sx, sy, 0) + 0.5f); + px[1] = uchar(srcAt(sx, sy, 1) + 0.5f); + px[2] = uchar(srcAt(sx, sy, 2) + 0.5f); + px[3] = uchar(a * 255.0f + 0.5f); + } else { + px[0] = uchar(srcAt(sx, sy, 0) * a + opts.bgR * (1 - a) + 0.5f); + px[1] = uchar(srcAt(sx, sy, 1) * a + opts.bgG * (1 - a) + 0.5f); + px[2] = uchar(srcAt(sx, sy, 2) * a + opts.bgB * (1 - a) + 0.5f); + } } } diff --git a/src/ImageTo3D/BackgroundRemover.h b/src/ImageTo3D/BackgroundRemover.h index 184a926ed..c85ca0961 100644 --- a/src/ImageTo3D/BackgroundRemover.h +++ b/src/ImageTo3D/BackgroundRemover.h @@ -45,6 +45,14 @@ class BackgroundRemover { // (default 0.85). Centering + tight framing is what stops the leftover // margin being reconstructed as background geometry. 0 disables cropping. float foregroundRatio = 0.85f; + // Emit an RGBA image carrying the segmentation as a real ALPHA MATTE + // instead of compositing over the solid background. The TRELLIS.2 + // backend needs this: its pipeline consumes RGBA-with-alpha directly + // (and a genuine matte keeps the upstream default remover — the + // non-commercial briaai/RMBG-2.0 — from ever loading; see + // docs/trellis2-dependencies.md). Default off preserves the TripoSR + // behaviour. + bool keepAlpha = false; }; struct Result { diff --git a/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp b/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp index d2ed833e6..d606558e0 100644 --- a/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp +++ b/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp @@ -116,3 +116,47 @@ TEST(CLIPipelineCmdGenerate3dCoverage, ValidImageWithoutModelOrOnnxFailsCleanly) const int rc = CLIPipeline::cmdGenerate3d(args.argc(), args.argv()); EXPECT_NE(rc, 0); } + +// ── TRELLIS.2 backend flags (this integration) ─────────────────────────────── + +TEST(CLIPipelineCmdGenerate3dCoverage, Trellis2BadPresetIsUsageError) +{ + Gen3dArgv args({"generate3d", kMissingImage, "--preset", "ultra"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(args.argc(), args.argv()), 2); + Gen3dArgv missing({"generate3d", kMissingImage, "--preset"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(missing.argc(), missing.argv()), 2); +} + +TEST(CLIPipelineCmdGenerate3dCoverage, Trellis2BadTargetTrisIsUsageError) +{ + Gen3dArgv neg({"generate3d", kMissingImage, "--target-tris", "-5"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(neg.argc(), neg.argv()), 2); + Gen3dArgv nan({"generate3d", kMissingImage, "--target-tris", "many"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(nan.argc(), nan.argv()), 2); + Gen3dArgv missing({"generate3d", kMissingImage, "--seed"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(missing.argc(), missing.argv()), 2); +} + +TEST(CLIPipelineCmdGenerate3dCoverage, Trellis2BackendAcceptedButUnknownRejected) +{ + // Unknown backend name → usage error. + Gen3dArgv bad({"generate3d", kMissingImage, "--backend", "dreamfusion"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(bad.argc(), bad.argv()), 2); + + // trellis2 is a valid backend; with a real image but a deliberately + // nonexistent runtime the command must fail at RUNTIME (1) with the + // install hint — never crash, never a usage error. + qputenv("QTMESH_TRELLIS2_ENV", "/nonexistent/qtmesh-trellis2-cli-ut"); + qputenv("QTMESH_TRELLIS2_PYTHON", "/nonexistent/python-cli-ut"); + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString png = QDir(tmp.path()).filePath("in.png"); + QImage img(16, 16, QImage::Format_RGB888); + img.fill(Qt::red); + ASSERT_TRUE(img.save(png, "PNG")); + const QByteArray pngBytes = png.toLocal8Bit(); + Gen3dArgv ok({"generate3d", pngBytes.constData(), "--backend", "trellis2"}); + EXPECT_EQ(CLIPipeline::cmdGenerate3d(ok.argc(), ok.argv()), 1); + qunsetenv("QTMESH_TRELLIS2_ENV"); + qunsetenv("QTMESH_TRELLIS2_PYTHON"); +} diff --git a/src/ImageTo3D/MeshGenBuilder.cpp b/src/ImageTo3D/MeshGenBuilder.cpp index 8b2d1886f..2278604a5 100644 --- a/src/ImageTo3D/MeshGenBuilder.cpp +++ b/src/ImageTo3D/MeshGenBuilder.cpp @@ -89,7 +89,13 @@ Ogre::Mesh* buildMesh(const MeshGenPredictor::Result& result, const QString& mes && !texturePngPath.isEmpty(); const bool hasColor = !hasUv && result.colors.size() == static_cast(result.vertexCount) * 3; - const std::vector normals = computeNormals(result.positions, result.indices); + // Prefer the predictor's precomputed smooth normals (TRELLIS.2's bake + // supplies position-welded ones so xatlas chart seams stay smooth); + // otherwise accumulate from the index buffer as before. + const std::vector normals = + result.normals.size() == result.positions.size() + ? result.normals + : computeNormals(result.positions, result.indices); auto& mm = Ogre::MeshManager::getSingleton(); const std::string name = meshName.toStdString(); @@ -296,6 +302,7 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, // exporters resolve it from a registered resource location). QString texPath; QString normalPath, roughnessPath; // optional #404 PBR stage outputs + QString metallicPath; // TRELLIS.2 real baked metallic if (!result.texture.isNull() && result.uvs.size() == static_cast(result.vertexCount) * 2) { QString dir = opts.textureDir; @@ -309,19 +316,41 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, if (result.texture.save(candidate, "PNG")) { texPath = candidate; + // TRELLIS.2 path: the predictor baked REAL PBR maps from the + // generation's attribute volume (Trellis2Bake) — persist and bind + // those, and skip the #404 guess-from-albedo synthesis entirely. + auto saveMap = [&](const QImage& img, const char* suffix) { + if (img.isNull()) + return QString(); + const QString p = QDir(dir).filePath( + unique + QStringLiteral("_%1.png").arg(QLatin1String(suffix))); + return img.save(p, "PNG") ? p : QString(); + }; + normalPath = saveMap(result.normalMap, "normal"); + roughnessPath = saveMap(result.roughnessMap, "roughness"); + metallicPath = saveMap(result.metallicMap, "metallic"); + // Optional PBR stage (#404): synthesize normal + roughness from the // baked diffuse BEFORE the resource location is (re)indexed so the // new PNGs land in the index below. Height is skipped — nothing in // this material path consumes it. Fails soft: a missing model / // non-ONNX build just leaves the maps empty (diffuse-only result). - if (opts.generatePbrMaps) { + // Synthesize ONLY the maps the predictor didn't bake for real: + // TRELLIS.2 provides all three; the game-ready TripoSR path bakes + // a real detail normal but still wants the #404 roughness guess. + if (opts.generatePbrMaps + && (normalPath.isEmpty() || roughnessPath.isEmpty())) { PbrMapSynth::Options po; po.generateHeight = false; + po.generateNormal = normalPath.isEmpty(); + po.generateRoughness = roughnessPath.isEmpty(); const PbrMapSynthResult pr = AIAssistManager::instance()->synthesizePbrMaps(texPath, po); if (pr.ok) { - normalPath = pr.normalPath; - roughnessPath = pr.roughnessPath; + if (normalPath.isEmpty()) + normalPath = pr.normalPath; + if (roughnessPath.isEmpty()) + roughnessPath = pr.roughnessPath; } else { Ogre::LogManager::getSingleton().logWarning( ("MeshGenBuilder: PBR map synthesis failed (" @@ -374,7 +403,8 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, // button: canonical named slots + FFP wiring + the RTSS normal-map // sub-render-state (without applyNormalMap the bind is invisible in the // viewport), then recompile. - if (!normalPath.isEmpty() || !roughnessPath.isEmpty()) { + if (!normalPath.isEmpty() || !roughnessPath.isEmpty() + || !metallicPath.isEmpty()) { const std::string matName = (unique + QStringLiteral("_mesh")).toStdString() + "_mat"; Ogre::MaterialPtr mat = @@ -402,6 +432,7 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, }; bindSlot("normal_map", normalPath); bindSlot("roughness", roughnessPath); + bindSlot("metallic", metallicPath); RTShaderHelper::wirePbrSlotsForFFP(mat.get()); mat->compile(); // NOTE: the RTSS SRS_NORMALMAP wiring is deferred to diff --git a/src/ImageTo3D/MeshGenController.cpp b/src/ImageTo3D/MeshGenController.cpp index e8d7fb39a..4bdde6738 100644 --- a/src/ImageTo3D/MeshGenController.cpp +++ b/src/ImageTo3D/MeshGenController.cpp @@ -4,6 +4,7 @@ #include "MeshGenPredictor.h" #include "TripoSGPredictor.h" +#include "Trellis2Predictor.h" #include "MeshGenBuilder.h" #include "BackgroundRemover.h" #include "MeshImporterExporter.h" @@ -17,7 +18,9 @@ #include #include #include +#include #include +#include #include #include // organizationName() — test-harness guard #include @@ -82,7 +85,19 @@ bool MeshGenController::available() const // sets this org name) skips the surface. Pure-data pieces are covered directly. if (QCoreApplication::organizationName() == QLatin1String("QtMeshEditorTests")) return false; - return MeshGenPredictor::isAvailable(); + // The TRELLIS.2 sidecar backend needs no ONNX build — a machine with its + // runtime installed gets the section even on a non-ONNX build. + return MeshGenPredictor::isAvailable() || Trellis2Predictor::runtimeAvailable(); +} + +bool MeshGenController::trellis2Available() const +{ + return Trellis2Predictor::runtimeAvailable(); +} + +QString MeshGenController::trellis2RuntimeHint() const +{ + return Trellis2Predictor::runtimeDescription(); } void MeshGenController::setBusy(bool b) @@ -266,13 +281,30 @@ void MeshGenController::generate(const QString& imagePath, int resolution, m_generatePbr = optBool("generate_pbr", true) && wantBake; const int textureSize = options.contains(QLatin1String("texture_size")) ? options.value(QLatin1String("texture_size")).toInt() : 1024; - // Backend: "triposr" (default, fast + textured) or "triposg" (rectified - // flow — higher-fidelity geometry, geometry-only, slower). - const bool useSG = options.value(QLatin1String("backend")).toString() - .compare(QLatin1String("triposg"), - Qt::CaseInsensitive) == 0; + // Backend: "trellis2" (the default whenever its sidecar runtime is + // installed), "triposr" (fast + textured) or "triposg" (rectified flow — + // higher-fidelity geometry, geometry-only, slower). Empty/unknown values + // resolve through defaultBackend(). + const QString backendStr = + options.value(QLatin1String("backend")).toString().toLower(); + MeshGenPredictor::Backend backend = MeshGenPredictor::defaultBackend(); + if (backendStr == QLatin1String("triposr")) + backend = MeshGenPredictor::Backend::TripoSR; + else if (backendStr == QLatin1String("triposg")) + backend = MeshGenPredictor::Backend::TripoSG; + else if (backendStr.startsWith(QLatin1String("trellis"))) + backend = MeshGenPredictor::Backend::Trellis2; + const bool useSG = (backend == MeshGenPredictor::Backend::TripoSG); + const bool useT2 = (backend == MeshGenPredictor::Backend::Trellis2); const int flowSteps = options.contains(QLatin1String("flow_steps")) ? options.value(QLatin1String("flow_steps")).toInt() : 25; + const unsigned t2Seed = options.contains(QLatin1String("seed")) + ? options.value(QLatin1String("seed")).toUInt() : 42u; + const QString t2Preset = options.contains(QLatin1String("preset")) + ? options.value(QLatin1String("preset")).toString().toLower() + : QStringLiteral("balanced"); + const int t2TargetTris = options.contains(QLatin1String("target_tris")) + ? options.value(QLatin1String("target_tris")).toInt() : 0; GamificationManager::noteFeature(QStringLiteral("image_to_3d")); @@ -288,7 +320,16 @@ void MeshGenController::generate(const QString& imagePath, int resolution, // must not run on the worker thread. Once present, the worker only reads // the files (no event loop needed). emit statusMessage(tr("Checking model…")); - if (useSG) { + if (useT2) { + if (!Trellis2Predictor::runtimeAvailable()) { + setBusy(false); + emit error(Trellis2Predictor::runtimeDescription()); + return; + } + // The alpha-matte model must be ensured HERE (main thread — nested + // event loop); the worker-side predictor only reads it. + BackgroundRemover::ensureModelBlocking(); + } else if (useSG) { // TripoSG always runs the fp32 DiT — the int8 tier is dropped // (quantized geometry degrades to blobs; no ARM speed win). const QString enc = TripoSGPredictor::ensureModelBlocking(false); @@ -350,7 +391,9 @@ void MeshGenController::generate(const QString& imagePath, int resolution, // connection so the GUI thread updates the bar. m_pending->worker = std::thread([this, image, res, rembg, wantSmooth, wantRefine, wantBake, - textureSize, useSG, flowSteps]() { + textureSize, useSG, useT2, flowSteps, + backend, t2Seed, t2Preset, t2TargetTris, + imageStem = fi.completeBaseName()]() { auto post = [this](const QString& stage, int done, int total) { QMetaObject::invokeMethod(this, "progress", Qt::QueuedConnection, Q_ARG(QString, stage), Q_ARG(int, done), Q_ARG(int, total)); @@ -360,7 +403,7 @@ void MeshGenController::generate(const QString& imagePath, int resolution, // worker started), so this thread only reads files — no event loop // needed. (The encode stage is reported by the predictor itself.) QImage subject = image; - if (rembg && !useSG) { + if (rembg && !useSG && !useT2) { // TripoSR path: composite over gray-128 (its training background). // The TripoSG path leaves removal to the predictor dispatch, which // composites over WHITE per its reference pipeline. @@ -376,17 +419,33 @@ void MeshGenController::generate(const QString& imagePath, int resolution, MeshGenPredictor::Options opts; opts.sdfResolution = res; opts.vertexColor = true; - // TripoSR removal already ran above; TripoSG's white-background - // removal happens inside the predictor dispatch. - opts.removeBackground = rembg && useSG; + // TripoSR removal already ran above; TripoSG's white-background and + // TRELLIS.2's keep-alpha matte removal happen inside the predictor + // dispatch. + opts.removeBackground = rembg && (useSG || useT2); opts.smoothMesh = wantSmooth; opts.refineSurface = wantRefine; opts.bakeTexture = wantBake; opts.textureSize = textureSize; - opts.backend = useSG ? MeshGenPredictor::Backend::TripoSG - : MeshGenPredictor::Backend::TripoSR; + opts.backend = backend; opts.flowSteps = flowSteps; opts.quality = m_quality; + // Game-ready simplification target — ALL backends (TripoSR/TripoSG + // run the weld/debris/simplify + detail-normal-bake pass in the + // predictor; TRELLIS.2 does it natively in its own pipeline). + opts.targetTriangles = t2TargetTris; + opts.bakeNormalMap = m_generatePbr; + if (useT2) { + opts.seed = t2Seed; + opts.trellis2Preset = t2Preset; + // Phase 9: keep the raw full-res generation in AppData so + // textures/LODs can be re-baked without re-running inference. + opts.trellis2SourceKeepDir = + QDir(QStandardPaths::writableLocation( + QStandardPaths::AppDataLocation)) + .filePath(QStringLiteral("generated_sources")); + opts.trellis2SourceKeepBaseName = imageStem; + } QMetaObject::invokeMethod(this, "statusMessage", Qt::QueuedConnection, Q_ARG(QString, tr("Reconstructing…"))); diff --git a/src/ImageTo3D/MeshGenController.h b/src/ImageTo3D/MeshGenController.h index f4245727c..84b175b01 100644 --- a/src/ImageTo3D/MeshGenController.h +++ b/src/ImageTo3D/MeshGenController.h @@ -95,6 +95,13 @@ class MeshGenController : public QObject // ── Pre-download support (AI Settings modal) ──────────────────────────── // Whether the decoder + the given tier's encoder are already on disk. + // TRELLIS.2 sidecar runtime probe (the backend combo preselects TRELLIS.2 + // and enables its option rows only when this is true) + the install hint + // shown when it isn't. Invokables (not properties): the runtime can be + // installed while the app runs, so QML re-queries on section open. + Q_INVOKABLE bool trellis2Available() const; + Q_INVOKABLE QString trellis2RuntimeHint() const; + Q_INVOKABLE bool modelsPresent(int quality = 0) const; // Download the decoder + the given tier's encoder (blocks on the caller's // event loop, driven by ModelDownloader → its progress bar updates in the diff --git a/src/ImageTo3D/MeshGenPredictor.cpp b/src/ImageTo3D/MeshGenPredictor.cpp index 533456b54..19e0d0a57 100644 --- a/src/ImageTo3D/MeshGenPredictor.cpp +++ b/src/ImageTo3D/MeshGenPredictor.cpp @@ -7,6 +7,8 @@ #include "BackgroundRemover.h" #include "OnnxRuntimeSettings.h" #include "TripoSGPredictor.h" // Backend::TripoSG dispatch +#include "Trellis2Predictor.h" // Backend::Trellis2 dispatch (no ONNX needed) +#include "Trellis2Bake.h" // game-ready simplify + detail-normal bake (all backends) #include #include @@ -102,16 +104,90 @@ std::vector MeshGenPredictor::buildGridPoints(int resolution, float radiu return pts; } +MeshGenPredictor::Backend MeshGenPredictor::defaultBackend() +{ + // TRELLIS.2 becomes the default the moment its runtime is installed on + // this machine; otherwise the local ONNX TripoSR path stays the default. + return Trellis2Predictor::runtimeAvailable() ? Backend::Trellis2 + : Backend::TripoSR; +} + +namespace { +// Backend::Trellis2 dispatch, shared by the ONNX and non-ONNX builds — the +// sidecar backend has no ONNX dependency (only the optional U²-Net matte +// does, and Trellis2Predictor degrades that gracefully). +MeshGenPredictor::Result predictTrellis2( + const QImage& image, + const MeshGenPredictor::Options& opts, + const MeshGenPredictor::ProgressFn& progress) +{ + Trellis2Predictor::Options t2; + t2.preset = opts.trellis2Preset; + t2.seed = opts.seed; + t2.targetTriangles = opts.targetTriangles; + t2.bakeTexture = opts.bakeTexture; + t2.textureSize = opts.textureSize; + t2.bakeNormalMap = opts.bakeNormalMap; + t2.removeBackground = opts.removeBackground; + t2.mock = opts.trellis2Mock; + t2.sourceKeepDir = opts.trellis2SourceKeepDir; + t2.sourceKeepBaseName = opts.trellis2SourceKeepBaseName; + return Trellis2Predictor::predict(image, t2, progress); +} + +// Game-ready pass for the LOCAL backends (TripoSR/TripoSG): weld, drop +// floating debris, simplify toward Options::targetTriangles. Marching-cubes +// output decimated blind turns into a blob and skins terribly — the fix is +// the standard high→low workflow: simplify hard here, then bake the lost +// detail back as textures (diffuse via the field bake, relief via +// bakeDetailNormal against the dense pre-simplify source kept in srcPos/Idx). +// Returns false only on a hard failure (result untouched, warning appended). +bool applyGameReady(MeshGenPredictor::Result& out, + int targetTriangles, + std::vector* srcPosOut, + std::vector* srcIdxOut) +{ + if (targetTriangles <= 0 || out.vertexCount <= 0) + return false; + Trellis2Bake::GameReadyOptions gr; + gr.targetTriangles = targetTriangles; + const Trellis2Bake::GameReadyResult processed = + Trellis2Bake::makeGameReady(out.positions, out.indices, gr); + if (!processed.ok) { + if (!out.warning.isEmpty()) + out.warning += QStringLiteral(" "); + out.warning += QStringLiteral("game-ready pass failed (%1) — keeping " + "the full-density mesh.") + .arg(processed.error); + return false; + } + if (srcPosOut) *srcPosOut = std::move(out.positions); + if (srcIdxOut) *srcIdxOut = std::move(out.indices); + out.positions = processed.positions; + out.indices = processed.indices; + out.vertexCount = static_cast(out.positions.size() / 3); + out.triangleCount = static_cast(out.indices.size() / 3); + // Per-vertex colours (if any) belonged to the old vertices. + out.colors.clear(); + out.uvs.clear(); + return true; +} +} // namespace + #ifndef ENABLE_ONNX bool MeshGenPredictor::isAvailable() { return false; } QString MeshGenPredictor::ensureModelBlocking(Quality) { return {}; } -MeshGenPredictor::Result MeshGenPredictor::predict(const QImage&, const QString&, - const QString&, const Options&, - const ProgressFn&) +MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, + const QString&, + const QString&, + const Options& opts, + const ProgressFn& progress) { + if (opts.backend == Backend::Trellis2) + return predictTrellis2(image, opts, progress); Result r; r.error = QStringLiteral( "Image-to-3D needs an ONNX-enabled build — rebuild with -DENABLE_ONNX."); @@ -218,6 +294,10 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, if (image.isNull()) return fail(QStringLiteral("MeshGen: input image is empty.")); + // ---- Backend dispatch: TRELLIS.2 (out-of-process sidecar) ---------------- + if (opts.backend == Backend::Trellis2) + return predictTrellis2(image, opts, progress); + // ---- Backend dispatch: TripoSG (rectified-flow, geometry-only) ---------- if (opts.backend == Backend::TripoSG) { QImage subject = image; @@ -247,6 +327,11 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, Result r = TripoSGPredictor::predict(subject, sg, progress); // TripoSG's field is already +Y-up — skip the TripoSR frame bake. r.bakeTripoSROrientation = false; + // Game-ready simplification (geometry-only backend — nothing to bake; + // the later GUI AI-texture pass unwraps/bakes the SIMPLIFIED mesh, + // which is exactly what you want for skinning-friendly assets). + if (r.ok) + applyGameReady(r, opts.targetTriangles, nullptr, nullptr); // TripoSG is geometry-only. Colour comes SOLELY from the AI image // generation pass (multi-view depth-ControlNet, run later in the GUI // layer) — no TripoSR field colouring. With no AI texture the mesh @@ -472,6 +557,18 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, MeshRefine::isoProjectStep(out.positions, f, grad, step); } + // ---- (4b) Game-ready simplification (Options::targetTriangles) -------- + // The dense pre-simplify mesh is kept as the bake SOURCE: the diffuse + // below re-bakes on the simplified mesh straight from the decoder + // field (density-independent), and (5b) bakes the lost geometric + // detail into a tangent-space normal map — the standard high→low + // workflow that keeps a 10–50k mesh from reading as a blob. + std::vector gameReadySrcPos; + std::vector gameReadySrcIdx; + const bool gameReady = + applyGameReady(out, opts.targetTriangles, + &gameReadySrcPos, &gameReadySrcIdx); + // ---- (5) Colour: baked texture (preferred) or per-vertex --------------- if (wantColor && out.vertexCount > 0 && opts.bakeTexture) { MeshGenBaker::Options bakeOpts; @@ -505,6 +602,36 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, } } + // ---- (5b) Detail normal map (game-ready path only) --------------------- + // Bake the dense source's smooth normals into the SAME atlas the + // diffuse bake produced, expressed in the simplified target's tangent + // frame. This is what preserves the perceived detail after a hard + // simplification. Skipped when there's no baked diffuse to share an + // unwrap with (vertex-colour mode carries no UVs). + if (gameReady && opts.bakeNormalMap && !out.uvs.empty() + && !out.texture.isNull()) { + Trellis2Bake::BakeOptions nbo; + if (progress) + nbo.progress = [&](int done, int total) { + return progress(Stage::Bake, done, total); + }; + const Trellis2Bake::NormalBakeResult nb = + Trellis2Bake::bakeDetailNormal( + out.positions, out.indices, out.uvs, + out.texture.width(), out.texture.height(), + gameReadySrcPos, gameReadySrcIdx, nbo); + if (nb.cancelled) + return fail(QStringLiteral("cancelled")); + if (nb.ok) { + out.normalMap = nb.normalMap; + } else { + if (!out.warning.isEmpty()) + out.warning += QStringLiteral(" "); + out.warning += QStringLiteral( + "detail-normal bake failed (%1).").arg(nb.error); + } + } + // ---- (6) Per-vertex colour (bake disabled or fell back) --------------- if (wantColor && out.vertexCount > 0 && out.uvs.empty()) { const size_t nv = static_cast(out.vertexCount); diff --git a/src/ImageTo3D/MeshGenPredictor.h b/src/ImageTo3D/MeshGenPredictor.h index 4c77bc5c9..a17c5f8b5 100644 --- a/src/ImageTo3D/MeshGenPredictor.h +++ b/src/ImageTo3D/MeshGenPredictor.h @@ -50,10 +50,20 @@ class MeshGenPredictor { // that the ONNX fp16 converters can't rewrite cleanly; int8 is smaller anyway.) enum class Quality { Fp32, Int8 }; - // Generation backend. TripoSR = the fast single-pass LRM (default); + // Generation backend. TripoSR = the fast single-pass LRM; // TripoSG = the 1.5B rectified-flow model (higher-fidelity geometry, // slower, geometry-only — see TripoSGPredictor). Both MIT code+weights. - enum class Backend { TripoSR, TripoSG }; + // Trellis2 = Microsoft TRELLIS.2 (MIT code+weights) via the out-of-process + // Python sidecar (ai/trellis2/, Linux + NVIDIA GPU) — the highest-quality + // tier and the DEFAULT whenever its runtime is installed (see + // Trellis2Predictor + defaultBackend()); mesh cleanup/UVs/PBR baking are + // done natively by Trellis2Bake, deliberately without NVIDIA + // nvdiffrast/nvdiffrec (docs/trellis2-dependencies.md). + enum class Backend { TripoSR, TripoSG, Trellis2 }; + + // The backend a surface should preselect when the user didn't choose one: + // Trellis2 when its runtime is available on this machine, else TripoSR. + static Backend defaultBackend(); struct Options { Options(); // out-of-line (same idiom as UniRig::Options) @@ -95,6 +105,25 @@ class MeshGenPredictor { Backend backend = Backend::TripoSR; int flowSteps = 25; float guidanceScale = 7.0f; + + // ---- TRELLIS.2-only options (Backend::Trellis2) ----------------------- + unsigned seed = 42; // deterministic generation seed + QString trellis2Preset = // fast | balanced | high + QStringLiteral("balanced"); + // Game-ready simplification target (Phase 8 presets: Low ~10k / + // Medium ~25k / High ~50k). 0 = keep the original TRELLIS.2 density. + int targetTriangles = 0; + // Bake a tangent-space normal map carrying the full-res source detail + // (only meaningful when the target was simplified; needs bakeTexture). + bool bakeNormalMap = true; + // Test hook: drive the sidecar's --mock synthetic generation (no GPU, + // no TRELLIS.2 models) — used by the plumbing e2e tests. + bool trellis2Mock = false; + // Phase 9: where to persist the raw generation (QTM3D interchange) so + // textures/LODs can be re-baked later without re-running inference. + // Empty = don't keep. The kept path lands in Result::sourceInterchangePath. + QString trellis2SourceKeepDir; + QString trellis2SourceKeepBaseName; }; struct Result { @@ -107,6 +136,11 @@ class MeshGenPredictor { // Baked-texture path (Options::bakeTexture): UV0 per vertex + the baked // diffuse image. Both empty/null when the bake was disabled or fell back. std::vector uvs; // Nx2 in [0,1] + // Optional precomputed smooth shading normals (Nx3). When present, + // MeshGenBuilder uses them instead of recomputing from the (possibly + // seam-split) index buffer — the TRELLIS.2 bake provides + // position-welded ones so chart seams stay smooth. + std::vector normals; QImage texture; int vertexCount = 0; int triangleCount = 0; @@ -116,6 +150,18 @@ class MeshGenPredictor { // TripoSG's field is already +Y-up (upstream exports the marching-cubes // trimesh as-is), so its dispatch sets this false to skip the bake. bool bakeTripoSROrientation = true; + + // ---- TRELLIS.2 extras (empty/null for the other backends) ------------- + // Real baked PBR maps from the sparse attribute volume (Trellis2Bake). + // When present, MeshGenBuilder binds them into the canonical + // normal_map/roughness/metallic slots and SKIPS the #404 PbrMapSynth + // guess-from-albedo chain. + QImage normalMap; // tangent-space, OpenGL +Y up + QImage roughnessMap; // grayscale + QImage metallicMap; // grayscale + // Phase 9: the preserved full-resolution generation (QTM3D interchange) + // so textures/LODs can be re-baked later without re-running inference. + QString sourceInterchangePath; }; // True only when built with ENABLE_ONNX. (Model presence is checked per call.) diff --git a/src/ImageTo3D/Trellis2Bake.cpp b/src/ImageTo3D/Trellis2Bake.cpp new file mode 100644 index 000000000..61fa4bd3c --- /dev/null +++ b/src/ImageTo3D/Trellis2Bake.cpp @@ -0,0 +1,1705 @@ +#include "Trellis2Bake.h" +#include "MeshRefine.h" +#include + +#include "MeshSegmenter.h" // connectedComponents (pure-data union-find) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Trellis2Bake { + +namespace { + +// ---- tiny vec3 helpers ------------------------------------------------------ +inline void sub3(const float* a, const float* b, float* o) +{ o[0] = a[0] - b[0]; o[1] = a[1] - b[1]; o[2] = a[2] - b[2]; } +inline float dot3(const float* a, const float* b) +{ return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; } +inline void cross3(const float* a, const float* b, float* o) +{ + o[0] = a[1] * b[2] - a[2] * b[1]; + o[1] = a[2] * b[0] - a[0] * b[2]; + o[2] = a[0] * b[1] - a[1] * b[0]; +} +inline float len3(const float* a) { return std::sqrt(dot3(a, a)); } +inline void normalize3(float* a) +{ + const float l = len3(a); + if (l > 1e-20f) { a[0] /= l; a[1] /= l; a[2] /= l; } +} +inline uint8_t toByte(float v) +{ return static_cast(std::clamp(v, 0.0f, 1.0f) * 255.0f + 0.5f); } + +inline uint64_t packKey(int64_t x, int64_t y, int64_t z) +{ + // 21 bits per axis, offset so slightly-negative cells stay valid. + const uint64_t bias = 1u << 20; + return ((static_cast(x + bias) & 0x1FFFFF) << 42) + | ((static_cast(y + bias) & 0x1FFFFF) << 21) + | (static_cast(z + bias) & 0x1FFFFF); +} + +// Area-weighted smooth vertex normals. +std::vector smoothNormals(const std::vector& positions, + const std::vector& indices) +{ + std::vector n(positions.size(), 0.0f); + for (size_t t = 0; t + 2 < indices.size(); t += 3) { + const float* p0 = &positions[indices[t + 0] * 3]; + const float* p1 = &positions[indices[t + 1] * 3]; + const float* p2 = &positions[indices[t + 2] * 3]; + float e1[3], e2[3], fn[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + cross3(e1, e2, fn); // length ∝ 2×area — the weighting + for (int k = 0; k < 3; ++k) { + n[indices[t + k] * 3 + 0] += fn[0]; + n[indices[t + k] * 3 + 1] += fn[1]; + n[indices[t + k] * 3 + 2] += fn[2]; + } + } + for (size_t v = 0; v + 2 < n.size(); v += 3) { + float* nv = &n[v]; + const float l = len3(nv); + if (l > 1e-20f) { nv[0] /= l; nv[1] /= l; nv[2] /= l; } + else { nv[0] = 0.0f; nv[1] = 1.0f; nv[2] = 0.0f; } + } + return n; +} + +// Smooth vertex normals with POSITION WELDING: vertices at bit-identical +// positions (e.g. xatlas chart-seam splits, marching-cubes duplicates) share +// one accumulated normal, so seams don't read as hard edges in a normal bake. +std::vector smoothNormalsWelded(const std::vector& positions, + const std::vector& indices) +{ + struct PosKey { + uint32_t a, b, c; + bool operator==(const PosKey& o) const + { return a == o.a && b == o.b && c == o.c; } + }; + struct PosKeyHash { + size_t operator()(const PosKey& k) const + { + uint64_t h = k.a; + h = h * 0x9E3779B97F4A7C15ull + k.b; + h = h * 0x9E3779B97F4A7C15ull + k.c; + return static_cast(h ^ (h >> 32)); + } + }; + const size_t nv = positions.size() / 3; + std::unordered_map canonOf; + canonOf.reserve(nv * 2); + std::vector canon(nv); + for (size_t v = 0; v < nv; ++v) { + PosKey k; + std::memcpy(&k.a, &positions[v * 3 + 0], 4); + std::memcpy(&k.b, &positions[v * 3 + 1], 4); + std::memcpy(&k.c, &positions[v * 3 + 2], 4); + canon[v] = canonOf.emplace(k, static_cast(v)).first->second; + } + std::vector acc(positions.size(), 0.0f); + for (size_t t = 0; t + 2 < indices.size(); t += 3) { + const float* p0 = &positions[indices[t + 0] * 3]; + const float* p1 = &positions[indices[t + 1] * 3]; + const float* p2 = &positions[indices[t + 2] * 3]; + float e1[3], e2[3], fn[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + cross3(e1, e2, fn); + for (int k = 0; k < 3; ++k) { + const uint32_t cv = canon[indices[t + k]]; + acc[cv * 3 + 0] += fn[0]; + acc[cv * 3 + 1] += fn[1]; + acc[cv * 3 + 2] += fn[2]; + } + } + std::vector n(positions.size()); + for (size_t v = 0; v < nv; ++v) { + float nv3[3] = {acc[canon[v] * 3 + 0], acc[canon[v] * 3 + 1], + acc[canon[v] * 3 + 2]}; + const float l = len3(nv3); + if (l > 1e-20f) { nv3[0] /= l; nv3[1] /= l; nv3[2] /= l; } + else { nv3[0] = 0.0f; nv3[1] = 1.0f; nv3[2] = 0.0f; } + std::memcpy(&n[v * 3], nv3, sizeof(nv3)); + } + return n; +} + +// ---- sparse uniform grid over source triangles for closest-point queries --- +class TriangleGrid { +public: + void build(const std::vector& positions, + const std::vector& indices) + { + m_positions = &positions; + m_indices = &indices; + m_triCount = static_cast(indices.size() / 3); + float mn[3] = {std::numeric_limits::max(), + std::numeric_limits::max(), + std::numeric_limits::max()}; + float mx[3] = {-mn[0], -mn[1], -mn[2]}; + const size_t nv = positions.size() / 3; + for (size_t v = 0; v < nv; ++v) { + for (int k = 0; k < 3; ++k) { + mn[k] = std::min(mn[k], positions[v * 3 + k]); + mx[k] = std::max(mx[k], positions[v * 3 + k]); + } + } + float diag[3] = {mx[0] - mn[0], mx[1] - mn[1], mx[2] - mn[2]}; + const float d = std::max(1e-6f, len3(diag)); + // Cell size tracks triangle density: a fixed coarse grid puts + // hundreds of triangles in every cell on multi-million-triangle + // TRELLIS sources and each texel query then tests them all — the + // bake went from seconds to tens of minutes. ~2 cells per average + // triangle edge keeps candidate sets small at any density. + const int gridN = std::clamp( + static_cast(2.0 * std::cbrt(static_cast( + std::max(1, m_triCount)))), 64, 256); + m_cell = d / static_cast(gridN); + for (int k = 0; k < 3; ++k) m_min[k] = mn[k]; + for (int k = 0; k < 3; ++k) { + m_lo[k] = cellIndex(mn[k], k); + m_hi[k] = cellIndex(mx[k], k); + } + m_cells.reserve(static_cast(m_triCount) * 2); + for (int t = 0; t < m_triCount; ++t) { + const float* p[3] = {&positions[indices[t * 3 + 0] * 3], + &positions[indices[t * 3 + 1] * 3], + &positions[indices[t * 3 + 2] * 3]}; + int lo[3], hi[3]; + for (int k = 0; k < 3; ++k) { + const float a = std::min({p[0][k], p[1][k], p[2][k]}); + const float b = std::max({p[0][k], p[1][k], p[2][k]}); + lo[k] = cellIndex(a, k); + hi[k] = cellIndex(b, k); + } + for (int x = lo[0]; x <= hi[0]; ++x) + for (int y = lo[1]; y <= hi[1]; ++y) + for (int z = lo[2]; z <= hi[2]; ++z) + m_cells[packKey(x, y, z)].push_back(t); + } + } + + // Closest point on the whole surface. Returns triangle index (or -1 for + // an empty mesh) and fills the closest point + barycentrics. + int closest(const float p[3], float outPoint[3], float outBary[3]) const + { + if (m_triCount == 0) + return -1; + const int cx = cellIndex(p[0], 0); + const int cy = cellIndex(p[1], 1); + const int cz = cellIndex(p[2], 2); + float bestD2 = std::numeric_limits::max(); + int bestTri = -1; + // NB deliberately NO per-query visited-triangle dedup: the old + // mutable stamp array made closest() single-threaded, and with the + // density-scaled grid a triangle spans so few cells that the odd + // duplicate distance test is cheaper than any synchronisation. + // closest() is const + thread-safe — the parallel bake depends on it. + auto visitCell = [&](int x, int y, int z) { + const auto it = m_cells.find(packKey(x, y, z)); + if (it == m_cells.end()) + return; + for (int t : it->second) { + const uint32_t* idx = &(*m_indices)[t * 3]; + const float* a = &(*m_positions)[idx[0] * 3]; + const float* b = &(*m_positions)[idx[1] * 3]; + const float* c = &(*m_positions)[idx[2] * 3]; + float cp[3], bc[3]; + closestPointOnTriangle(a, b, c, p, cp, bc); + float dvec[3]; + sub3(cp, p, dvec); + const float d2 = dot3(dvec, dvec); + if (d2 < bestD2) { + bestD2 = d2; + bestTri = t; + std::memcpy(outPoint, cp, sizeof(cp)); + std::memcpy(outBary, bc, sizeof(bc)); + } + } + }; + // Rings beyond the occupied grid bounds contain nothing — cap there. + const int maxRing = std::max({std::abs(cx - m_lo[0]), std::abs(m_hi[0] - cx), + std::abs(cy - m_lo[1]), std::abs(m_hi[1] - cy), + std::abs(cz - m_lo[2]), std::abs(m_hi[2] - cz)}) + + 1; + for (int r = 0; r <= maxRing; ++r) { + // Once we have a hit, stop when the ring's nearest possible + // distance already exceeds the best. + if (bestTri >= 0) { + const float ringMin = (r - 1) * m_cell; + if (ringMin > 0.0f && ringMin * ringMin > bestD2) + break; + } + // True O(r^2) shell iteration — the 6 faces of the ring cube. + // (An earlier full-cube scan with a shell test was O(r^3) per + // ring and cratered on texels far from the source surface.) + if (r == 0) { + visitCell(cx, cy, cz); + continue; + } + for (int dx = -r; dx <= r; ++dx) { + for (int dy = -r; dy <= r; ++dy) { + visitCell(cx + dx, cy + dy, cz - r); + visitCell(cx + dx, cy + dy, cz + r); + } + for (int dz = -r + 1; dz <= r - 1; ++dz) { + visitCell(cx + dx, cy - r, cz + dz); + visitCell(cx + dx, cy + r, cz + dz); + } + } + for (int dy = -r + 1; dy <= r - 1; ++dy) { + for (int dz = -r + 1; dz <= r - 1; ++dz) { + visitCell(cx - r, cy + dy, cz + dz); + visitCell(cx + r, cy + dy, cz + dz); + } + } + } + if (bestTri < 0) { + // Grid rings exhausted without a candidate (degenerate layout) — + // brute-force so the bake never silently fails. + for (int t = 0; t < m_triCount; ++t) { + const uint32_t* idx = &(*m_indices)[t * 3]; + float cp[3], bc[3]; + closestPointOnTriangle(&(*m_positions)[idx[0] * 3], + &(*m_positions)[idx[1] * 3], + &(*m_positions)[idx[2] * 3], p, cp, bc); + float dvec[3]; + sub3(cp, p, dvec); + const float d2 = dot3(dvec, dvec); + if (d2 < bestD2) { + bestD2 = d2; + bestTri = t; + std::memcpy(outPoint, cp, sizeof(cp)); + std::memcpy(outBary, bc, sizeof(bc)); + } + } + } + return bestTri; + } + +private: + int cellIndex(float v, int axis) const + { + return static_cast(std::floor((v - m_min[axis]) / m_cell)); + } + + const std::vector* m_positions = nullptr; + const std::vector* m_indices = nullptr; + int m_triCount = 0; + float m_min[3] = {0, 0, 0}; + int m_lo[3] = {0, 0, 0}; + int m_hi[3] = {0, 0, 0}; + float m_cell = 1.0f; + std::unordered_map> m_cells; +}; + +} // namespace + +// ---- closest point on triangle (Ericson, RTCD §5.1.5) ----------------------- +void closestPointOnTriangle(const float a[3], const float b[3], + const float c[3], const float p[3], + float outClosest[3], float outBary[3]) +{ + float ab[3], ac[3], ap[3]; + sub3(b, a, ab); + sub3(c, a, ac); + sub3(p, a, ap); + const float d1 = dot3(ab, ap); + const float d2 = dot3(ac, ap); + auto emitBary = [&](float u, float v, float w) { + outBary[0] = u; outBary[1] = v; outBary[2] = w; + for (int k = 0; k < 3; ++k) + outClosest[k] = u * a[k] + v * b[k] + w * c[k]; + }; + if (d1 <= 0.0f && d2 <= 0.0f) { emitBary(1, 0, 0); return; } + + float bp[3]; + sub3(p, b, bp); + const float d3 = dot3(ab, bp); + const float d4 = dot3(ac, bp); + if (d3 >= 0.0f && d4 <= d3) { emitBary(0, 1, 0); return; } + + const float vc = d1 * d4 - d3 * d2; + if (vc <= 0.0f && d1 >= 0.0f && d3 <= 0.0f) { + const float v = d1 / (d1 - d3); + emitBary(1.0f - v, v, 0.0f); + return; + } + + float cp[3]; + sub3(p, c, cp); + const float d5 = dot3(ab, cp); + const float d6 = dot3(ac, cp); + if (d6 >= 0.0f && d5 <= d6) { emitBary(0, 0, 1); return; } + + const float vb = d5 * d2 - d1 * d6; + if (vb <= 0.0f && d2 >= 0.0f && d6 <= 0.0f) { + const float w = d2 / (d2 - d6); + emitBary(1.0f - w, 0.0f, w); + return; + } + + const float va = d3 * d6 - d5 * d4; + if (va <= 0.0f && (d4 - d3) >= 0.0f && (d5 - d6) >= 0.0f) { + const float w = (d4 - d3) / ((d4 - d3) + (d5 - d6)); + emitBary(0.0f, 1.0f - w, w); + return; + } + + const float denom = 1.0f / (va + vb + vc); + const float v = vb * denom; + const float w = vc * denom; + emitBary(1.0f - v - w, v, w); +} + +// ---- SparseVolumeSampler ----------------------------------------------------- +void SparseVolumeSampler::build(const uint32_t* coords, const uint8_t* attrs, + int count, float voxelSize, + const float origin[3]) +{ + m_map.clear(); + m_attrs = attrs; + m_count = count; + m_voxelSize = voxelSize > 0.0f ? voxelSize : 1.0f; + for (int k = 0; k < 3; ++k) + m_origin[k] = origin[k]; + m_map.reserve(static_cast(count) * 2); + for (int i = 0; i < count; ++i) + m_map.emplace(packKey(coords[i * 3 + 0], coords[i * 3 + 1], + coords[i * 3 + 2]), i); +} + +bool SparseVolumeSampler::sample(const float p[3], float out[6]) const +{ + // Neutral defaults (mid gray, dielectric, matte, opaque). + out[0] = out[1] = out[2] = 0.5f; + out[3] = 0.0f; + out[4] = 0.8f; + out[5] = 1.0f; + if (m_count <= 0) + return false; + + // Continuous voxel coordinate with voxel CENTRE at ijk + 0.5. + float g[3]; + for (int k = 0; k < 3; ++k) + g[k] = (p[k] - m_origin[k]) / m_voxelSize - 0.5f; + const int64_t bx = static_cast(std::floor(g[0])); + const int64_t by = static_cast(std::floor(g[1])); + const int64_t bz = static_cast(std::floor(g[2])); + const float fx = g[0] - bx, fy = g[1] - by, fz = g[2] - bz; + + // ALPHA-WEIGHTED trilinear for the colour/material channels: TRELLIS + // wraps the surface in a two-layer band — an inner layer carrying the + // real attributes (alpha≈1) and an OUTER transparent-black "air" layer + // (alpha≈0). Bake texels sit between the layers, so a plain trilinear + // average mixes the surface colour ~50/50 with transparent black and the + // result renders as dark mottling (measured on a real generation: 45% of + // occupied voxels are the alpha≈0 skin). Weighting by voxel alpha is the + // standard premultiplied fix; the alpha CHANNEL itself stays plainly + // interpolated so genuine transparency still comes through. + float acc[6] = {0, 0, 0, 0, 0, 0}; + float wsum = 0.0f; // plain trilinear weight (alpha channel) + float wa = 0.0f; // alpha-weighted (colour/material channels) + for (int dx = 0; dx <= 1; ++dx) { + for (int dy = 0; dy <= 1; ++dy) { + for (int dz = 0; dz <= 1; ++dz) { + const auto it = m_map.find(packKey(bx + dx, by + dy, bz + dz)); + if (it == m_map.end()) + continue; + const float w = (dx ? fx : 1.0f - fx) + * (dy ? fy : 1.0f - fy) + * (dz ? fz : 1.0f - fz); + if (w <= 0.0f) + continue; + const uint8_t* row = m_attrs + static_cast(it->second) * 6; + const float va = row[5] / 255.0f; + for (int c = 0; c < 5; ++c) + acc[c] += w * va * (row[c] / 255.0f); + acc[5] += w * va; + wsum += w; + wa += w * va; + } + } + } + // Accept the trilinear result only when the neighbourhood carries REAL + // coverage. The transparent skin is not alpha == 0 but alpha ≈ 2/255, so + // a pure-skin neighbourhood still produces wa ≈ 0.007·wsum — a tiny + // absolute threshold happily "alpha-weighted" black skin against nothing + // (measured: ~1/3 of bake texels). Coverage below 25% falls through to + // the nearest-OPAQUE search instead. + if (wsum > 1e-6f && wa > 0.25f * wsum) { + for (int c = 0; c < 5; ++c) + out[c] = acc[c] / wa; + out[5] = acc[5] / wsum; // true coverage + return true; + } + + // Nearest occupied voxel in growing Chebyshev shells (surface points can + // land just outside the occupied band after simplification). + const int64_t rx = static_cast(std::llround(g[0])); + const int64_t ry = static_cast(std::llround(g[1])); + const int64_t rz = static_cast(std::llround(g[2])); + // Nearest OPAQUE voxel only. TRELLIS can mark whole surface patches + // transparent with the real colour several voxels deeper (measured skin + // thickness up to ~3-5 voxels on real generations) — a transparent hit + // carries no usable colour, so the skin is excluded outright and the + // search reaches deep enough to cross it. + for (int r = 1; r <= 8; ++r) { + int best = -1; + float bestD2 = std::numeric_limits::max(); + for (int64_t x = rx - r; x <= rx + r; ++x) { + for (int64_t y = ry - r; y <= ry + r; ++y) { + for (int64_t z = rz - r; z <= rz + r; ++z) { + if (std::max({std::llabs(x - rx), std::llabs(y - ry), + std::llabs(z - rz)}) != r) + continue; + const auto it = m_map.find(packKey(x, y, z)); + if (it == m_map.end()) + continue; + const float va = m_attrs[static_cast(it->second) * 6 + 5] + / 255.0f; + if (va < 0.25f) + continue; // skin — no usable colour + const float d2 = float(x - g[0]) * float(x - g[0]) + + float(y - g[1]) * float(y - g[1]) + + float(z - g[2]) * float(z - g[2]); + if (d2 < bestD2) { + bestD2 = d2; + best = it->second; + } + } + } + } + if (best >= 0) { + const uint8_t* row = m_attrs + static_cast(best) * 6; + for (int c = 0; c < 6; ++c) + out[c] = row[c] / 255.0f; + return true; + } + } + return false; +} + +// ---- unifyWinding -------------------------------------------------------------- +int unifyWinding(const std::vector& positions, + std::vector& indices) +{ + const size_t faceCount = indices.size() / 3; + if (faceCount == 0) + return 0; + // Undirected edge -> up to two (face, direction) uses. Edges used by 3+ + // faces are non-manifold and excluded from propagation. + struct EdgeUse { int32_t face[2]; uint8_t dir[2]; uint8_t n; }; + std::unordered_map edges; + edges.reserve(indices.size()); + auto edgeKey = [](uint32_t a, uint32_t b) { + const uint32_t lo = std::min(a, b), hi = std::max(a, b); + return (static_cast(lo) << 32) | hi; + }; + for (size_t f = 0; f < faceCount; ++f) { + for (int k = 0; k < 3; ++k) { + const uint32_t a = indices[f * 3 + k]; + const uint32_t b = indices[f * 3 + (k + 1) % 3]; + if (a == b) + continue; + EdgeUse& e = edges[edgeKey(a, b)]; + if (e.n < 2) { + e.face[e.n] = static_cast(f); + e.dir[e.n] = a < b ? 0 : 1; + } + if (e.n < 255) + ++e.n; + } + } + + // BFS: consistent orientation means the two faces traverse the shared + // edge in OPPOSITE directions (after accounting for flips applied so far). + std::vector state(faceCount, 0); // 0 unvisited, 1 keep, 2 flip + std::vector queue; + std::vector component; + std::vector compOf(faceCount, -1); + std::vector> comps; + int flipped = 0; + for (size_t seed = 0; seed < faceCount; ++seed) { + if (state[seed]) + continue; + state[seed] = 1; + queue.clear(); + component.clear(); + queue.push_back(static_cast(seed)); + component.push_back(static_cast(seed)); + while (!queue.empty()) { + const uint32_t f = queue.back(); + queue.pop_back(); + const bool fFlip = state[f] == 2; + for (int k = 0; k < 3; ++k) { + const uint32_t a = indices[f * 3 + k]; + const uint32_t b = indices[f * 3 + (k + 1) % 3]; + if (a == b) + continue; + const auto it = edges.find(edgeKey(a, b)); + if (it == edges.end() || it->second.n != 2) + continue; // border or non-manifold — no propagation + const EdgeUse& e = it->second; + const int slot = e.face[0] == static_cast(f) ? 0 : 1; + const int32_t g = e.face[1 - slot]; + if (g < 0 || state[g]) + continue; + // Effective directions after the flips chosen so far: a flip + // reverses every edge direction of the face. + const bool dirF = (e.dir[slot] != 0) != fFlip; + const bool dirG = (e.dir[1 - slot] != 0); + // Consistent when directions differ; if they'd match, flip g. + const bool gFlip = (dirG == dirF); + state[g] = gFlip ? 2 : 1; + queue.push_back(static_cast(g)); + component.push_back(static_cast(g)); + } + } + // Orient the whole component OUTWARD. Signed volume works for closed + // shells but is meaningless for the open sheets and small islands a + // non-manifold source shatters into (BFS can't cross non-manifold + // edges, so complex organic decodes yield MANY islands — misoriented + // small ones rendered as holes on real generations). The centroid + // heuristic — do face normals point away from the island's own + // centroid relative to the global centroid? — behaves like signed + // volume on closed shells and stays meaningful on patches: score = + // Σ area·dot(n̂, faceCentroid − globalCentroid). + double cx = 0, cy = 0, cz = 0; + { + size_t n = 0; + for (uint32_t f : component) { + for (int k = 0; k < 3; ++k) { + const float* pv = &positions[indices[f * 3 + k] * 3]; + cx += pv[0]; cy += pv[1]; cz += pv[2]; + } + n += 3; + } + if (n) { cx /= n; cy /= n; cz /= n; } + } + double score = 0.0; + for (uint32_t f : component) { + const float* p0 = &positions[indices[f * 3 + 0] * 3]; + const float* p1 = &positions[indices[f * 3 + 1] * 3]; + const float* p2 = &positions[indices[f * 3 + 2] * 3]; + float e1[3], e2[3], fn[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + cross3(e1, e2, fn); // length ∝ area + const double gx = (p0[0] + p1[0] + p2[0]) / 3.0 - cx; + const double gy = (p0[1] + p1[1] + p2[1]) / 3.0 - cy; + const double gz = (p0[2] + p1[2] + p2[2]) / 3.0 - cz; + double v = fn[0] * gx + fn[1] * gy + fn[2] * gz; + if (state[f] == 2) + v = -v; + score += v; + } + const bool flipComponent = score < 0.0; + for (uint32_t f : component) { + const bool doFlip = (state[f] == 2) != flipComponent; + if (doFlip) { + std::swap(indices[f * 3 + 1], indices[f * 3 + 2]); + ++flipped; + } + compOf[f] = static_cast(comps.size()); + } + comps.push_back(component); + } + + // ---- Phase 2: re-orient SMALL islands against the dominant surface ------ + // A fuzzy voxel decode (fur, hair, foliage) shatters into hundreds of + // 1-3-triangle wisp islands attached to the body only through + // non-manifold edges or shared vertices — BFS can't reach them, and for + // a near-flat wisp the centroid score above is ~0, a coin flip (measured + // on a real generation: 34% of small-island faces flipped → rendered as + // dark pepper speckle under backface culling). Re-orient each small + // island to agree with the LARGEST island's smooth normal field at the + // vertices they share; islands sharing nothing keep the centroid choice. + if (comps.size() > 1) { + size_t largest = 0; + for (size_t c = 1; c < comps.size(); ++c) + if (comps[c].size() > comps[largest].size()) + largest = c; + const size_t smallLimit = std::max(100, faceCount / 100); + if (comps[largest].size() > smallLimit) { + // Area-weighted vertex normals of the dominant island (current, + // post-phase-1 winding). + std::unordered_map> bigN; + for (uint32_t f : comps[largest]) { + const float* p0 = &positions[indices[f * 3 + 0] * 3]; + const float* p1 = &positions[indices[f * 3 + 1] * 3]; + const float* p2 = &positions[indices[f * 3 + 2] * 3]; + float e1[3], e2[3], fn[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + cross3(e1, e2, fn); + for (int k = 0; k < 3; ++k) { + auto& n = bigN[indices[f * 3 + k]]; + n[0] += fn[0]; n[1] += fn[1]; n[2] += fn[2]; + } + } + for (size_t c = 0; c < comps.size(); ++c) { + if (c == largest || comps[c].size() > smallLimit) + continue; + double agree = 0.0; + for (uint32_t f : comps[c]) { + const float* p0 = &positions[indices[f * 3 + 0] * 3]; + const float* p1 = &positions[indices[f * 3 + 1] * 3]; + const float* p2 = &positions[indices[f * 3 + 2] * 3]; + float e1[3], e2[3], fn[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + cross3(e1, e2, fn); + for (int k = 0; k < 3; ++k) { + const auto it = bigN.find(indices[f * 3 + k]); + if (it == bigN.end()) + continue; + agree += fn[0] * it->second[0] + + fn[1] * it->second[1] + + fn[2] * it->second[2]; + } + } + if (agree < 0.0) { + for (uint32_t f : comps[c]) { + std::swap(indices[f * 3 + 1], indices[f * 3 + 2]); + ++flipped; + } + } + } + } + } + return flipped; +} + +// ---- makeGameReady ----------------------------------------------------------- +GameReadyResult makeGameReady(const std::vector& positions, + const std::vector& indices, + const GameReadyOptions& opts) +{ + GameReadyResult r; + const size_t nv = positions.size() / 3; + if (nv == 0 || indices.size() < 3 || indices.size() % 3 != 0 + || positions.size() % 3 != 0) { + r.error = QStringLiteral("makeGameReady: empty/degenerate input mesh."); + return r; + } + for (uint32_t i : indices) { + if (i >= nv) { + r.error = QStringLiteral("makeGameReady: index out of range."); + return r; + } + } + r.inputTriangles = static_cast(indices.size() / 3); + + // ---- 1. weld near-duplicate vertices (quantized remap) ----------------- + float mn[3] = {positions[0], positions[1], positions[2]}; + float mx[3] = {positions[0], positions[1], positions[2]}; + for (size_t v = 0; v < nv; ++v) { + for (int k = 0; k < 3; ++k) { + mn[k] = std::min(mn[k], positions[v * 3 + k]); + mx[k] = std::max(mx[k], positions[v * 3 + k]); + } + } + float diagv[3] = {mx[0] - mn[0], mx[1] - mn[1], mx[2] - mn[2]}; + const float diag = std::max(1e-9f, len3(diagv)); + const float eps = opts.weldEpsilonAbsolute > 0.0f + ? opts.weldEpsilonAbsolute + : (opts.weldEpsilonFraction > 0.0f ? diag * opts.weldEpsilonFraction + : 0.0f); + + std::vector keys(nv * 3); + for (size_t v = 0; v < nv * 3; ++v) { + keys[v] = eps > 0.0f + ? static_cast(std::llround(positions[v] / eps)) + : 0; + } + std::vector remap(nv); + size_t weldedCount; + std::vector idx; + std::vector pos; + if (eps > 0.0f) { + weldedCount = meshopt_generateVertexRemap( + remap.data(), indices.data(), indices.size(), keys.data(), nv, + sizeof(int32_t) * 3); + } else { + weldedCount = meshopt_generateVertexRemap( + remap.data(), indices.data(), indices.size(), positions.data(), nv, + sizeof(float) * 3); + } + pos.resize(weldedCount * 3); + meshopt_remapVertexBuffer(pos.data(), positions.data(), nv, + sizeof(float) * 3, remap.data()); + idx.resize(indices.size()); + meshopt_remapIndexBuffer(idx.data(), indices.data(), indices.size(), + remap.data()); + r.weldedVertices = static_cast(nv - weldedCount); + + // ---- 2. drop degenerate triangles --------------------------------------- + size_t w = 0; + for (size_t t = 0; t + 2 < idx.size(); t += 3) { + if (idx[t] == idx[t + 1] || idx[t + 1] == idx[t + 2] + || idx[t] == idx[t + 2]) + continue; + idx[w++] = idx[t]; + idx[w++] = idx[t + 1]; + idx[w++] = idx[t + 2]; + } + r.removedTriangles += static_cast((idx.size() - w) / 3); + idx.resize(w); + if (idx.empty()) { + r.error = QStringLiteral("makeGameReady: mesh degenerated to nothing."); + return r; + } + + // ---- 3. drop tiny disconnected components ------------------------------- + std::vector islandId; + const int islands = MeshSegmenter::connectedComponents( + static_cast(weldedCount), idx.data(), + static_cast(idx.size()), islandId); + if (islands > 1) { + std::vector triPerIsland(islands, 0); + for (size_t t = 0; t + 2 < idx.size(); t += 3) + ++triPerIsland[islandId[idx[t]]]; + const int largest = static_cast( + std::max_element(triPerIsland.begin(), triPerIsland.end()) + - triPerIsland.begin()); + const int totalTris = static_cast(idx.size() / 3); + const int threshold = std::max( + opts.minComponentTriangles, + static_cast(opts.minComponentFraction * totalTris)); + std::vector keep(islands, 0); + for (int i = 0; i < islands; ++i) + keep[i] = (i == largest || triPerIsland[i] >= threshold) ? 1 : 0; + size_t w2 = 0; + for (size_t t = 0; t + 2 < idx.size(); t += 3) { + if (!keep[islandId[idx[t]]]) + continue; + idx[w2++] = idx[t]; + idx[w2++] = idx[t + 1]; + idx[w2++] = idx[t + 2]; + } + for (int i = 0; i < islands; ++i) + if (!keep[i]) ++r.removedComponents; + r.removedTriangles += static_cast((idx.size() - w2) / 3); + idx.resize(w2); + } + + // ---- 3b. unify winding ---------------------------------------------------- + // Must run before normals/culling ever see the mesh — raw dual-grid + // output ships large flipped patches (they render as holes under + // backface culling and poison the smooth normals the bake relies on). + unifyWinding(pos, idx); + + // ---- 4. compact unreferenced vertices ----------------------------------- + auto compact = [](std::vector& positionsIo, + std::vector& indicesIo) { + const size_t count = positionsIo.size() / 3; + std::vector map(count, UINT32_MAX); + uint32_t next = 0; + for (uint32_t& i : indicesIo) { + if (map[i] == UINT32_MAX) + map[i] = next++; + i = map[i]; + } + std::vector outp(static_cast(next) * 3); + for (size_t v = 0; v < count; ++v) { + if (map[v] == UINT32_MAX) + continue; + std::memcpy(&outp[map[v] * 3], &positionsIo[v * 3], + sizeof(float) * 3); + } + positionsIo.swap(outp); + }; + compact(pos, idx); + + // ---- 4b. optional Taubin pre-smooth -------------------------------------- + // Voxel decodes of fuzzy subjects (fur, hair, foliage) carry sub-voxel + // micro-pits and wisps that read as dark "pepper" speckle in renders and + // confuse QEM into collapsing thin double-walled features into flipped + // soup. A few volume-preserving Taubin passes flatten noise below the + // voxel scale while leaving real shape (runs on the WELDED mesh, so the + // Laplacian sees true adjacency, and before simplification so QEM ranks + // clean geometry). + if (opts.taubinIterations > 0) + MeshRefine::taubinSmooth(pos, idx, opts.taubinIterations); + + // ---- 5. simplify toward the target -------------------------------------- + if (opts.targetTriangles > 0 + && static_cast(idx.size() / 3) > opts.targetTriangles) { + std::vector simplified(idx.size()); + float resultError = 0.0f; + const size_t targetIndexCount = + static_cast(opts.targetTriangles) * 3; + size_t newCount = meshopt_simplify( + simplified.data(), idx.data(), idx.size(), pos.data(), + pos.size() / 3, sizeof(float) * 3, targetIndexCount, + opts.simplifyTargetError, /*options=*/0, &resultError); + // Game-ready budgets treat the COUNT as the contract (the lost + // detail comes back via the normal-map bake): on dense organic + // sources (a 4.8M-tri TRELLIS decode) the relative error cap stops + // collapsing millions of triangles short of the budget — and the + // downstream xatlas unwrap of that "simplified" mesh then takes tens + // of minutes (its chart compute is superlinear). If the capped pass + // landed far off target, redo it uncapped. + if (newCount > targetIndexCount * 2) { + newCount = meshopt_simplify( + simplified.data(), idx.data(), idx.size(), pos.data(), + pos.size() / 3, sizeof(float) * 3, targetIndexCount, + std::numeric_limits::max(), /*options=*/0, + &resultError); + } + // Topology-preserving QEM can still be STUCK far above the budget on + // non-manifold sources (raw TRELLIS dual-grid output: ~4.9M tris + // refused to go below ~2.7M even uncapped). Game presets promise a + // usable budget — fall back to the topology-free sloppy simplifier, + // which always reaches it; the detail returns via the normal bake. + if (newCount > targetIndexCount * 2) { + newCount = meshopt_simplifySloppy( + simplified.data(), idx.data(), idx.size(), pos.data(), + pos.size() / 3, sizeof(float) * 3, targetIndexCount, + std::numeric_limits::max(), &resultError); + } + simplified.resize(newCount); + idx.swap(simplified); + r.simplifyError = resultError; + compact(pos, idx); + } + + // ---- 6. cache-friendly ordering ------------------------------------------ + meshopt_optimizeVertexCache(idx.data(), idx.data(), idx.size(), + pos.size() / 3); + + r.positions = std::move(pos); + r.indices = std::move(idx); + r.outputTriangles = static_cast(r.indices.size() / 3); + r.ok = true; + return r; +} + +// ---- bake --------------------------------------------------------------------- +BakeResult bake(const std::vector& targetPositions, + const std::vector& targetIndices, + const std::vector& sourcePositions, + const std::vector& sourceIndices, + const SparseVolumeSampler& volume, + const BakeOptions& opts) +{ + BakeResult r; + const size_t nv = targetPositions.size() / 3; + if (nv == 0 || targetIndices.size() < 3 || targetIndices.size() % 3 != 0) { + r.error = QStringLiteral("bake: empty/degenerate target mesh."); + return r; + } + for (uint32_t i : targetIndices) { + if (i >= nv) { + r.error = QStringLiteral("bake: target index out of range."); + return r; + } + } + const size_t snv = sourcePositions.size() / 3; + if (snv == 0 || sourceIndices.size() < 3 || sourceIndices.size() % 3 != 0) { + r.error = QStringLiteral("bake: empty/degenerate source mesh."); + return r; + } + for (uint32_t i : sourceIndices) { + if (i >= snv) { + r.error = QStringLiteral("bake: source index out of range."); + return r; + } + } + const int texSize = std::clamp(opts.textureSize, 64, 8192); + const int ss = std::clamp(opts.supersample, 1, 2); + + // ---- 1. xatlas unwrap of the target ------------------------------------- + xatlas::Atlas* atlas = xatlas::Create(); + xatlas::MeshDecl decl; + decl.vertexCount = static_cast(nv); + decl.vertexPositionData = targetPositions.data(); + decl.vertexPositionStride = sizeof(float) * 3; + decl.indexCount = static_cast(targetIndices.size()); + decl.indexData = targetIndices.data(); + decl.indexFormat = xatlas::IndexFormat::UInt32; + const auto err = xatlas::AddMesh(atlas, decl); + if (err != xatlas::AddMeshError::Success) { + r.error = QStringLiteral("bake: xatlas::AddMesh failed: %1") + .arg(QString::fromLatin1(xatlas::StringForEnum(err))); + xatlas::Destroy(atlas); + return r; + } + // Cancellation during the unwrap: ComputeCharts can run for minutes on + // dense targets and used to be un-cancellable (the Cancel button only + // reached the sampling phase). xatlas's progress callback returns false + // to abort; throttled so the callback overhead stays negligible. + struct UnwrapCancelCtx { + const std::function* progress = nullptr; + int counter = 0; + bool cancelled = false; + } ucc; + if (opts.progress) { + ucc.progress = &opts.progress; + xatlas::SetProgressCallback( + atlas, + [](xatlas::ProgressCategory, int, void* user) -> bool { + auto* c = static_cast(user); + if ((++c->counter & 63) != 0) + return true; + if (!(*c->progress)(0, 1)) { + c->cancelled = true; + return false; + } + return true; + }, + &ucc); + } + xatlas::PackOptions pack; + pack.resolution = static_cast(texSize); + pack.padding = std::max(1, opts.dilatePx); + pack.bilinear = true; + xatlas::Generate(atlas, /*chartOptions=*/{}, pack); + if (ucc.cancelled) { + xatlas::Destroy(atlas); + r = BakeResult{}; + r.cancelled = true; + r.error = QStringLiteral("cancelled"); + return r; + } + if (atlas->meshCount != 1 || atlas->width == 0 || atlas->height == 0) { + r.error = QStringLiteral("bake: xatlas produced no atlas."); + xatlas::Destroy(atlas); + return r; + } + const xatlas::Mesh& xm = atlas->meshes[0]; + const int W = static_cast(atlas->width); + const int H = static_cast(atlas->height); + + r.positions.resize(static_cast(xm.vertexCount) * 3); + r.uvs.resize(static_cast(xm.vertexCount) * 2); + std::vector xref(xm.vertexCount); + for (uint32_t v = 0; v < xm.vertexCount; ++v) { + const xatlas::Vertex& xv = xm.vertexArray[v]; + xref[v] = xv.xref; + std::memcpy(&r.positions[v * 3], + &targetPositions[static_cast(xv.xref) * 3], + sizeof(float) * 3); + r.uvs[v * 2 + 0] = xv.uv[0] / float(W); + r.uvs[v * 2 + 1] = xv.uv[1] / float(H); + } + r.indices.assign(xm.indexArray, xm.indexArray + xm.indexCount); + r.vertexCount = static_cast(xm.vertexCount); + r.triangleCount = static_cast(xm.indexCount / 3); + + // ---- 2. target normals + tangents (Lengyel), source normals ------------- + // Target normals come from the ORIGINAL (pre-split) mesh and are carried + // through the xatlas re-index via xref — computing them on the split mesh + // would flatten every chart seam into a hard edge, and an identity bake + // (source == target) would stop being the flat (128,128,255) map. + std::vector tNormals(static_cast(r.vertexCount) * 3); + { + const std::vector origNormals = + smoothNormals(targetPositions, targetIndices); + for (int v = 0; v < r.vertexCount; ++v) + std::memcpy(&tNormals[static_cast(v) * 3], + &origNormals[static_cast(xref[v]) * 3], + sizeof(float) * 3); + } + const std::vector sNormals = + smoothNormals(sourcePositions, sourceIndices); + std::vector tTangent; // xyzw per vertex (w = handedness) + if (opts.bakeNormalMap) { + std::vector tan1(r.positions.size(), 0.0f); + std::vector tan2(r.positions.size(), 0.0f); + for (size_t t = 0; t + 2 < r.indices.size(); t += 3) { + const uint32_t i0 = r.indices[t], i1 = r.indices[t + 1], + i2 = r.indices[t + 2]; + const float* p0 = &r.positions[i0 * 3]; + const float* p1 = &r.positions[i1 * 3]; + const float* p2 = &r.positions[i2 * 3]; + const float* u0 = &r.uvs[i0 * 2]; + const float* u1 = &r.uvs[i1 * 2]; + const float* u2 = &r.uvs[i2 * 2]; + float e1[3], e2[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + const float du1 = u1[0] - u0[0], dv1 = u1[1] - u0[1]; + const float du2 = u2[0] - u0[0], dv2 = u2[1] - u0[1]; + const float det = du1 * dv2 - du2 * dv1; + if (std::fabs(det) < 1e-20f) + continue; + const float rd = 1.0f / det; + const float sdir[3] = {(e1[0] * dv2 - e2[0] * dv1) * rd, + (e1[1] * dv2 - e2[1] * dv1) * rd, + (e1[2] * dv2 - e2[2] * dv1) * rd}; + const float tdir[3] = {(e2[0] * du1 - e1[0] * du2) * rd, + (e2[1] * du1 - e1[1] * du2) * rd, + (e2[2] * du1 - e1[2] * du2) * rd}; + for (uint32_t i : {i0, i1, i2}) { + for (int k = 0; k < 3; ++k) { + tan1[i * 3 + k] += sdir[k]; + tan2[i * 3 + k] += tdir[k]; + } + } + } + tTangent.resize(static_cast(r.vertexCount) * 4); + for (int v = 0; v < r.vertexCount; ++v) { + const float* n = &tNormals[static_cast(v) * 3]; + const float* t1 = &tan1[static_cast(v) * 3]; + float t[3] = {t1[0] - n[0] * dot3(n, t1), + t1[1] - n[1] * dot3(n, t1), + t1[2] - n[2] * dot3(n, t1)}; + if (len3(t) < 1e-12f) { + // Degenerate UV — pick any tangent orthogonal to n. + const float up[3] = {0.0f, + std::fabs(n[1]) < 0.9f ? 1.0f : 0.0f, + std::fabs(n[1]) < 0.9f ? 0.0f : 1.0f}; + cross3(up, n, t); + } + normalize3(t); + float bc[3]; + cross3(n, t, bc); + const float wsign = + dot3(bc, &tan2[static_cast(v) * 3]) < 0.0f ? -1.0f + : 1.0f; + tTangent[static_cast(v) * 4 + 0] = t[0]; + tTangent[static_cast(v) * 4 + 1] = t[1]; + tTangent[static_cast(v) * 4 + 2] = t[2]; + tTangent[static_cast(v) * 4 + 3] = wsign; + } + } + + // ---- 3. source closest-point accelerator --------------------------------- + TriangleGrid grid; + grid.build(sourcePositions, sourceIndices); + + // ---- 4. rasterize + sample ------------------------------------------------- + QImage baseColor(W, H, QImage::Format_RGBA8888); + baseColor.fill(QColor(110, 110, 110, 255)); + QImage rough(W, H, QImage::Format_Grayscale8); + rough.fill(204); + QImage metal(W, H, QImage::Format_Grayscale8); + metal.fill(0); + QImage normal; + if (opts.bakeNormalMap) { + normal = QImage(W, H, QImage::Format_RGB888); + normal.fill(QColor(128, 128, 255)); + } + std::vector covered(static_cast(W) * H, 0); + + // All atlas (xm) references end here — the r.* copies carry everything + // the sampling needs, so the atlas is freed before the heavy phase. + xatlas::Destroy(atlas); + + // ---- Phase 1: serial UV rasterization → texel job list ------------------- + // Cheap (pure 2D coverage). Each covered texel becomes one independent + // job; first triangle wins a texel (xatlas charts don't overlap). + struct TexelJob { uint32_t lin; uint32_t tri; }; + std::vector jobs; + jobs.reserve(static_cast(W) * H / 2); + for (uint32_t t = 0; t + 2 < static_cast(r.indices.size()); t += 3) { + const uint32_t i0 = r.indices[t], i1 = r.indices[t + 1], + i2 = r.indices[t + 2]; + const float uv0[2] = {r.uvs[i0 * 2] * W, r.uvs[i0 * 2 + 1] * H}; + const float uv1[2] = {r.uvs[i1 * 2] * W, r.uvs[i1 * 2 + 1] * H}; + const float uv2[2] = {r.uvs[i2 * 2] * W, r.uvs[i2 * 2 + 1] * H}; + const int minX = std::max(0, static_cast(std::floor( + std::min({uv0[0], uv1[0], uv2[0]})))); + const int maxX = std::min(W - 1, static_cast(std::ceil( + std::max({uv0[0], uv1[0], uv2[0]})))); + const int minY = std::max(0, static_cast(std::floor( + std::min({uv0[1], uv1[1], uv2[1]})))); + const int maxY = std::min(H - 1, static_cast(std::ceil( + std::max({uv0[1], uv1[1], uv2[1]})))); + const float denom = (uv1[1] - uv2[1]) * (uv0[0] - uv2[0]) + + (uv2[0] - uv1[0]) * (uv0[1] - uv2[1]); + if (std::fabs(denom) < 1e-12f) + continue; + const float inv = 1.0f / denom; + for (int y = minY; y <= maxY; ++y) { + for (int x = minX; x <= maxX; ++x) { + const size_t lin = static_cast(y) * W + x; + if (covered[lin]) + continue; + const float px = x + 0.5f, py = y + 0.5f; + const float w0 = ((uv1[1] - uv2[1]) * (px - uv2[0]) + + (uv2[0] - uv1[0]) * (py - uv2[1])) * inv; + const float w1 = ((uv2[1] - uv0[1]) * (px - uv2[0]) + + (uv0[0] - uv2[0]) * (py - uv2[1])) * inv; + const float w2 = 1.0f - w0 - w1; + const float eps = -0.001f; + if (w0 < eps || w1 < eps || w2 < eps) + continue; + covered[lin] = 1; + jobs.push_back({static_cast(lin), t}); + } + } + } + if (jobs.empty()) { + r = BakeResult{}; + r.error = QStringLiteral("bake: no texels covered (unwrap failed?)."); + return r; + } + + // ---- Phase 2: parallel sampling ------------------------------------------- + // Each job owns a unique texel: workers write disjoint pixels through raw + // bits() pointers (detached up front), the grid/volume queries are const, + // and the main thread pumps progress/cancellation off an atomic counter. + // This is what keeps multi-million-triangle TRELLIS sources at seconds- + // to-minutes instead of tens of minutes single-threaded. + uchar* bcBits = baseColor.bits(); + const qsizetype bcBpl = baseColor.bytesPerLine(); + uchar* roBits = rough.bits(); + const qsizetype roBpl = rough.bytesPerLine(); + uchar* meBits = metal.bits(); + const qsizetype meBpl = metal.bytesPerLine(); + uchar* noBits = opts.bakeNormalMap ? normal.bits() : nullptr; + const qsizetype noBpl = opts.bakeNormalMap ? normal.bytesPerLine() : 0; + + std::atomic nextJob{0}; + std::atomic doneJobs{0}; + std::atomic abortBake{false}; + const unsigned nThreads = std::min( + 16u, std::max(1u, std::thread::hardware_concurrency())); + auto workerFn = [&]() { + constexpr size_t kChunk = 1024; + for (;;) { + const size_t start = nextJob.fetch_add(kChunk); + if (start >= jobs.size() || abortBake.load(std::memory_order_relaxed)) + return; + const size_t endJ = std::min(jobs.size(), start + kChunk); + for (size_t j = start; j < endJ; ++j) { + const uint32_t lin = jobs[j].lin; + const uint32_t t = jobs[j].tri; + const int x = static_cast(lin % W); + const int y = static_cast(lin / W); + const uint32_t i0 = r.indices[t], i1 = r.indices[t + 1], + i2 = r.indices[t + 2]; + const float uv0[2] = {r.uvs[i0 * 2] * W, r.uvs[i0 * 2 + 1] * H}; + const float uv1[2] = {r.uvs[i1 * 2] * W, r.uvs[i1 * 2 + 1] * H}; + const float uv2[2] = {r.uvs[i2 * 2] * W, r.uvs[i2 * 2 + 1] * H}; + const float denom = (uv1[1] - uv2[1]) * (uv0[0] - uv2[0]) + + (uv2[0] - uv1[0]) * (uv0[1] - uv2[1]); + if (std::fabs(denom) < 1e-12f) + continue; + const float inv = 1.0f / denom; + + float accAttr[6] = {0, 0, 0, 0, 0, 0}; + float accN[3] = {0, 0, 0}; + int samples = 0; + for (int sy = 0; sy < ss; ++sy) { + for (int sx = 0; sx < ss; ++sx) { + const float px = ss == 1 ? x + 0.5f + : x + (sx + 0.5f) / ss; + const float py = ss == 1 ? y + 0.5f + : y + (sy + 0.5f) / ss; + float w0 = ((uv1[1] - uv2[1]) * (px - uv2[0]) + + (uv2[0] - uv1[0]) * (py - uv2[1])) * inv; + float w1 = ((uv2[1] - uv0[1]) * (px - uv2[0]) + + (uv0[0] - uv2[0]) * (py - uv2[1])) * inv; + float w2 = 1.0f - w0 - w1; + w0 = std::max(w0, 0.0f); + w1 = std::max(w1, 0.0f); + w2 = std::max(w2, 0.0f); + const float wsumB = w0 + w1 + w2; + if (wsumB < 1e-12f) + continue; + w0 /= wsumB; w1 /= wsumB; w2 /= wsumB; + + float P[3], Nt[3]; + for (int k = 0; k < 3; ++k) { + P[k] = w0 * r.positions[i0 * 3 + k] + + w1 * r.positions[i1 * 3 + k] + + w2 * r.positions[i2 * 3 + k]; + Nt[k] = w0 * tNormals[i0 * 3 + k] + + w1 * tNormals[i1 * 3 + k] + + w2 * tNormals[i2 * 3 + k]; + } + normalize3(Nt); + + float S[3], sb[3]; + const int sTri = grid.closest(P, S, sb); + float attr[6]; + if (!volume.sample(sTri >= 0 ? S : P, attr)) + continue; // no opaque voxel within reach — + // attr is unwritten; skip so the + // texel dilates from valid + // neighbours instead of averaging + // garbage (rendered as dark pepper + // speckle on fuzzy subjects) + for (int c = 0; c < 6; ++c) + accAttr[c] += attr[c]; + + if (opts.bakeNormalMap) { + float Ns[3] = {0.0f, 1.0f, 0.0f}; + if (sTri >= 0) { + const uint32_t* sidx = &sourceIndices[sTri * 3]; + for (int k = 0; k < 3; ++k) + Ns[k] = sb[0] * sNormals[sidx[0] * 3 + k] + + sb[1] * sNormals[sidx[1] * 3 + k] + + sb[2] * sNormals[sidx[2] * 3 + k]; + normalize3(Ns); + } else { + std::memcpy(Ns, Nt, sizeof(Ns)); + } + float T[3], wsign = 1.0f; + for (int k = 0; k < 3; ++k) + T[k] = w0 * tTangent[i0 * 4 + k] + + w1 * tTangent[i1 * 4 + k] + + w2 * tTangent[i2 * 4 + k]; + wsign = (w0 * tTangent[i0 * 4 + 3] + + w1 * tTangent[i1 * 4 + 3] + + w2 * tTangent[i2 * 4 + 3]) < 0.0f + ? -1.0f : 1.0f; + const float ndt = dot3(Nt, T); + for (int k = 0; k < 3; ++k) + T[k] -= Nt[k] * ndt; + if (len3(T) < 1e-12f) { + const float up[3] = {0, 1, 0}; + cross3(up, Nt, T); + } + normalize3(T); + float B[3]; + cross3(Nt, T, B); + for (int k = 0; k < 3; ++k) + B[k] *= wsign; + float nts[3] = {dot3(Ns, T), dot3(Ns, B), + dot3(Ns, Nt)}; + normalize3(nts); + for (int k = 0; k < 3; ++k) + accN[k] += nts[k]; + } + ++samples; + } + } + if (samples == 0) { + // Every subsample failed: un-cover the texel so the + // border dilation + background fill treat it like an + // unrasterized one. Jobs own disjoint texels, so this + // byte write is race-free. + covered[lin] = 0; + continue; + } + const float invS = 1.0f / samples; + uchar* bc = bcBits + static_cast(y) * bcBpl + + static_cast(x) * 4; + bc[0] = toByte(accAttr[0] * invS); + bc[1] = toByte(accAttr[1] * invS); + bc[2] = toByte(accAttr[2] * invS); + // OPAQUE by design: game-ready assets bake the surface as + // solid (the whole sampler intentionally reads through the + // transparent TRELLIS skin). Writing the volume's alpha here + // let low-alpha rim texels alpha-blend at render time and + // flash as bright seam speckle in every viewer. + bc[3] = 255; + roBits[static_cast(y) * roBpl + x] = + toByte(accAttr[4] * invS); + meBits[static_cast(y) * meBpl + x] = + toByte(accAttr[3] * invS); + if (noBits) { + float n[3] = {accN[0] * invS, accN[1] * invS, + accN[2] * invS}; + normalize3(n); + uchar* np = noBits + static_cast(y) * noBpl + + static_cast(x) * 3; + np[0] = toByte(n[0] * 0.5f + 0.5f); + np[1] = toByte(n[1] * 0.5f + 0.5f); + np[2] = toByte(n[2] * 0.5f + 0.5f); + } + } + doneJobs.fetch_add(endJ - start); + } + }; + { + std::vector workers; + workers.reserve(nThreads); + for (unsigned i = 0; i < nThreads; ++i) + workers.emplace_back(workerFn); + while (doneJobs.load() < jobs.size() + && !abortBake.load(std::memory_order_relaxed)) { + if (opts.progress + && !opts.progress(static_cast(doneJobs.load()), + static_cast(jobs.size()))) + abortBake.store(true); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + for (auto& th : workers) + th.join(); + } + if (abortBake.load()) { + r = BakeResult{}; + r.cancelled = true; + r.error = QStringLiteral("cancelled"); + return r; + } + + + size_t coveredCount = 0; + for (uint8_t c : covered) + coveredCount += c; + if (coveredCount == 0) { + r = BakeResult{}; + r.error = QStringLiteral("bake: no texels covered (unwrap failed?)."); + return r; + } + + // ---- 5. dilate chart borders on every channel ----------------------------- + struct Channel { + QImage* img; + int bpp; + }; + std::vector channels = {{&baseColor, 4}, {&rough, 1}, {&metal, 1}}; + if (opts.bakeNormalMap) + channels.push_back({&normal, 3}); + for (int pass = 0; pass < opts.dilatePx; ++pass) { + std::vector next = covered; + for (int y = 0; y < H; ++y) { + for (int x = 0; x < W; ++x) { + const size_t lin = static_cast(y) * W + x; + if (covered[lin]) + continue; + for (int dy = -1; dy <= 1 && !next[lin]; ++dy) { + for (int dx = -1; dx <= 1; ++dx) { + const int sx = x + dx, sy = y + dy; + if (sx < 0 || sy < 0 || sx >= W || sy >= H) + continue; + if (!covered[static_cast(sy) * W + sx]) + continue; + for (const Channel& ch : channels) { + std::memcpy( + ch.img->scanLine(y) + + static_cast(x) * ch.bpp, + ch.img->scanLine(sy) + + static_cast(sx) * ch.bpp, + static_cast(ch.bpp)); + } + next[lin] = 1; + break; + } + } + } + } + covered.swap(next); + } + + // Mip-safe background: the dilation ring (a few texels) disappears at + // higher mip levels, and on heavily-charted meshes (a simplified TRELLIS + // dual grid produces thousands of small charts) the neutral background + // then bleeds through as bright seam networks at render time. Fill ALL + // remaining background with the mesh's average baked colour so every mip + // level samples plausible values. + { + double sum[5] = {0, 0, 0, 0, 0}; + size_t n = 0; + for (int y = 0; y < H; ++y) { + const uchar* bcRow = bcBits + static_cast(y) * bcBpl; + const uchar* roRow = roBits + static_cast(y) * roBpl; + const uchar* meRow = meBits + static_cast(y) * meBpl; + for (int x = 0; x < W; ++x) { + if (!covered[static_cast(y) * W + x]) + continue; + sum[0] += bcRow[x * 4 + 0]; + sum[1] += bcRow[x * 4 + 1]; + sum[2] += bcRow[x * 4 + 2]; + sum[3] += roRow[x]; + sum[4] += meRow[x]; + ++n; + } + } + if (n > 0) { + const uchar avg[5] = { + static_cast(sum[0] / n + 0.5), + static_cast(sum[1] / n + 0.5), + static_cast(sum[2] / n + 0.5), + static_cast(sum[3] / n + 0.5), + static_cast(sum[4] / n + 0.5)}; + for (int y = 0; y < H; ++y) { + uchar* bcRow = bcBits + static_cast(y) * bcBpl; + uchar* roRow = roBits + static_cast(y) * roBpl; + uchar* meRow = meBits + static_cast(y) * meBpl; + for (int x = 0; x < W; ++x) { + if (covered[static_cast(y) * W + x]) + continue; + bcRow[x * 4 + 0] = avg[0]; + bcRow[x * 4 + 1] = avg[1]; + bcRow[x * 4 + 2] = avg[2]; + bcRow[x * 4 + 3] = 255; + roRow[x] = avg[3]; + meRow[x] = avg[4]; + } + } + } + } + + if (opts.progress) + opts.progress(static_cast(jobs.size()), static_cast(jobs.size())); + + r.baseColor = std::move(baseColor); + r.roughness = std::move(rough); + r.metallic = std::move(metal); + if (opts.bakeNormalMap) + r.normalMap = std::move(normal); + r.normals = tNormals; // welded smooth shading normals (see header) + r.ok = true; + return r; +} + +// ---- bakeDetailNormal ---------------------------------------------------------- +NormalBakeResult bakeDetailNormal(const std::vector& targetPositions, + const std::vector& targetIndices, + const std::vector& targetUvs, + int width, int height, + const std::vector& sourcePositions, + const std::vector& sourceIndices, + const BakeOptions& opts) +{ + NormalBakeResult r; + const size_t nv = targetPositions.size() / 3; + if (nv == 0 || targetIndices.size() < 3 || targetIndices.size() % 3 != 0 + || targetUvs.size() != nv * 2) { + r.error = QStringLiteral("detail normal: invalid target mesh/uvs."); + return r; + } + for (uint32_t i : targetIndices) { + if (i >= nv) { + r.error = QStringLiteral("detail normal: target index out of range."); + return r; + } + } + const size_t snv = sourcePositions.size() / 3; + if (snv == 0 || sourceIndices.size() < 3 || sourceIndices.size() % 3 != 0) { + r.error = QStringLiteral("detail normal: invalid source mesh."); + return r; + } + for (uint32_t i : sourceIndices) { + if (i >= snv) { + r.error = QStringLiteral("detail normal: source index out of range."); + return r; + } + } + if (width < 8 || height < 8 || width > 16384 || height > 16384) { + r.error = QStringLiteral("detail normal: bad atlas size."); + return r; + } + const int W = width, H = height; + + // Shading normals: position-welded so chart seams stay smooth; tangents + // accumulated per split vertex (UV seams SHOULD split the tangent basis). + const std::vector tNormals = + smoothNormalsWelded(targetPositions, targetIndices); + const std::vector sNormals = + smoothNormalsWelded(sourcePositions, sourceIndices); + std::vector tan1(targetPositions.size(), 0.0f); + std::vector tan2(targetPositions.size(), 0.0f); + for (size_t t = 0; t + 2 < targetIndices.size(); t += 3) { + const uint32_t i0 = targetIndices[t], i1 = targetIndices[t + 1], + i2 = targetIndices[t + 2]; + const float* p0 = &targetPositions[i0 * 3]; + const float* p1 = &targetPositions[i1 * 3]; + const float* p2 = &targetPositions[i2 * 3]; + const float* u0 = &targetUvs[i0 * 2]; + const float* u1 = &targetUvs[i1 * 2]; + const float* u2 = &targetUvs[i2 * 2]; + float e1[3], e2[3]; + sub3(p1, p0, e1); + sub3(p2, p0, e2); + const float du1 = u1[0] - u0[0], dv1 = u1[1] - u0[1]; + const float du2 = u2[0] - u0[0], dv2 = u2[1] - u0[1]; + const float det = du1 * dv2 - du2 * dv1; + if (std::fabs(det) < 1e-20f) + continue; + const float rd = 1.0f / det; + const float sdir[3] = {(e1[0] * dv2 - e2[0] * dv1) * rd, + (e1[1] * dv2 - e2[1] * dv1) * rd, + (e1[2] * dv2 - e2[2] * dv1) * rd}; + const float tdir[3] = {(e2[0] * du1 - e1[0] * du2) * rd, + (e2[1] * du1 - e1[1] * du2) * rd, + (e2[2] * du1 - e1[2] * du2) * rd}; + for (uint32_t i : {i0, i1, i2}) { + for (int k = 0; k < 3; ++k) { + tan1[i * 3 + k] += sdir[k]; + tan2[i * 3 + k] += tdir[k]; + } + } + } + std::vector tTangent(nv * 4); + for (size_t v = 0; v < nv; ++v) { + const float* n = &tNormals[v * 3]; + const float* t1 = &tan1[v * 3]; + float t[3] = {t1[0] - n[0] * dot3(n, t1), t1[1] - n[1] * dot3(n, t1), + t1[2] - n[2] * dot3(n, t1)}; + if (len3(t) < 1e-12f) { + const float up[3] = {0.0f, std::fabs(n[1]) < 0.9f ? 1.0f : 0.0f, + std::fabs(n[1]) < 0.9f ? 0.0f : 1.0f}; + cross3(up, n, t); + } + normalize3(t); + float bc[3]; + cross3(n, t, bc); + tTangent[v * 4 + 0] = t[0]; + tTangent[v * 4 + 1] = t[1]; + tTangent[v * 4 + 2] = t[2]; + tTangent[v * 4 + 3] = dot3(bc, &tan2[v * 3]) < 0.0f ? -1.0f : 1.0f; + } + + TriangleGrid grid; + grid.build(sourcePositions, sourceIndices); + + QImage normal(W, H, QImage::Format_RGB888); + normal.fill(QColor(128, 128, 255)); + std::vector covered(static_cast(W) * H, 0); + const int progressTotal = W * H; + int processed = 0, sinceProgress = 0; + + for (size_t t = 0; t + 2 < targetIndices.size(); t += 3) { + const uint32_t i0 = targetIndices[t], i1 = targetIndices[t + 1], + i2 = targetIndices[t + 2]; + // Atlas-pixel UVs (uvs are normalized [0,1]). + const float uv0[2] = {targetUvs[i0 * 2] * W, targetUvs[i0 * 2 + 1] * H}; + const float uv1[2] = {targetUvs[i1 * 2] * W, targetUvs[i1 * 2 + 1] * H}; + const float uv2[2] = {targetUvs[i2 * 2] * W, targetUvs[i2 * 2 + 1] * H}; + const int minX = std::max(0, static_cast(std::floor( + std::min({uv0[0], uv1[0], uv2[0]})))); + const int maxX = std::min(W - 1, static_cast(std::ceil( + std::max({uv0[0], uv1[0], uv2[0]})))); + const int minY = std::max(0, static_cast(std::floor( + std::min({uv0[1], uv1[1], uv2[1]})))); + const int maxY = std::min(H - 1, static_cast(std::ceil( + std::max({uv0[1], uv1[1], uv2[1]})))); + const float denom = (uv1[1] - uv2[1]) * (uv0[0] - uv2[0]) + + (uv2[0] - uv1[0]) * (uv0[1] - uv2[1]); + if (std::fabs(denom) < 1e-12f) + continue; + const float inv = 1.0f / denom; + for (int y = minY; y <= maxY; ++y) { + for (int x = minX; x <= maxX; ++x) { + const size_t lin = static_cast(y) * W + x; + if (covered[lin]) + continue; + const float px = x + 0.5f, py = y + 0.5f; + const float w0 = ((uv1[1] - uv2[1]) * (px - uv2[0]) + + (uv2[0] - uv1[0]) * (py - uv2[1])) * inv; + const float w1 = ((uv2[1] - uv0[1]) * (px - uv2[0]) + + (uv0[0] - uv2[0]) * (py - uv2[1])) * inv; + const float w2 = 1.0f - w0 - w1; + const float eps = -0.001f; + if (w0 < eps || w1 < eps || w2 < eps) + continue; + covered[lin] = 1; + + float P[3], Nt[3], T[3]; + for (int k = 0; k < 3; ++k) { + P[k] = w0 * targetPositions[i0 * 3 + k] + + w1 * targetPositions[i1 * 3 + k] + + w2 * targetPositions[i2 * 3 + k]; + Nt[k] = w0 * tNormals[i0 * 3 + k] + + w1 * tNormals[i1 * 3 + k] + + w2 * tNormals[i2 * 3 + k]; + T[k] = w0 * tTangent[i0 * 4 + k] + + w1 * tTangent[i1 * 4 + k] + + w2 * tTangent[i2 * 4 + k]; + } + normalize3(Nt); + const float wsign = (w0 * tTangent[i0 * 4 + 3] + + w1 * tTangent[i1 * 4 + 3] + + w2 * tTangent[i2 * 4 + 3]) < 0.0f + ? -1.0f : 1.0f; + const float ndt = dot3(Nt, T); + for (int k = 0; k < 3; ++k) + T[k] -= Nt[k] * ndt; + if (len3(T) < 1e-12f) { + const float up[3] = {0, 1, 0}; + cross3(up, Nt, T); + } + normalize3(T); + float B[3]; + cross3(Nt, T, B); + for (int k = 0; k < 3; ++k) + B[k] *= wsign; + + float S[3], sb[3]; + const int sTri = grid.closest(P, S, sb); + float Ns[3]; + if (sTri >= 0) { + const uint32_t* sidx = &sourceIndices[sTri * 3]; + for (int k = 0; k < 3; ++k) + Ns[k] = sb[0] * sNormals[sidx[0] * 3 + k] + + sb[1] * sNormals[sidx[1] * 3 + k] + + sb[2] * sNormals[sidx[2] * 3 + k]; + normalize3(Ns); + } else { + std::memcpy(Ns, Nt, sizeof(Ns)); + } + float nts[3] = {dot3(Ns, T), dot3(Ns, B), dot3(Ns, Nt)}; + normalize3(nts); + uchar* np = normal.scanLine(y) + static_cast(x) * 3; + np[0] = toByte(nts[0] * 0.5f + 0.5f); + np[1] = toByte(nts[1] * 0.5f + 0.5f); + np[2] = toByte(nts[2] * 0.5f + 0.5f); + + ++processed; + if (opts.progress && ++sinceProgress >= 8192) { + sinceProgress = 0; + if (!opts.progress(std::min(processed, progressTotal - 1), + progressTotal)) { + r = NormalBakeResult{}; + r.cancelled = true; + r.error = QStringLiteral("cancelled"); + return r; + } + } + } + } + } + + size_t coveredCount = 0; + for (uint8_t c : covered) + coveredCount += c; + if (coveredCount == 0) { + r.error = QStringLiteral("detail normal: no texels covered."); + return r; + } + + // Border dilation (same policy as bake()). + for (int pass = 0; pass < opts.dilatePx; ++pass) { + std::vector next = covered; + for (int y = 0; y < H; ++y) { + for (int x = 0; x < W; ++x) { + const size_t lin = static_cast(y) * W + x; + if (covered[lin]) + continue; + for (int dy = -1; dy <= 1 && !next[lin]; ++dy) { + for (int dx = -1; dx <= 1; ++dx) { + const int sx = x + dx, sy = y + dy; + if (sx < 0 || sy < 0 || sx >= W || sy >= H) + continue; + if (!covered[static_cast(sy) * W + sx]) + continue; + std::memcpy(normal.scanLine(y) + static_cast(x) * 3, + normal.scanLine(sy) + static_cast(sx) * 3, + 3); + next[lin] = 1; + break; + } + } + } + } + covered.swap(next); + } + + if (opts.progress) + opts.progress(progressTotal, progressTotal); + r.normalMap = std::move(normal); + r.ok = true; + return r; +} + +} // namespace Trellis2Bake diff --git a/src/ImageTo3D/Trellis2Bake.h b/src/ImageTo3D/Trellis2Bake.h new file mode 100644 index 000000000..60c40700a --- /dev/null +++ b/src/ImageTo3D/Trellis2Bake.h @@ -0,0 +1,198 @@ +#ifndef TRELLIS2_BAKE_H +#define TRELLIS2_BAKE_H + +#include +#include + +#include +#include +#include +#include + +// TRELLIS.2 game-ready processing + multi-channel PBR texture baking — the +// QtMeshEditor-native replacement for the functionality the upstream +// reference implementation delegates to NVIDIA nvdiffrast/nvdiffrec (both +// under the NVIDIA Source Code License, research/evaluation only — excluded +// from this project; see docs/trellis2-dependencies.md). +// +// This is ORIGINAL QtMeshEditor code built from standard, publicly documented +// graphics algorithms and the project's existing permissive dependencies: +// * welding / simplification — meshoptimizer (MIT, already used) +// * tiny-component removal — MeshSegmenter::connectedComponents +// * UV unwrap — xatlas (MIT, already used) +// * UV-space triangle rasterization — scanline barycentric coverage (the +// MeshGenBaker #764 approach, extended to multi-channel) +// * source-surface correspondence — closest point on triangle (Ericson, +// "Real-Time Collision Detection") over a sparse uniform grid +// * attribute lookup — trilinear interpolation over the +// sparse voxel attribute volume TRELLIS.2 generates +// * normal map — source smooth normal expressed in +// the target's Lengyel tangent frame (OpenGL +Y-up convention, matching +// NormalMapGenerator's default) +// No NVIDIA source was read, copied, translated or linked for any of it. +// +// Pure data (Qt-only, no Ogre/GL/ONNX) — unit-tested headlessly in +// Trellis2Bake_test.cpp. +namespace Trellis2Bake { + +// ---- Sparse PBR attribute volume ------------------------------------------ +// TRELLIS.2 emits per-occupied-voxel attributes: base_color.rgb, metallic, +// roughness, alpha (6 × u8). Sampling is trilinear over the occupied +// neighbours (weights renormalized over present voxels — the volume only +// exists in a shell around the surface), with a nearest-occupied fallback. +class SparseVolumeSampler { +public: + // coords: Lx3 integer voxel coordinates; attrs: Lx6 (order above). + // Voxel centre of (i,j,k) sits at origin + (ijk + 0.5) * voxelSize. + void build(const uint32_t* coords, const uint8_t* attrs, int count, + float voxelSize, const float origin[3]); + bool valid() const { return m_count > 0; } + // Sample at a world-space (TRELLIS-space) point. out[6] in 0..1. + // Returns false (and writes neutral defaults) when nothing occupied is + // anywhere near the point. + bool sample(const float p[3], float out[6]) const; + +private: + std::unordered_map m_map; + const uint8_t* m_attrs = nullptr; + int m_count = 0; + float m_voxelSize = 1.0f; + float m_origin[3] = {-0.5f, -0.5f, -0.5f}; +}; + +// ---- Closest point on triangle (exposed for unit tests) -------------------- +// Standard Ericson closest-point-on-triangle. Writes the closest point and +// its barycentric coordinates (w.r.t. a,b,c). +void closestPointOnTriangle(const float a[3], const float b[3], + const float c[3], const float p[3], + float outClosest[3], float outBary[3]); + +// ---- Game-ready processing (Phase 8) --------------------------------------- +struct GameReadyOptions { + // Near-duplicate weld tolerance as a fraction of the bbox diagonal + // (0 disables the epsilon and welds bit-identical positions only). + float weldEpsilonFraction = 1e-5f; + // Absolute weld tolerance — overrides the fraction when > 0. The TRELLIS + // dual-grid callers set voxelSize/8 (the upstream reference weld): raw + // dual-grid output is non-manifold seam soup at sub-voxel scale, and + // without a voxel-scale weld the topology-preserving simplifier can get + // stuck millions of triangles above the requested budget. + float weldEpsilonAbsolute = 0.0f; + // Disconnected components smaller than BOTH thresholds are dropped + // (floating debris from the sparse-voxel extraction). The largest + // component is always kept. + int minComponentTriangles = 16; + float minComponentFraction = 0.002f; // × total triangle count + // 0 = keep original density. Otherwise meshopt_simplify toward this + // triangle count (no exact-count promise — border locking can stop + // earlier; the achieved count is in the result). + int targetTriangles = 0; + float simplifyTargetError = 0.01f; // relative to bbox extent + // Taubin λ|μ smoothing passes applied to the welded mesh before + // simplification (0 = off). The TRELLIS dual-grid callers enable this: + // fuzzy subjects (fur/hair) decode with sub-voxel micro-pits that render + // as dark speckle and derail QEM on thin double-walled features. + int taubinIterations = 0; +}; + +struct GameReadyResult { + bool ok = false; + QString error; + std::vector positions; // compacted + std::vector indices; + int inputTriangles = 0; + int outputTriangles = 0; + int weldedVertices = 0; // vertices merged by the weld + int removedComponents = 0; // debris islands dropped + int removedTriangles = 0; // triangles dropped with them (+degenerates) + float simplifyError = 0.0f; // meshopt result_error (0 when not simplified) +}; + +GameReadyResult makeGameReady(const std::vector& positions, + const std::vector& indices, + const GameReadyOptions& opts); + +// Unify triangle winding (exposed for unit tests; makeGameReady runs it). +// Raw TRELLIS dual-grid output arrives with large patches of flipped +// triangles (~17% conflicting directed edges measured on a real generation) +// — under backface culling the flipped patches simply vanish, and they +// corrupt area-weighted smooth normals. BFS over the 2-manifold edge graph +// flips faces to a consistent orientation; each connected component is then +// oriented OUTWARD by its signed volume. Non-manifold edges (3+ faces) are +// not used for propagation. Returns the number of faces flipped. +int unifyWinding(const std::vector& positions, + std::vector& indices); + +// ---- Multi-channel bake (Phase 7) ------------------------------------------ +struct BakeOptions { + int textureSize = 2048; // clamped to [64, 8192] + int dilatePx = 4; // chart-border dilation passes + int supersample = 1; // 1 or 2 (2 = 2×2 subsamples per texel) + bool bakeNormalMap = true; // bake source detail normals (for simplified targets) + // Laplacian smoothing iterations applied to the SOURCE normal field + // before it feeds the detail-normal bake. Raw dual-grid surfaces carry + // voxel-scale normal noise that otherwise bakes into a glittery normal + // map (white specular speckle); ~8 iterations flattens the noise while + // keeping shape-scale relief. 0 disables. + int sourceNormalSmoothIterations = 8; + // done/total covered texels; return false to cancel. + std::function progress; +}; + +struct BakeResult { + bool ok = false; + bool cancelled = false; + QString error; + // Target mesh re-indexed by xatlas (chart seams split), with UV0. + std::vector positions; + std::vector indices; + std::vector uvs; // [0,1], V not flipped (MeshGenBaker convention) + // Position-welded smooth shading normals for the re-indexed target (the + // same ones the bake's tangent frames used). Exporting these instead of + // recomputing from the seam-split mesh keeps chart seams smooth. + std::vector normals; // Nx3 + int vertexCount = 0; + int triangleCount = 0; + QImage baseColor; // RGBA8888 (alpha channel from the volume) + QImage roughness; // Grayscale8 + QImage metallic; // Grayscale8 + QImage normalMap; // RGB888 tangent-space, OpenGL +Y up (null if disabled) +}; + +// Bake the SOURCE surface's TRELLIS.2 attributes onto the (possibly +// simplified) TARGET mesh: unwrap target with xatlas, rasterize each chart in +// UV space, project every covered texel to the closest point on the source +// surface, and trilinearly sample the sparse attribute volume there. Standard +// offline texture baking — no differentiable rendering involved. +BakeResult bake(const std::vector& targetPositions, + const std::vector& targetIndices, + const std::vector& sourcePositions, + const std::vector& sourceIndices, + const SparseVolumeSampler& volume, + const BakeOptions& opts); + +// ---- Detail-normal bake onto EXISTING UVs (TripoSR/TripoSG game-ready) ----- +// The high-poly → low-poly workflow for the local ONNX backends: the target +// was already unwrapped + diffuse-baked (MeshGenBaker), so this bakes ONLY a +// tangent-space detail normal map into that SAME atlas — sampling smooth +// normals from the dense pre-simplification source mesh. Seam-split vertices +// are position-welded for the target's shading normals so chart seams don't +// read as hard edges. Same rasterizer/dilation as bake(). +struct NormalBakeResult { + bool ok = false; + bool cancelled = false; + QString error; + QImage normalMap; // RGB888 tangent-space, OpenGL +Y up (W×H as given) +}; + +NormalBakeResult bakeDetailNormal(const std::vector& targetPositions, + const std::vector& targetIndices, + const std::vector& targetUvs, // Nx2 [0,1] + int width, int height, + const std::vector& sourcePositions, + const std::vector& sourceIndices, + const BakeOptions& opts); + +} // namespace Trellis2Bake + +#endif // TRELLIS2_BAKE_H diff --git a/src/ImageTo3D/Trellis2Bake_test.cpp b/src/ImageTo3D/Trellis2Bake_test.cpp new file mode 100644 index 000000000..54bd22ea8 --- /dev/null +++ b/src/ImageTo3D/Trellis2Bake_test.cpp @@ -0,0 +1,385 @@ +// Unit tests for Trellis2Bake — the QtMeshEditor-native game-ready processing +// + multi-channel PBR baker that replaces the upstream nvdiffrast/nvdiffrec +// path (docs/trellis2-dependencies.md). Pure data — no Ogre/GL/ONNX/Python. +#include "Trellis2Bake.h" + +#include + +#include +#include + +namespace { + +// Append a unit-ish cube (12 tris) centred at (cx,cy,cz) with half-size h. +void appendCube(std::vector& pos, std::vector& idx, + float cx, float cy, float cz, float h) +{ + const uint32_t base = static_cast(pos.size() / 3); + const float v[8][3] = { + {cx - h, cy - h, cz - h}, {cx + h, cy - h, cz - h}, + {cx + h, cy + h, cz - h}, {cx - h, cy + h, cz - h}, + {cx - h, cy - h, cz + h}, {cx + h, cy - h, cz + h}, + {cx + h, cy + h, cz + h}, {cx - h, cy + h, cz + h}}; + for (const auto& p : v) { pos.push_back(p[0]); pos.push_back(p[1]); pos.push_back(p[2]); } + const uint32_t f[12][3] = { + {0, 2, 1}, {0, 3, 2}, {4, 5, 6}, {4, 6, 7}, + {0, 1, 5}, {0, 5, 4}, {2, 3, 7}, {2, 7, 6}, + {1, 2, 6}, {1, 6, 5}, {3, 0, 4}, {3, 4, 7}}; + for (const auto& t : f) { + idx.push_back(base + t[0]); + idx.push_back(base + t[1]); + idx.push_back(base + t[2]); + } +} + +// A dense volume block filled with a constant attribute row. +struct ConstVolume { + std::vector coords; + std::vector attrs; + Trellis2Bake::SparseVolumeSampler sampler; + void build(int res, const uint8_t row[6], float voxelSize, + const float origin[3]) + { + coords.clear(); + attrs.clear(); + for (int x = 0; x < res; ++x) + for (int y = 0; y < res; ++y) + for (int z = 0; z < res; ++z) { + coords.push_back(x); coords.push_back(y); coords.push_back(z); + for (int c = 0; c < 6; ++c) attrs.push_back(row[c]); + } + sampler.build(coords.data(), attrs.data(), + static_cast(coords.size() / 3), voxelSize, origin); + } +}; + +} // namespace + +// ---- closestPointOnTriangle -------------------------------------------------- + +TEST(Trellis2BakeTest, ClosestPointOnTriangleRegions) +{ + const float a[3] = {0, 0, 0}, b[3] = {1, 0, 0}, c[3] = {0, 1, 0}; + float cp[3], bc[3]; + + // Interior projection. + const float pIn[3] = {0.25f, 0.25f, 1.0f}; + Trellis2Bake::closestPointOnTriangle(a, b, c, pIn, cp, bc); + EXPECT_NEAR(cp[0], 0.25f, 1e-5f); + EXPECT_NEAR(cp[1], 0.25f, 1e-5f); + EXPECT_NEAR(cp[2], 0.0f, 1e-5f); + EXPECT_NEAR(bc[0] + bc[1] + bc[2], 1.0f, 1e-5f); + + // Vertex region. + const float pV[3] = {-1.0f, -1.0f, 0.0f}; + Trellis2Bake::closestPointOnTriangle(a, b, c, pV, cp, bc); + EXPECT_NEAR(bc[0], 1.0f, 1e-5f); + + // Edge region (edge ab). + const float pE[3] = {0.5f, -2.0f, 0.0f}; + Trellis2Bake::closestPointOnTriangle(a, b, c, pE, cp, bc); + EXPECT_NEAR(cp[0], 0.5f, 1e-5f); + EXPECT_NEAR(cp[1], 0.0f, 1e-5f); + EXPECT_NEAR(bc[2], 0.0f, 1e-5f); +} + +// ---- SparseVolumeSampler ------------------------------------------------------- + +TEST(Trellis2BakeTest, VolumeSamplerTrilinearAndFallback) +{ + // Two voxels along +x: red at (0,0,0), green at (1,0,0). + const uint32_t coords[6] = {0, 0, 0, 1, 0, 0}; + const uint8_t attrs[12] = {255, 0, 0, 0, 255, 255, + 0, 255, 0, 255, 0, 255}; + const float origin[3] = {0, 0, 0}; + Trellis2Bake::SparseVolumeSampler s; + s.build(coords, attrs, 2, 1.0f, origin); + + float out[6]; + // At the centre of voxel 0 → exactly red. + const float p0[3] = {0.5f, 0.5f, 0.5f}; + EXPECT_TRUE(s.sample(p0, out)); + EXPECT_NEAR(out[0], 1.0f, 1e-3f); + EXPECT_NEAR(out[1], 0.0f, 1e-3f); + // Halfway between the two centres → 50/50 blend. + const float pMid[3] = {1.0f, 0.5f, 0.5f}; + EXPECT_TRUE(s.sample(pMid, out)); + EXPECT_NEAR(out[0], 0.5f, 2e-2f); + EXPECT_NEAR(out[1], 0.5f, 2e-2f); + // A couple of voxels away → nearest-occupied fallback still answers. + const float pFar[3] = {3.2f, 0.5f, 0.5f}; + EXPECT_TRUE(s.sample(pFar, out)); + EXPECT_NEAR(out[1], 1.0f, 1e-3f); // nearest is the green voxel + // Nowhere near anything → neutral defaults, false. + const float pNo[3] = {50.0f, 50.0f, 50.0f}; + EXPECT_FALSE(s.sample(pNo, out)); + EXPECT_NEAR(out[4], 0.8f, 1e-4f); // default roughness +} + +// ---- makeGameReady ------------------------------------------------------------- + +TEST(Trellis2BakeTest, GameReadyWeldsAndDropsDebris) +{ + std::vector pos; + std::vector idx; + appendCube(pos, idx, 0, 0, 0, 0.5f); // main body, 12 tris + appendCube(pos, idx, 3.0f, 0, 0, 0.01f); // tiny floating debris + // Duplicate the main cube's vertices by re-appending an identical cube in + // place — every position collides, so welding should fuse them. + appendCube(pos, idx, 0, 0, 0, 0.5f); + + // Defaults: threshold = max(16, 0.002×36) = 16 → the 12-triangle debris + // cube is below it and gets dropped; the 24-tri welded main body stays. + Trellis2Bake::GameReadyOptions opts; + const auto r = Trellis2Bake::makeGameReady(pos, idx, opts); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_GT(r.weldedVertices, 0); + EXPECT_EQ(r.removedComponents, 1); // the debris cube + // The duplicated cube's triangles collapse onto the same welded verts and + // survive as duplicate faces (dedup is not this pass's job), but the + // debris' 12 triangles must be gone. + EXPECT_EQ(r.outputTriangles, 24); + EXPECT_EQ(r.positions.size() % 3, 0u); + for (uint32_t i : r.indices) + EXPECT_LT(i, r.positions.size() / 3); +} + +TEST(Trellis2BakeTest, GameReadySimplifiesTowardTarget) +{ + // A tessellated plane: 32x32 quads = 2048 tris. + std::vector pos; + std::vector idx; + const int n = 33; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + pos.push_back(x / float(n - 1)); + pos.push_back(y / float(n - 1)); + pos.push_back(0.0f); + } + for (int y = 0; y + 1 < n; ++y) + for (int x = 0; x + 1 < n; ++x) { + const uint32_t a = y * n + x, b = a + 1, c = a + n, d = c + 1; + idx.insert(idx.end(), {a, b, c, b, d, c}); + } + + Trellis2Bake::GameReadyOptions opts; + opts.targetTriangles = 64; + opts.simplifyTargetError = 0.5f; // flat plane — everything collapsible + const auto r = Trellis2Bake::makeGameReady(pos, idx, opts); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_LT(r.outputTriangles, 300); // dramatically reduced + EXPECT_GE(r.outputTriangles, 2); + EXPECT_EQ(r.inputTriangles, 2048); +} + +TEST(Trellis2BakeTest, GameReadyRejectsGarbage) +{ + EXPECT_FALSE(Trellis2Bake::makeGameReady({}, {}, {}).ok); + const std::vector pos = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + const std::vector bad = {0, 1, 9}; + EXPECT_FALSE(Trellis2Bake::makeGameReady(pos, bad, {}).ok); +} + +// ---- bake ------------------------------------------------------------------------ + +TEST(Trellis2BakeTest, BakeTransfersVolumeAttributesAndFlatNormal) +{ + // Source = target = one cube; constant gold-ish metallic volume covering it. + std::vector pos; + std::vector idx; + appendCube(pos, idx, 0.5f, 0.5f, 0.5f, 0.5f); // cube spanning [0,1]^3 + + const uint8_t row[6] = {255, 204, 51, 230, 64, 255}; // rgb, metal, rough, alpha + const float origin[3] = {-0.5f, -0.5f, -0.5f}; + ConstVolume vol; + vol.build(8, row, 0.25f, origin); // 8^3 voxels of size 0.25 → covers [-0.5,1.5] + + Trellis2Bake::BakeOptions opts; + opts.textureSize = 128; + opts.dilatePx = 2; + const auto r = Trellis2Bake::bake(pos, idx, pos, idx, vol.sampler, opts); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_GT(r.vertexCount, 0); + EXPECT_EQ(r.uvs.size(), static_cast(r.vertexCount) * 2); + ASSERT_FALSE(r.baseColor.isNull()); + ASSERT_FALSE(r.roughness.isNull()); + ASSERT_FALSE(r.metallic.isNull()); + ASSERT_FALSE(r.normalMap.isNull()); + + // Sample the centre texel of some covered chart: hunt for a pixel whose + // basecolor matches the constant volume row (most of the atlas should). + int matches = 0, covered = 0; + for (int y = 0; y < r.baseColor.height(); y += 4) { + for (int x = 0; x < r.baseColor.width(); x += 4) { + const QColor c = r.baseColor.pixelColor(x, y); + if (c.alpha() == 0) + continue; + ++covered; + if (std::abs(c.red() - 255) <= 2 && std::abs(c.green() - 204) <= 2 + && std::abs(c.blue() - 51) <= 2) + ++matches; + } + } + ASSERT_GT(covered, 0); + EXPECT_GT(matches, covered / 2); + + // Metallic/roughness lanes carry the constant values on covered texels. + bool sawMetal = false; + for (int y = 0; y < r.metallic.height() && !sawMetal; ++y) + for (int x = 0; x < r.metallic.width(); ++x) + if (std::abs(int(r.metallic.scanLine(y)[x]) - 230) <= 2) { + sawMetal = true; + break; + } + EXPECT_TRUE(sawMetal); + + // Source == target ⇒ the baked detail normal is the flat (128,128,255) + // tangent-space "up" on face interiors. Count near-flat texels. + int flat = 0, normCovered = 0; + for (int y = 0; y < r.normalMap.height(); y += 4) { + for (int x = 0; x < r.normalMap.width(); x += 4) { + const QColor a = r.baseColor.pixelColor(x, y); + if (a.alpha() == 0) + continue; + const uchar* p = r.normalMap.scanLine(y) + size_t(x) * 3; + ++normCovered; + if (std::abs(int(p[0]) - 128) <= 6 && std::abs(int(p[1]) - 128) <= 6 + && p[2] >= 240) + ++flat; + } + } + ASSERT_GT(normCovered, 0); + EXPECT_GT(flat, normCovered * 3 / 4); +} + +TEST(Trellis2BakeTest, BakeHonoursCancellation) +{ + std::vector pos; + std::vector idx; + appendCube(pos, idx, 0.5f, 0.5f, 0.5f, 0.5f); + const uint8_t row[6] = {200, 200, 200, 0, 128, 255}; + const float origin[3] = {-0.5f, -0.5f, -0.5f}; + ConstVolume vol; + vol.build(4, row, 0.5f, origin); + + Trellis2Bake::BakeOptions opts; + opts.textureSize = 512; + opts.progress = [](int, int) { return false; }; // cancel immediately + const auto r = Trellis2Bake::bake(pos, idx, pos, idx, vol.sampler, opts); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(r.cancelled); +} + +TEST(Trellis2BakeTest, BakeRejectsBadInput) +{ + const std::vector pos = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + const std::vector idx = {0, 1, 2}; + Trellis2Bake::SparseVolumeSampler empty; + EXPECT_FALSE(Trellis2Bake::bake({}, {}, pos, idx, empty, {}).ok); + const std::vector oob = {0, 1, 7}; + EXPECT_FALSE(Trellis2Bake::bake(pos, oob, pos, idx, empty, {}).ok); + EXPECT_FALSE(Trellis2Bake::bake(pos, idx, pos, oob, empty, {}).ok); +} + +TEST(Trellis2BakeTest, DetailNormalIdentityIsFlat) +{ + // A unit quad, target == source, planar UVs: the detail map must be flat + // (128,128,255) on every covered texel. + const std::vector pos = {0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0}; + const std::vector idx = {0, 1, 2, 0, 2, 3}; + const std::vector uvs = {0, 0, 1, 0, 1, 1, 0, 1}; + + const auto r = Trellis2Bake::bakeDetailNormal(pos, idx, uvs, 64, 64, + pos, idx, {}); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + ASSERT_FALSE(r.normalMap.isNull()); + int flat = 0, total = 0; + for (int y = 4; y < 60; y += 4) { + for (int x = 4; x < 60; x += 4) { + const uchar* p = r.normalMap.constScanLine(y) + size_t(x) * 3; + ++total; + if (std::abs(int(p[0]) - 128) <= 2 && std::abs(int(p[1]) - 128) <= 2 + && p[2] >= 250) + ++flat; + } + } + EXPECT_EQ(flat, total); +} + +TEST(Trellis2BakeTest, DetailNormalEncodesSourceRelief) +{ + // Target = flat quad; source = the same quad "tented" along its middle + // (centre row of vertices raised). The baked texels must tilt away from + // flat where the source normal disagrees with the target normal. + const std::vector tpos = {0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0}; + const std::vector tidx = {0, 1, 2, 0, 2, 3}; + const std::vector tuvs = {0, 0, 1, 0, 1, 1, 0, 1}; + // Source: 2x1 strip with a ridge at x=0.5 raised by z=0.15. + const std::vector spos = { + 0, 0, 0, 0.5f, 0, 0.15f, 1, 0, 0, + 0, 1, 0, 0.5f, 1, 0.15f, 1, 1, 0}; + const std::vector sidx = {0, 1, 4, 0, 4, 3, 1, 2, 5, 1, 5, 4}; + + const auto r = Trellis2Bake::bakeDetailNormal(tpos, tidx, tuvs, 64, 64, + spos, sidx, {}); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + int tilted = 0; + for (int y = 4; y < 60; y += 4) + for (int x = 4; x < 60; x += 4) { + const uchar* p = r.normalMap.constScanLine(y) + size_t(x) * 3; + if (std::abs(int(p[0]) - 128) > 8) // red channel = tangent-x tilt + ++tilted; + } + EXPECT_GT(tilted, 20); +} + +TEST(Trellis2BakeTest, DetailNormalRejectsBadInput) +{ + const std::vector pos = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + const std::vector idx = {0, 1, 2}; + const std::vector uvs = {0, 0, 1, 0, 0, 1}; + EXPECT_FALSE(Trellis2Bake::bakeDetailNormal({}, {}, {}, 64, 64, pos, idx, {}).ok); + EXPECT_FALSE(Trellis2Bake::bakeDetailNormal(pos, idx, {0, 0}, 64, 64, + pos, idx, {}).ok); // uv size + EXPECT_FALSE(Trellis2Bake::bakeDetailNormal(pos, idx, uvs, 2, 2, + pos, idx, {}).ok); // atlas size + const std::vector oob = {0, 1, 9}; + EXPECT_FALSE(Trellis2Bake::bakeDetailNormal(pos, oob, uvs, 64, 64, + pos, idx, {}).ok); + EXPECT_FALSE(Trellis2Bake::bakeDetailNormal(pos, idx, uvs, 64, 64, + pos, oob, {}).ok); +} + +TEST(Trellis2BakeTest, UnifyWindingFixesFlippedPatchesAndOrientsOutward) +{ + // A cube with half its faces deliberately flipped must come out fully + // consistent AND outward (positive signed volume). + std::vector pos; + std::vector idx; + appendCube(pos, idx, 0, 0, 0, 0.5f); + for (size_t t = 0; t < idx.size(); t += 6) // flip every other tri + std::swap(idx[t + 1], idx[t + 2]); + + const int flips = Trellis2Bake::unifyWinding(pos, idx); + EXPECT_GT(flips, 0); + + // Consistency: every interior directed edge appears exactly once. + std::map, int> dir; + for (size_t t = 0; t + 2 < idx.size(); t += 3) + for (int k = 0; k < 3; ++k) + ++dir[{idx[t + k], idx[t + (k + 1) % 3]}]; + for (const auto& e : dir) + EXPECT_EQ(e.second, 1); + + // Outward: positive signed volume. + double vol = 0.0; + for (size_t t = 0; t + 2 < idx.size(); t += 3) { + const float* a = &pos[idx[t] * 3]; + const float* b = &pos[idx[t + 1] * 3]; + const float* c = &pos[idx[t + 2] * 3]; + vol += a[0] * (b[1] * c[2] - b[2] * c[1]) + + a[1] * (b[2] * c[0] - b[0] * c[2]) + + a[2] * (b[0] * c[1] - b[1] * c[0]); + } + EXPECT_GT(vol, 0.0); +} diff --git a/src/ImageTo3D/Trellis2Guard_test.cpp b/src/ImageTo3D/Trellis2Guard_test.cpp new file mode 100644 index 000000000..34a92dcbc --- /dev/null +++ b/src/ImageTo3D/Trellis2Guard_test.cpp @@ -0,0 +1,152 @@ +// Phase 13 enforcement: the TRELLIS.2 integration must never (re)introduce the +// prohibited NVIDIA research-only libraries. nvdiffrast and nvdiffrec are +// under the NVIDIA Source Code License ("research or evaluation purposes +// only") and are excluded from QtMeshEditor's TRELLIS.2 backend end-to-end — +// see docs/trellis2-dependencies.md. This test scans the sidecar sources and +// dependency manifests from the repo (QTMESH_UT_SOURCE_ROOT) and fails if any +// non-allowlisted reference appears. scripts/check-trellis2-restricted-deps.sh +// is the same gate for CI contexts without the test binary. +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +QString sourceRoot() +{ +#ifdef QTMESH_UT_SOURCE_ROOT + return QString::fromUtf8(QTMESH_UT_SOURCE_ROOT); +#else + return {}; +#endif +} + +// Lines that may legitimately NAME the prohibited packages: prohibition +// comments/docs, the startup guard, and the dependency report. Everything is +// judged line-by-line so an actual `import nvdiffrast` can never hide behind +// an allowlisted keyword elsewhere in the file. +bool lineIsAllowlisted(const QString& line) +{ + const QString t = line.trimmed(); + // Python/requirements comments and Markdown prose are documentation. + if (t.startsWith(QLatin1Char('#')) || t.startsWith(QLatin1Char('*')) + || t.startsWith(QLatin1Char('-')) || t.startsWith(QLatin1Char('|')) + || t.startsWith(QLatin1Char('>'))) + return true; + // The explicit guard/report constructs in generate.py. + if (t.contains(QLatin1String("PROHIBITED_MODULES")) + || t.contains(QLatin1String("NOT INSTALLED"))) + return true; + // Docstring prose (module docstrings mention the exclusion by name). + if (t.startsWith(QLatin1Char('"')) || t.startsWith(QLatin1String("'''")) + || t.contains(QLatin1String("are PROHIBITED")) + || t.contains(QLatin1String("prohibited"))) + return true; + return false; +} + +} // namespace + +TEST(Trellis2GuardTest, SidecarNeverImportsRestrictedNvidiaLibraries) +{ + const QString root = sourceRoot(); + ASSERT_FALSE(root.isEmpty()); + const QDir dir(QDir(root).filePath(QStringLiteral("ai/trellis2"))); + ASSERT_TRUE(dir.exists()) << "ai/trellis2 sidecar directory missing"; + + const QStringList banned = {QStringLiteral("nvdiffrast"), + QStringLiteral("nvdiffrec")}; + QStringList violations; + QDirIterator it(dir.absolutePath(), + {QStringLiteral("*.py"), QStringLiteral("*.txt"), + QStringLiteral("*.toml"), QStringLiteral("*.cfg")}, + QDir::Files, QDirIterator::Subdirectories); + int scanned = 0; + while (it.hasNext()) { + const QString path = it.next(); + if (path.contains(QLatin1String("/env/")) + || path.contains(QLatin1String("/TRELLIS.2/"))) + continue; // only OUR sidecar files, not a user-installed runtime + QFile f(path); + ASSERT_TRUE(f.open(QIODevice::ReadOnly)); + ++scanned; + int lineNo = 0; + while (!f.atEnd()) { + const QString line = QString::fromUtf8(f.readLine()); + ++lineNo; + for (const QString& b : banned) { + if (!line.contains(b, Qt::CaseInsensitive)) + continue; + // A real import/require is a violation no matter what. + const bool isImport = + line.contains(QLatin1String("import nvdiffrast")) + || line.contains(QLatin1String("import nvdiffrec")) + || line.contains(QLatin1String("from nvdiffrast")) + || line.contains(QLatin1String("from nvdiffrec")); + if (isImport || !lineIsAllowlisted(line)) + violations << QStringLiteral("%1:%2: %3") + .arg(path).arg(lineNo) + .arg(line.trimmed()); + } + } + } + EXPECT_GT(scanned, 2); // requirements.txt + generate.py + qtm3d.py at least + EXPECT_TRUE(violations.isEmpty()) + << "prohibited nvdiffrast/nvdiffrec reference(s):\n" + << violations.join(QLatin1Char('\n')).toStdString(); +} + +TEST(Trellis2GuardTest, RequirementsNeverListRestrictedOrTrapPackages) +{ + const QString root = sourceRoot(); + ASSERT_FALSE(root.isEmpty()); + QFile req(QDir(root).filePath(QStringLiteral("ai/trellis2/requirements.txt"))); + ASSERT_TRUE(req.open(QIODevice::ReadOnly)); + while (!req.atEnd()) { + const QString line = QString::fromUtf8(req.readLine()).trimmed(); + if (line.isEmpty() || line.startsWith(QLatin1Char('#'))) + continue; + const QString pkg = line.split(QRegularExpression( + QStringLiteral("[=<>!\\[; ]"))).value(0).toLower(); + EXPECT_NE(pkg, QStringLiteral("nvdiffrast")) << line.toStdString(); + EXPECT_NE(pkg, QStringLiteral("nvdiffrec")) << line.toStdString(); + // The PyPI package named `cumesh` is an unrelated, unlicensed project — + // CuMesh must be built from the pinned JeffreyXiang/CuMesh checkout. + EXPECT_NE(pkg, QStringLiteral("cumesh")) << line.toStdString(); + } +} + +TEST(Trellis2GuardTest, CppIntegrationNeverReferencesRestrictedLibraries) +{ + const QString root = sourceRoot(); + ASSERT_FALSE(root.isEmpty()); + // The C++ replacement layer must be genuinely independent: outside of + // exclusion comments, the sources may not reference the NVIDIA libraries + // at all (no bindings, no dlopen, no subprocess invocations). + const QStringList files = { + QStringLiteral("src/ImageTo3D/Trellis2Predictor.cpp"), + QStringLiteral("src/ImageTo3D/Trellis2Bake.cpp"), + QStringLiteral("src/ImageTo3D/Trellis2Interchange.cpp"), + }; + for (const QString& rel : files) { + QFile f(QDir(root).filePath(rel)); + ASSERT_TRUE(f.open(QIODevice::ReadOnly)) << rel.toStdString(); + int lineNo = 0; + while (!f.atEnd()) { + const QString line = QString::fromUtf8(f.readLine()); + ++lineNo; + if (!line.contains(QLatin1String("nvdiff"), Qt::CaseInsensitive)) + continue; + const QString t = line.trimmed(); + EXPECT_TRUE(t.startsWith(QLatin1String("//")) + || t.startsWith(QLatin1String("*"))) + << rel.toStdString() << ":" << lineNo << ": " + << t.toStdString(); + } + } +} diff --git a/src/ImageTo3D/Trellis2Interchange.cpp b/src/ImageTo3D/Trellis2Interchange.cpp new file mode 100644 index 000000000..2b41bd572 --- /dev/null +++ b/src/ImageTo3D/Trellis2Interchange.cpp @@ -0,0 +1,429 @@ +#include "Trellis2Interchange.h" + +#include +#include +#include + +#include +#include + +namespace Trellis2Interchange { + +namespace { + +constexpr char kMagic[8] = {'Q', 'T', 'M', 'E', 'S', 'H', '3', 'D'}; +constexpr uint32_t kVersion = 1; + +inline qint64 align16(qint64 n) { return (n + 15) & ~qint64(15); } + +struct ArrayRef { + QString dtype; + QList shape; + qint64 offset = 0; + qint64 byteLength = 0; + bool valid = false; +}; + +ArrayRef arrayRef(const QJsonObject& arrays, const QString& name) +{ + ArrayRef ref; + if (!arrays.contains(name)) + return ref; + const QJsonObject o = arrays.value(name).toObject(); + ref.dtype = o.value(QStringLiteral("dtype")).toString(); + for (const QJsonValue& v : o.value(QStringLiteral("shape")).toArray()) + ref.shape.append(static_cast(v.toDouble(-1))); + ref.offset = static_cast(o.value(QStringLiteral("offset")).toDouble(-1)); + ref.byteLength = + static_cast(o.value(QStringLiteral("byteLength")).toDouble(-1)); + ref.valid = !ref.dtype.isEmpty() && !ref.shape.isEmpty() + && ref.offset >= 0 && ref.byteLength >= 0; + return ref; +} + +qint64 elementSize(const QString& dtype) +{ + if (dtype == QLatin1String("f32") || dtype == QLatin1String("u32") + || dtype == QLatin1String("i32")) + return 4; + if (dtype == QLatin1String("f16") || dtype == QLatin1String("u16")) + return 2; + if (dtype == QLatin1String("u8")) + return 1; + return 0; +} + +qint64 elementCount(const ArrayRef& ref) +{ + qint64 n = 1; + for (qint64 d : ref.shape) { + if (d < 0) + return -1; + n *= d; + } + return n; +} + +// Bounds-check an array against the blob section, verify byteLength matches +// dtype*shape, and return a pointer into `blob`. +const uint8_t* checkedBlob(const QByteArray& blob, qint64 blobBase, + const ArrayRef& ref, QString* error) +{ + const qint64 esize = elementSize(ref.dtype); + if (esize == 0) { + *error = QStringLiteral("unsupported dtype '%1'").arg(ref.dtype); + return nullptr; + } + const qint64 count = elementCount(ref); + if (count < 0 || count * esize != ref.byteLength) { + *error = QStringLiteral("array size mismatch (dtype %1)").arg(ref.dtype); + return nullptr; + } + const qint64 start = blobBase + ref.offset; + if (start < 0 || start + ref.byteLength > blob.size()) { + *error = QStringLiteral("array exceeds file bounds"); + return nullptr; + } + return reinterpret_cast(blob.constData()) + start; +} + +} // namespace + +ReadResult read(const QString& path) +{ + ReadResult r; + QFile f(path); + if (!f.open(QIODevice::ReadOnly)) { + r.error = QStringLiteral("cannot open %1").arg(path); + return r; + } + const QByteArray blob = f.readAll(); + if (blob.size() < 16 || std::memcmp(blob.constData(), kMagic, 8) != 0) { + r.error = QStringLiteral("not a QTM3D file"); + return r; + } + uint32_t version = 0, jsonLen = 0; + std::memcpy(&version, blob.constData() + 8, 4); + std::memcpy(&jsonLen, blob.constData() + 12, 4); + if (version != kVersion) { + r.error = QStringLiteral("unsupported QTM3D version %1").arg(version); + return r; + } + if (qint64(16) + jsonLen > blob.size()) { + r.error = QStringLiteral("truncated manifest"); + return r; + } + QJsonParseError perr{}; + const QJsonDocument doc = QJsonDocument::fromJson( + QByteArray(blob.constData() + 16, static_cast(jsonLen)), &perr); + if (perr.error != QJsonParseError::NoError || !doc.isObject()) { + r.error = QStringLiteral("bad manifest JSON: %1").arg(perr.errorString()); + return r; + } + const QJsonObject manifest = doc.object(); + const QJsonObject arrays = manifest.value(QStringLiteral("arrays")).toObject(); + const qint64 blobBase = align16(16 + jsonLen); + + Data& d = r.data; + d.meta = manifest.value(QStringLiteral("meta")).toObject(); + d.resolution = d.meta.value(QStringLiteral("resolution")).toInt(0); + d.voxelSize = + static_cast(d.meta.value(QStringLiteral("voxelSize")).toDouble(0.0)); + const QJsonArray originArr = d.meta.value(QStringLiteral("origin")).toArray(); + for (int i = 0; i < 3 && i < originArr.size(); ++i) + d.origin[i] = static_cast(originArr.at(i).toDouble(d.origin[i])); + + QString err; + + // ---- positions (required, f32 [N,3]) ----------------------------------- + { + const ArrayRef ref = arrayRef(arrays, QStringLiteral("positions")); + if (!ref.valid || ref.shape.size() != 2 || ref.shape[1] != 3 + || ref.dtype != QLatin1String("f32")) { + r.error = QStringLiteral("missing/invalid 'positions' array"); + return r; + } + const uint8_t* p = checkedBlob(blob, blobBase, ref, &err); + if (!p) { r.error = QStringLiteral("positions: %1").arg(err); return r; } + d.vertexCount = static_cast(ref.shape[0]); + d.positions.resize(static_cast(d.vertexCount) * 3); + std::memcpy(d.positions.data(), p, static_cast(ref.byteLength)); + } + + // ---- indices (required, u32 [M,3]) -------------------------------------- + { + const ArrayRef ref = arrayRef(arrays, QStringLiteral("indices")); + if (!ref.valid || ref.shape.size() != 2 || ref.shape[1] != 3 + || ref.dtype != QLatin1String("u32")) { + r.error = QStringLiteral("missing/invalid 'indices' array"); + return r; + } + const uint8_t* p = checkedBlob(blob, blobBase, ref, &err); + if (!p) { r.error = QStringLiteral("indices: %1").arg(err); return r; } + d.triangleCount = static_cast(ref.shape[0]); + d.indices.resize(static_cast(d.triangleCount) * 3); + std::memcpy(d.indices.data(), p, static_cast(ref.byteLength)); + for (uint32_t idx : d.indices) { + if (idx >= static_cast(d.vertexCount)) { + r.error = QStringLiteral("index out of range (%1 >= %2)") + .arg(idx).arg(d.vertexCount); + return r; + } + } + } + + // ---- voxel_coords (optional, u16/u32 [L,3]) ------------------------------ + { + const ArrayRef ref = arrayRef(arrays, QStringLiteral("voxel_coords")); + if (ref.valid) { + if (ref.shape.size() != 2 || ref.shape[1] != 3 + || (ref.dtype != QLatin1String("u16") + && ref.dtype != QLatin1String("u32"))) { + r.error = QStringLiteral("invalid 'voxel_coords' array"); + return r; + } + const uint8_t* p = checkedBlob(blob, blobBase, ref, &err); + if (!p) { r.error = QStringLiteral("voxel_coords: %1").arg(err); return r; } + d.voxelCount = static_cast(ref.shape[0]); + d.voxelCoords.resize(static_cast(d.voxelCount) * 3); + if (ref.dtype == QLatin1String("u32")) { + std::memcpy(d.voxelCoords.data(), p, + static_cast(ref.byteLength)); + } else { + const uint16_t* s = reinterpret_cast(p); + for (size_t i = 0; i < d.voxelCoords.size(); ++i) + d.voxelCoords[i] = s[i]; + } + } + } + + // ---- voxel_attrs (required alongside voxel_coords, u8 [L,6]) ------------- + { + const ArrayRef ref = arrayRef(arrays, QStringLiteral("voxel_attrs")); + if (ref.valid) { + if (ref.shape.size() != 2 || ref.shape[1] != 6 + || ref.dtype != QLatin1String("u8") + || ref.shape[0] != d.voxelCount) { + r.error = QStringLiteral("invalid 'voxel_attrs' array"); + return r; + } + const uint8_t* p = checkedBlob(blob, blobBase, ref, &err); + if (!p) { r.error = QStringLiteral("voxel_attrs: %1").arg(err); return r; } + d.voxelAttrs.assign(p, p + ref.byteLength); + } else if (d.voxelCount > 0) { + r.error = QStringLiteral("'voxel_coords' present without 'voxel_attrs'"); + return r; + } + } + + // ---- vertex_colors (optional, u8 [N,4]) ---------------------------------- + { + const ArrayRef ref = arrayRef(arrays, QStringLiteral("vertex_colors")); + if (ref.valid) { + if (ref.shape.size() != 2 || ref.shape[1] != 4 + || ref.dtype != QLatin1String("u8") + || ref.shape[0] != d.vertexCount) { + r.error = QStringLiteral("invalid 'vertex_colors' array"); + return r; + } + const uint8_t* p = checkedBlob(blob, blobBase, ref, &err); + if (!p) { r.error = QStringLiteral("vertex_colors: %1").arg(err); return r; } + d.vertexColors.assign(p, p + ref.byteLength); + } + } + + if (d.vertexCount <= 0 || d.triangleCount <= 0) { + r.error = QStringLiteral("empty mesh"); + return r; + } + r.ok = true; + return r; +} + +bool write(const QString& path, const Data& data, QString* error) +{ + auto failWith = [error](const QString& msg) { + if (error) *error = msg; + return false; + }; + if (data.positions.size() != static_cast(data.vertexCount) * 3 + || data.indices.size() != static_cast(data.triangleCount) * 3) + return failWith(QStringLiteral("inconsistent counts")); + + struct Entry { + const char* name; + const char* dtype; + qint64 rows; + qint64 cols; + const void* ptr; + qint64 bytes; + }; + std::vector entries; + entries.push_back({"positions", "f32", data.vertexCount, 3, + data.positions.data(), + qint64(data.positions.size() * sizeof(float))}); + entries.push_back({"indices", "u32", data.triangleCount, 3, + data.indices.data(), + qint64(data.indices.size() * sizeof(uint32_t))}); + if (data.voxelCount > 0) { + entries.push_back({"voxel_coords", "u32", data.voxelCount, 3, + data.voxelCoords.data(), + qint64(data.voxelCoords.size() * sizeof(uint32_t))}); + entries.push_back({"voxel_attrs", "u8", data.voxelCount, 6, + data.voxelAttrs.data(), + qint64(data.voxelAttrs.size())}); + } + if (!data.vertexColors.empty()) + entries.push_back({"vertex_colors", "u8", data.vertexCount, 4, + data.vertexColors.data(), + qint64(data.vertexColors.size())}); + + QJsonObject arrays; + qint64 offset = 0; + for (const Entry& e : entries) { + QJsonObject o; + o.insert(QStringLiteral("dtype"), QLatin1String(e.dtype)); + o.insert(QStringLiteral("shape"), + QJsonArray{static_cast(e.rows), + static_cast(e.cols)}); + o.insert(QStringLiteral("offset"), static_cast(offset)); + o.insert(QStringLiteral("byteLength"), static_cast(e.bytes)); + arrays.insert(QLatin1String(e.name), o); + offset = align16(offset + e.bytes); + } + + QJsonObject meta = data.meta; + meta.insert(QStringLiteral("resolution"), data.resolution); + meta.insert(QStringLiteral("voxelSize"), static_cast(data.voxelSize)); + meta.insert(QStringLiteral("origin"), + QJsonArray{static_cast(data.origin[0]), + static_cast(data.origin[1]), + static_cast(data.origin[2])}); + + QJsonObject manifest; + manifest.insert(QStringLiteral("generator"), QStringLiteral("qtmesh-trellis2")); + manifest.insert(QStringLiteral("formatVersion"), + static_cast(kVersion)); + manifest.insert(QStringLiteral("meta"), meta); + manifest.insert(QStringLiteral("arrays"), arrays); + const QByteArray json = + QJsonDocument(manifest).toJson(QJsonDocument::Compact); + + QFile f(path); + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) + return failWith(QStringLiteral("cannot write %1").arg(path)); + f.write(kMagic, 8); + const uint32_t version = kVersion; + const uint32_t jsonLen = static_cast(json.size()); + f.write(reinterpret_cast(&version), 4); + f.write(reinterpret_cast(&jsonLen), 4); + f.write(json); + const qint64 headerLen = 16 + json.size(); + const qint64 blobBase = align16(headerLen); + static const char zeros[16] = {}; + f.write(zeros, blobBase - headerLen); + qint64 pos = 0; + for (const Entry& e : entries) { + const qint64 wanted = + static_cast(arrays.value(QLatin1String(e.name)) + .toObject() + .value(QStringLiteral("offset")) + .toDouble()); + if (wanted > pos) { + f.write(zeros, wanted - pos); + pos = wanted; + } + f.write(reinterpret_cast(e.ptr), e.bytes); + pos += e.bytes; + } + return true; +} + +ReadResult readTrellisCppDump(const QString& path) +{ + ReadResult r; + QFile f(path); + if (!f.open(QIODevice::ReadOnly)) { + r.error = QStringLiteral("cannot open %1").arg(path); + return r; + } + const QByteArray blob = f.readAll(); + if (blob.size() < 16) { + r.error = QStringLiteral("truncated trellis.cpp dump"); + return r; + } + int32_t V = 0, F = 0, Mv = 0, res = 0; + std::memcpy(&V, blob.constData() + 0, 4); + std::memcpy(&F, blob.constData() + 4, 4); + std::memcpy(&Mv, blob.constData() + 8, 4); + std::memcpy(&res, blob.constData() + 12, 4); + if (V <= 0 || F <= 0 || Mv < 0 || res <= 0 || res > 4096 + || V > 100000000 || F > 200000000 || Mv > 200000000) { + r.error = QStringLiteral("implausible trellis.cpp dump header " + "(V=%1 F=%2 Mv=%3 res=%4)") + .arg(V).arg(F).arg(Mv).arg(res); + return r; + } + const qint64 expected = 16 + + qint64(V) * 3 * 4 + qint64(F) * 3 * 4 + + qint64(Mv) * 3 * 4 + qint64(Mv) * 6 * 4; + if (blob.size() < expected) { + r.error = QStringLiteral("trellis.cpp dump shorter than header claims " + "(%1 < %2 bytes)").arg(blob.size()).arg(expected); + return r; + } + + Data& d = r.data; + const char* p = blob.constData() + 16; + d.vertexCount = V; + d.triangleCount = F; + d.voxelCount = Mv; + d.resolution = res; + d.voxelSize = 1.0f / static_cast(res); + d.origin[0] = d.origin[1] = d.origin[2] = -0.5f; + d.positions.resize(static_cast(V) * 3); + std::memcpy(d.positions.data(), p, d.positions.size() * 4); + p += static_cast(V) * 12; + d.indices.resize(static_cast(F) * 3); + // faces are i32 in the dump; reinterpret via copy with range check. + { + const int32_t* fp = reinterpret_cast(p); + for (size_t i = 0; i < d.indices.size(); ++i) { + const int32_t idx = fp[i]; + if (idx < 0 || idx >= V) { + r = ReadResult{}; + r.error = QStringLiteral("dump face index out of range (%1)").arg(idx); + return r; + } + d.indices[i] = static_cast(idx); + } + p += static_cast(F) * 12; + } + if (Mv > 0) { + d.voxelCoords.resize(static_cast(Mv) * 3); + const int32_t* cp = reinterpret_cast(p); + for (size_t i = 0; i < d.voxelCoords.size(); ++i) { + const int32_t c = cp[i]; + if (c < 0 || c >= res) { + r = ReadResult{}; + r.error = QStringLiteral("dump voxel coord out of range (%1)").arg(c); + return r; + } + d.voxelCoords[i] = static_cast(c); + } + p += static_cast(Mv) * 12; + d.voxelAttrs.resize(static_cast(Mv) * 6); + const float* ap = reinterpret_cast(p); + for (size_t i = 0; i < d.voxelAttrs.size(); ++i) { + const float v = ap[i]; + const float cl = v < 0.0f ? 0.0f : (v > 1.0f ? 1.0f : v); + d.voxelAttrs[i] = static_cast(cl * 255.0f + 0.5f); + } + } + d.meta.insert(QStringLiteral("generator"), QStringLiteral("trellis.cpp")); + d.meta.insert(QStringLiteral("resolution"), res); + r.ok = true; + return r; +} + +} // namespace Trellis2Interchange diff --git a/src/ImageTo3D/Trellis2Interchange.h b/src/ImageTo3D/Trellis2Interchange.h new file mode 100644 index 000000000..cb39006ca --- /dev/null +++ b/src/ImageTo3D/Trellis2Interchange.h @@ -0,0 +1,77 @@ +#ifndef TRELLIS2_INTERCHANGE_H +#define TRELLIS2_INTERCHANGE_H + +#include +#include + +#include +#include + +// QTM3D interchange container (epic: TRELLIS.2 backend). The Python sidecar +// (ai/trellis2/generate.py + qtm3d.py) writes the RAW TRELLIS.2 generation — +// vertices, faces and the sparse PBR attribute volume — into this trivial +// little-endian binary format, and QtMeshEditor's own C++ pipeline +// (Trellis2Bake) takes over from there. Deliberately not NPZ/GLB: no zip +// dependency, no detour through upstream export code (whose texture bake +// depends on the license-prohibited nvdiffrast — see +// docs/trellis2-dependencies.md). +// +// Layout: "QTMESH3D" magic, u32 version(=1), u32 jsonLen, UTF-8 JSON +// manifest, zero-pad to a 16-byte boundary, raw array blobs (each 16-byte +// aligned; offsets in the manifest are relative to the blob-section start). +// Manifest: { generator, formatVersion, meta{...}, +// arrays: { name: {dtype, shape, offset, byteLength} } }. +// +// Pure data (Qt-only, no Ogre/GL) so it unit-tests headlessly +// (Trellis2Interchange_test.cpp), mirroring the MeshGenBaker convention. +namespace Trellis2Interchange { + +// One decoded generation. Attribute channel order in `voxelAttrs` follows the +// TRELLIS.2 pbr_attr_layout: base_color.rgb, metallic, roughness, alpha — +// 6 bytes per occupied voxel, 0..255. +struct Data { + std::vector positions; // Nx3, TRELLIS space (aabb ~[-0.5,0.5]^3) + std::vector indices; // Mx3 + std::vector voxelCoords; // Lx3 integer voxel coordinates + std::vector voxelAttrs; // Lx6 (see channel order above) + std::vector vertexColors; // Nx4 rgba, optional (empty if absent) + + int vertexCount = 0; + int triangleCount = 0; + int voxelCount = 0; + int resolution = 0; // voxel grid resolution (meta) + float voxelSize = 0.0f; // meta; 1/resolution for TRELLIS.2 + float origin[3] = {-0.5f, -0.5f, -0.5f}; + + QJsonObject meta; // full "meta" object (seed, preset, timings…) +}; + +struct ReadResult { + bool ok = false; + QString error; + Data data; +}; + +// Parse + validate a .qtm3d file. Never throws; every failure lands in +// `error`. Validation: magic/version, manifest shape, blob bounds, index +// range, coords within the declared resolution, array-size consistency. +ReadResult read(const QString& path); + +// Write `data` back out (used by unit tests and tooling; the production +// writer is the Python sidecar). Returns false + `error` on failure. +bool write(const QString& path, const Data& data, QString* error = nullptr); + +// Parse a trellis.cpp `--dump-post` raw dump (the C++/GGML runtime flavor, +// issue #966) into the same Data. Binary little-endian layout: +// i32 V, F, Mv, res; +// f32 verts[V*3]; i32 faces[F*3]; i32 coords[Mv*3]; f32 pbr6[Mv*6] +// pbr6 channels per voxel: base_color.rgb, metallic, roughness, alpha in +// [0,1] (quantized to the u8 voxelAttrs lanes; note the QTM3D channel order +// is basecolor.rgb, metallic, roughness, alpha — identical). Geometry is in +// TRELLIS space (aabb [-0.5,0.5]^3), voxel centre at (ijk+0.5)/res - 0.5 — +// the same convention the Python sidecar emits. +ReadResult readTrellisCppDump(const QString& path); + +} // namespace Trellis2Interchange + +#endif // TRELLIS2_INTERCHANGE_H diff --git a/src/ImageTo3D/Trellis2Interchange_test.cpp b/src/ImageTo3D/Trellis2Interchange_test.cpp new file mode 100644 index 000000000..12ddcc99c --- /dev/null +++ b/src/ImageTo3D/Trellis2Interchange_test.cpp @@ -0,0 +1,173 @@ +// Unit tests for the QTM3D interchange container (TRELLIS.2 backend). +// Pure data — no Ogre/GL/ONNX/Python (MeshGenBaker_test.cpp convention). +#include "Trellis2Interchange.h" + +#include + +#include +#include +#include + +namespace { + +Trellis2Interchange::Data makeSample() +{ + Trellis2Interchange::Data d; + d.positions = {0.0f, 0.0f, 0.0f, + 0.5f, 0.0f, 0.0f, + 0.0f, 0.5f, 0.0f, + 0.0f, 0.0f, 0.5f}; + d.indices = {0, 1, 2, 0, 2, 3}; + d.vertexCount = 4; + d.triangleCount = 2; + d.voxelCoords = {1, 2, 3, 4, 5, 6}; + d.voxelAttrs = {255, 0, 0, 10, 200, 255, + 0, 255, 0, 250, 20, 128}; + d.voxelCount = 2; + d.resolution = 64; + d.voxelSize = 1.0f / 64.0f; + d.origin[0] = d.origin[1] = d.origin[2] = -0.5f; + d.vertexColors = {255, 0, 0, 255, 0, 255, 0, 255, + 0, 0, 255, 255, 128, 128, 128, 255}; + d.meta.insert(QStringLiteral("seed"), 7); + return d; +} + +} // namespace + +TEST(Trellis2InterchangeTest, RoundTripPreservesEverything) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString path = QDir(tmp.path()).filePath("roundtrip.qtm3d"); + + const Trellis2Interchange::Data d = makeSample(); + QString err; + ASSERT_TRUE(Trellis2Interchange::write(path, d, &err)) << err.toStdString(); + + const auto r = Trellis2Interchange::read(path); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_EQ(r.data.vertexCount, 4); + EXPECT_EQ(r.data.triangleCount, 2); + EXPECT_EQ(r.data.voxelCount, 2); + EXPECT_EQ(r.data.positions, d.positions); + EXPECT_EQ(r.data.indices, d.indices); + EXPECT_EQ(r.data.voxelCoords, d.voxelCoords); + EXPECT_EQ(r.data.voxelAttrs, d.voxelAttrs); + EXPECT_EQ(r.data.vertexColors, d.vertexColors); + EXPECT_EQ(r.data.resolution, 64); + EXPECT_FLOAT_EQ(r.data.voxelSize, 1.0f / 64.0f); + EXPECT_FLOAT_EQ(r.data.origin[0], -0.5f); + EXPECT_EQ(r.data.meta.value(QStringLiteral("seed")).toInt(), 7); +} + +TEST(Trellis2InterchangeTest, OptionalArraysCanBeAbsent) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString path = QDir(tmp.path()).filePath("minimal.qtm3d"); + Trellis2Interchange::Data d = makeSample(); + d.voxelCoords.clear(); + d.voxelAttrs.clear(); + d.voxelCount = 0; + d.vertexColors.clear(); + ASSERT_TRUE(Trellis2Interchange::write(path, d)); + const auto r = Trellis2Interchange::read(path); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_EQ(r.data.voxelCount, 0); + EXPECT_TRUE(r.data.vertexColors.empty()); +} + +TEST(Trellis2InterchangeTest, RejectsBadMagicAndTruncation) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString bad = QDir(tmp.path()).filePath("bad.qtm3d"); + { + QFile f(bad); + ASSERT_TRUE(f.open(QIODevice::WriteOnly)); + f.write("NOTQTM3Dxxxxxxxxxxxx", 20); + } + EXPECT_FALSE(Trellis2Interchange::read(bad).ok); + + // Truncate a valid file mid-blob → bounds check must fire. + const QString path = QDir(tmp.path()).filePath("trunc.qtm3d"); + ASSERT_TRUE(Trellis2Interchange::write(path, makeSample())); + QFile f(path); + ASSERT_TRUE(f.open(QIODevice::ReadWrite)); + ASSERT_TRUE(f.resize(f.size() - 24)); + f.close(); + const auto r = Trellis2Interchange::read(path); + EXPECT_FALSE(r.ok); +} + +TEST(Trellis2InterchangeTest, RejectsOutOfRangeIndices) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString path = QDir(tmp.path()).filePath("oob.qtm3d"); + Trellis2Interchange::Data d = makeSample(); + d.indices[1] = 99; // >= vertexCount + ASSERT_TRUE(Trellis2Interchange::write(path, d)); + const auto r = Trellis2Interchange::read(path); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(r.error.contains(QStringLiteral("out of range"))); +} + +TEST(Trellis2InterchangeTest, MissingFileFailsCleanly) +{ + const auto r = Trellis2Interchange::read( + QStringLiteral("/nonexistent/nowhere.qtm3d")); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(r.error.isEmpty()); +} + + +TEST(Trellis2InterchangeTest, ReadsTrellisCppDump) +{ + // Binary layout of trellis-cli --dump-post: i32 V,F,Mv,res; f32 verts; + // i32 faces; i32 coords; f32 pbr6. + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString path = QDir(tmp.path()).filePath("post.trellisraw"); + { + QFile f(path); + ASSERT_TRUE(f.open(QIODevice::WriteOnly)); + const int32_t hdr[4] = {3, 1, 2, 64}; + const float verts[9] = {0, 0, 0, 0.5f, 0, 0, 0, 0.5f, 0}; + const int32_t faces[3] = {0, 1, 2}; + const int32_t coords[6] = {1, 2, 3, 60, 61, 62}; + const float pbr6[12] = {1.0f, 0.8f, 0.2f, 0.9f, 0.25f, 1.0f, + 0.0f, 0.5f, 1.0f, 0.0f, 0.75f, 0.5f}; + f.write(reinterpret_cast(hdr), sizeof(hdr)); + f.write(reinterpret_cast(verts), sizeof(verts)); + f.write(reinterpret_cast(faces), sizeof(faces)); + f.write(reinterpret_cast(coords), sizeof(coords)); + f.write(reinterpret_cast(pbr6), sizeof(pbr6)); + } + const auto r = Trellis2Interchange::readTrellisCppDump(path); + ASSERT_TRUE(r.ok) << r.error.toStdString(); + EXPECT_EQ(r.data.vertexCount, 3); + EXPECT_EQ(r.data.triangleCount, 1); + EXPECT_EQ(r.data.voxelCount, 2); + EXPECT_EQ(r.data.resolution, 64); + EXPECT_FLOAT_EQ(r.data.voxelSize, 1.0f / 64.0f); + EXPECT_FLOAT_EQ(r.data.origin[0], -0.5f); + EXPECT_EQ(r.data.voxelCoords[3], 60u); + EXPECT_EQ(r.data.voxelAttrs[0], 255); // 1.0 -> 255 + EXPECT_EQ(r.data.voxelAttrs[4], 64); // 0.25 -> 64 + EXPECT_EQ(r.data.voxelAttrs[11], 128); // 0.5 -> 128 + + // Face index out of range must be rejected. + { + QFile f(path); + ASSERT_TRUE(f.open(QIODevice::ReadWrite)); + f.seek(16 + 9 * 4); // first face index + const int32_t bad = 7; + f.write(reinterpret_cast(&bad), 4); + } + EXPECT_FALSE(Trellis2Interchange::readTrellisCppDump(path).ok); + // Truncation must be rejected. + EXPECT_FALSE(Trellis2Interchange::readTrellisCppDump( + QStringLiteral("/nonexistent/x.trellisraw")).ok); +} diff --git a/src/ImageTo3D/Trellis2Predictor.cpp b/src/ImageTo3D/Trellis2Predictor.cpp new file mode 100644 index 000000000..ae9405114 --- /dev/null +++ b/src/ImageTo3D/Trellis2Predictor.cpp @@ -0,0 +1,612 @@ +#include "Trellis2Predictor.h" + +#include "BackgroundRemover.h" +#include "Trellis2Bake.h" +#include "Trellis2Interchange.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr const char* kEnvDirVar = "QTMESH_TRELLIS2_ENV"; +constexpr const char* kEnvPythonVar = "QTMESH_TRELLIS2_PYTHON"; +constexpr const char* kDirSettingsKey = "ai/trellis2Env"; +constexpr const char* kPythonSettingsKey = "ai/trellis2Python"; +constexpr const char* kEnvCliVar = "QTMESH_TRELLIS2_CLI"; +constexpr const char* kEnvCliModelsVar = "QTMESH_TRELLIS2_CLI_MODELS"; +constexpr const char* kCliSettingsKey = "ai/trellis2Cli"; +constexpr const char* kCliModelsSettingsKey = "ai/trellis2CliModels"; + +QJsonObject runtimeManifest(const QString& dir) +{ + QFile f(QDir(dir).filePath(QStringLiteral("runtime.json"))); + if (!f.open(QIODevice::ReadOnly)) + return {}; + const QJsonDocument doc = QJsonDocument::fromJson(f.readAll()); + return doc.isObject() ? doc.object() : QJsonObject{}; +} + +MeshGenPredictor::Result failResult(const QString& message) +{ + MeshGenPredictor::Result r; + r.error = message; + return r; +} + +} // namespace + +Trellis2Predictor::Options::Options() = default; + +bool Trellis2Predictor::isAvailable() +{ + return true; // no ONNX requirement; the runtime probe is what gates it +} + +QString Trellis2Predictor::runtimeDir() +{ + QString dir = qEnvironmentVariable(kEnvDirVar); + if (dir.isEmpty()) + dir = QSettings().value(QLatin1String(kDirSettingsKey)).toString(); + if (dir.isEmpty()) { + const QString base = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + dir = QDir(base).filePath(QStringLiteral("trellis2")); + } + return QDir(dir).exists() ? dir : QString(); +} + +QString Trellis2Predictor::resolvePython(const QString& dir) +{ + QString py = qEnvironmentVariable(kEnvPythonVar); + if (py.isEmpty()) + py = QSettings().value(QLatin1String(kPythonSettingsKey)).toString(); + if (!py.isEmpty()) + return QFileInfo::exists(py) ? py : QString(); + if (dir.isEmpty()) + return {}; + const QJsonObject manifest = runtimeManifest(dir); + py = manifest.value(QStringLiteral("python")).toString(); + if (!py.isEmpty() && QFileInfo::exists(py)) + return py; +#ifdef Q_OS_WIN + py = QDir(dir).filePath(QStringLiteral("env/Scripts/python.exe")); +#else + py = QDir(dir).filePath(QStringLiteral("env/bin/python")); +#endif + return QFileInfo::exists(py) ? py : QString(); +} + +QString Trellis2Predictor::pythonPath() +{ + return resolvePython(runtimeDir()); +} + +QString Trellis2Predictor::generateScriptPath() +{ + const QString dir = runtimeDir(); + if (dir.isEmpty()) + return {}; + const QJsonObject manifest = runtimeManifest(dir); + QString gen = manifest.value(QStringLiteral("generate")).toString(); + if (!gen.isEmpty() && QFileInfo::exists(gen)) + return gen; + gen = QDir(dir).filePath(QStringLiteral("generate.py")); + return QFileInfo::exists(gen) ? gen : QString(); +} + +QString Trellis2Predictor::trellisCliPath() +{ + QString cli = qEnvironmentVariable(kEnvCliVar); + if (cli.isEmpty()) + cli = QSettings().value(QLatin1String(kCliSettingsKey)).toString(); + if (!cli.isEmpty()) + return QFileInfo::exists(cli) ? cli : QString(); + return QStandardPaths::findExecutable(QStringLiteral("trellis-cli")); +} + +QString Trellis2Predictor::trellisCliModelsDir() +{ + QString dir = qEnvironmentVariable(kEnvCliModelsVar); + if (dir.isEmpty()) + dir = QSettings().value(QLatin1String(kCliModelsSettingsKey)).toString(); + if (dir.isEmpty()) { + const QString cli = trellisCliPath(); + if (!cli.isEmpty()) + dir = QDir(QFileInfo(cli).absolutePath()) + .filePath(QStringLiteral("models")); + } + return (!dir.isEmpty() && QDir(dir).exists()) ? dir : QString(); +} + +bool Trellis2Predictor::trellisCliAvailable() +{ + const QString cli = trellisCliPath(); + const QString models = trellisCliModelsDir(); + if (cli.isEmpty() || models.isEmpty()) + return false; + // Minimum 512-pipeline set (the cascade models are optional extras). + const QDir d(models); + return d.exists(QStringLiteral("ss_flow.gguf")) + && d.exists(QStringLiteral("shape_flow_512.gguf")) + && d.exists(QStringLiteral("tex_flow_512.gguf")) + && d.exists(QStringLiteral("shape_dec.gguf")) + && d.exists(QStringLiteral("tex_dec.gguf")) + && d.exists(QStringLiteral("ss_dec.gguf")) + && d.exists(QStringLiteral("dinov3.gguf")); +} + +Trellis2Predictor::RuntimeKind Trellis2Predictor::runtimeKind() +{ + // trellis.cpp preferred: no Python, quantized weights, Metal/Vulkan/CUDA. + if (trellisCliAvailable()) + return RuntimeKind::TrellisCpp; + if (!pythonPath().isEmpty() && !generateScriptPath().isEmpty()) + return RuntimeKind::PythonSidecar; + return RuntimeKind::None; +} + +bool Trellis2Predictor::runtimeAvailable() +{ + return runtimeKind() != RuntimeKind::None; +} + +QString Trellis2Predictor::runtimeDescription() +{ + switch (runtimeKind()) { + case RuntimeKind::TrellisCpp: + return QStringLiteral("TRELLIS.2 runtime: trellis.cpp (%1, models %2)") + .arg(trellisCliPath(), trellisCliModelsDir()); + case RuntimeKind::PythonSidecar: + return QStringLiteral("TRELLIS.2 runtime: Python sidecar (%1)") + .arg(runtimeDir()); + case RuntimeKind::None: + break; + } + return QStringLiteral( + "TRELLIS.2 runtime not installed. Either build trellis.cpp and point " + "QTMESH_TRELLIS2_CLI / QSettings ai/trellis2Cli at trellis-cli (with " + "its GGUF models next to it), or run `python3 ai/trellis2/install.py` " + "(Linux + NVIDIA GPU) for the Python sidecar. See docs/TRELLIS2.md."); +} + +MeshGenPredictor::Result Trellis2Predictor::predict( + const QImage& image, + const Options& opts, + const MeshGenPredictor::ProgressFn& progress) +{ + using Stage = MeshGenPredictor::Stage; + if (image.isNull()) + return failResult(QStringLiteral("trellis2: input image is null.")); + RuntimeKind kind = runtimeKind(); + // --mock is a Python-sidecar feature (synthetic generation for GPU-less + // plumbing tests) — route mock runs there even when trellis.cpp is found. + if ((opts.mock || qEnvironmentVariableIsSet("QTMESH_TRELLIS2_MOCK")) + && kind == RuntimeKind::TrellisCpp + && !pythonPath().isEmpty() && !generateScriptPath().isEmpty()) { + kind = RuntimeKind::PythonSidecar; + } + if (kind == RuntimeKind::None) + return failResult(runtimeDescription()); + + auto report = [&progress](Stage s, int done, int total) -> bool { + return !progress || progress(s, done, total); + }; + + // ---- 1. alpha matte (QtMeshEditor-side background removal) -------------- + // TRELLIS.2's preprocess uses a supplied alpha channel directly and never + // touches its rembg model on such input — which is exactly how the + // non-commercial briaai/RMBG-2.0 stays unused (docs/trellis2-dependencies.md). + QImage subject = image; + const bool inputHasMatte = [&image]() { + if (!image.hasAlphaChannel()) + return false; + const QImage a = image.convertToFormat(QImage::Format_RGBA8888); + for (int y = 0; y < a.height(); ++y) { + const uchar* line = a.constScanLine(y); + for (int x = 0; x < a.width(); ++x) + if (line[x * 4 + 3] != 255) + return true; + } + return false; + }(); + QString warning; + bool matteReady = inputHasMatte; + if (!inputHasMatte && opts.removeBackground && !opts.mock) { + if (!report(Stage::Encode, 0, 1)) + return failResult(QStringLiteral("cancelled")); + if (BackgroundRemover::isAvailable()) { + // ensureModelBlocking spins a nested QEventLoop — only safe on the + // main thread. On a worker (the GUI controller's std::thread) the + // model must have been ensured up front; here we just read it. + const bool onMainThread = + QCoreApplication::instance() + && QThread::currentThread() + == QCoreApplication::instance()->thread(); + const QString model = onMainThread + ? BackgroundRemover::ensureModelBlocking() + : (BackgroundRemover::modelPresent() + ? BackgroundRemover::modelPath() : QString()); + BackgroundRemover::Options bg; + bg.keepAlpha = true; + const BackgroundRemover::Result cut = + BackgroundRemover::removeBackground(subject, model, bg); + if (cut.ok) { + subject = cut.image; + matteReady = true; + } else + warning = QStringLiteral( + "background removal unavailable (%1); the whole frame is " + "treated as foreground.").arg(cut.error); + } else { + warning = QStringLiteral( + "built without ENABLE_ONNX — no U²-Net background removal; " + "the whole frame is treated as foreground."); + } + } + + // ---- 2. sidecar process -------------------------------------------------- + QTemporaryDir tmp; + if (!tmp.isValid()) + return failResult(QStringLiteral("trellis2: cannot create temp dir.")); + const QString inputPng = QDir(tmp.path()).filePath(QStringLiteral("input.png")); + const QString outQtm3d = QDir(tmp.path()).filePath(QStringLiteral("out.qtm3d")); + if (!subject.save(inputPng, "PNG")) + return failResult(QStringLiteral("trellis2: cannot write temp input image.")); + + Trellis2Interchange::Data srcData; + QString pythonQtm3dPath; // set by the Python flavor (for the source copy) + + // Phase 9 hook + long-run test seam: QTMESH_TRELLIS2_IMPORT= + // skips inference entirely and re-runs the native pipeline (game-ready + + // bake) on a previously preserved generation — re-bake textures / + // re-target LODs without paying the model again. + const QString importPath = qEnvironmentVariable("QTMESH_TRELLIS2_IMPORT"); + if (!importPath.isEmpty()) { + Trellis2Interchange::ReadResult rr = Trellis2Interchange::read(importPath); + if (!rr.ok) + return failResult(QStringLiteral( + "trellis2: cannot import preserved source %1: %2") + .arg(importPath, rr.error)); + srcData = std::move(rr.data); + pythonQtm3dPath = importPath; // re-preserve by copying, if asked + } else if (kind == RuntimeKind::TrellisCpp) { + // trellis.cpp flavor (#966): run trellis-cli with --dump-post so it emits + // the RAW decoded mesh + sparse PBR volume and exits before its own + // remesh/UV/bake — QtMeshEditor keeps the whole asset pipeline (game-ready + // simplify + native texture bake), exactly like the Python flavor. + const QString cli = trellisCliPath(); + const QString models = trellisCliModelsDir(); + const QString outDump = + QDir(tmp.path()).filePath(QStringLiteral("out.trellisraw")); + int res = 1024; + const QString presetName = opts.preset.toLower(); + if (presetName == QLatin1String("fast")) + res = 512; + else if (presetName == QLatin1String("high")) + res = 1536; + if (res > 512 + && !QDir(models).exists(QStringLiteral("shape_flow_1024.gguf"))) { + if (!warning.isEmpty()) + warning += QStringLiteral(" "); + warning += QStringLiteral( + "trellis.cpp models dir has no 1024-cascade weights — using the " + "512 pipeline."); + res = 512; + } + QStringList args{QStringLiteral("--image"), inputPng, + QStringLiteral("--dump-post"), outDump, + QStringLiteral("--models"), models, + QStringLiteral("--res"), QString::number(res), + QStringLiteral("--seed"), QString::number(opts.seed)}; + QProcess proc; + proc.setProgram(cli); + proc.setArguments(args); + // stderr (backend banner, ggml logs) forwards to ours; stdout carries the + // "[k/7]" stage lines we map onto the shared Stage enum. + proc.setProcessChannelMode(QProcess::ForwardedErrorChannel); + proc.start(); + if (!proc.waitForStarted(15000)) + return failResult(QStringLiteral("trellis2: failed to start %1").arg(cli)); + Stage lastStage = Stage::Encode; + int lastDone = 0, lastTotal = 2; + bool cancelled = false; + QByteArray pending; + auto handleLine = [&](const QByteArray& line) { + if (line.size() >= 5 && line[0] == '[' && line[2] == '/' + && line[4] == ']' && line[1] >= '1' && line[1] <= '9') { + const int k = line[1] - '0'; + if (k <= 2) { lastStage = Stage::Encode; lastDone = k - 1; lastTotal = 2; } + else if (k <= 6) { lastStage = Stage::Denoise; lastDone = k - 3; lastTotal = 4; } + else { lastStage = Stage::Decode; lastDone = 0; lastTotal = 1; } + } + }; + while (proc.state() != QProcess::NotRunning) { + proc.waitForReadyRead(300); + pending += proc.readAllStandardOutput(); + int nl; + while ((nl = pending.indexOf('\n')) >= 0) { + handleLine(pending.left(nl)); + pending.remove(0, nl + 1); + } + if (!report(lastStage, lastDone, lastTotal)) { + cancelled = true; + proc.terminate(); + if (!proc.waitForFinished(5000)) + proc.kill(); + proc.waitForFinished(2000); + break; + } + } + if (cancelled) + return failResult(QStringLiteral("cancelled")); + if (proc.exitStatus() != QProcess::NormalExit || proc.exitCode() != 0) + return failResult(QStringLiteral( + "trellis2: trellis-cli exited with code %1 (see stderr log).") + .arg(proc.exitCode())); + Trellis2Interchange::ReadResult rr = + Trellis2Interchange::readTrellisCppDump(outDump); + if (!rr.ok) + return failResult(QStringLiteral("trellis2: bad trellis.cpp dump: %1") + .arg(rr.error)); + rr.data.meta.insert(QStringLiteral("seed"), + static_cast(opts.seed)); + rr.data.meta.insert(QStringLiteral("preset"), presetName); + srcData = std::move(rr.data); + } else { + const QString python = pythonPath(); + const QString script = generateScriptPath(); + + QStringList args{script, + QStringLiteral("--input"), inputPng, + QStringLiteral("--output"), outQtm3d, + QStringLiteral("--seed"), QString::number(opts.seed), + QStringLiteral("--preset"), opts.preset.toLower()}; + if (opts.steps > 0) + args << QStringLiteral("--steps") << QString::number(opts.steps); + // QTMESH_TRELLIS2_MOCK: environment escape hatch so the CLI/e2e harness can + // exercise the full plumbing (sidecar → interchange → bake → export) on + // machines without a CUDA GPU. + if (opts.mock || qEnvironmentVariableIsSet("QTMESH_TRELLIS2_MOCK")) + args << QStringLiteral("--mock"); + if (!matteReady) + args << QStringLiteral("--allow-opaque"); + + QProcess proc; + proc.setProgram(python); + proc.setArguments(args); + proc.setWorkingDirectory(QFileInfo(script).absolutePath()); + // Child stderr (tqdm, HF download logs) goes straight to our stderr so the + // pipe can never fill up and stall the sidecar; stdout carries the JSON + // protocol only. + proc.setProcessChannelMode(QProcess::ForwardedErrorChannel); + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + env.insert(QStringLiteral("PYTHONUNBUFFERED"), QStringLiteral("1")); + env.insert(QStringLiteral("QTMESH_TRELLIS2_ENV"), runtimeDir()); + proc.setProcessEnvironment(env); + proc.start(); + if (!proc.waitForStarted(15000)) + return failResult(QStringLiteral("trellis2: failed to start %1").arg(python)); + + // Sidecar stage → (Stage, done, total). The generation itself is one long + // coarse step; the native bake below reports fine-grained progress. + Stage lastStage = Stage::Encode; + int lastDone = 0, lastTotal = 1; + QString sidecarError; + QJsonObject doneInfo; + bool cancelled = false; + + auto handleLine = [&](const QByteArray& line) { + const QJsonDocument doc = QJsonDocument::fromJson(line); + if (!doc.isObject()) + return; + const QJsonObject o = doc.object(); + const QString event = o.value(QStringLiteral("event")).toString(); + if (event == QLatin1String("stage")) { + const QString s = o.value(QStringLiteral("stage")).toString(); + if (s == QLatin1String("load_model")) { + lastStage = Stage::Encode; lastDone = 0; lastTotal = 2; + } else if (s == QLatin1String("preprocess")) { + lastStage = Stage::Encode; lastDone = 1; lastTotal = 2; + } else if (s == QLatin1String("generate")) { + lastStage = Stage::Denoise; lastDone = 0; lastTotal = 1; + } else if (s == QLatin1String("extract")) { + lastStage = Stage::Denoise; lastDone = 1; lastTotal = 1; + } else if (s == QLatin1String("attributes")) { + lastStage = Stage::Decode; lastDone = 0; lastTotal = 1; + } else if (s == QLatin1String("write")) { + lastStage = Stage::Decode; lastDone = 1; lastTotal = 1; + } + } else if (event == QLatin1String("progress")) { + lastDone = o.value(QStringLiteral("done")).toInt(lastDone); + lastTotal = o.value(QStringLiteral("total")).toInt(lastTotal); + } else if (event == QLatin1String("error")) { + sidecarError = o.value(QStringLiteral("message")).toString(); + } else if (event == QLatin1String("done")) { + doneInfo = o; + } + }; + + QByteArray pending; + while (proc.state() != QProcess::NotRunning) { + proc.waitForReadyRead(300); + pending += proc.readAllStandardOutput(); + int nl; + while ((nl = pending.indexOf('\n')) >= 0) { + handleLine(pending.left(nl)); + pending.remove(0, nl + 1); + } + // Re-report the latest stage so cancellation stays responsive during + // the long silent stretches of a generation. + if (!report(lastStage, lastDone, lastTotal)) { + cancelled = true; + proc.terminate(); + if (!proc.waitForFinished(5000)) + proc.kill(); + proc.waitForFinished(2000); + break; + } + } + pending += proc.readAllStandardOutput(); + for (const QByteArray& line : pending.split('\n')) + if (!line.trimmed().isEmpty()) + handleLine(line); + + if (cancelled) + return failResult(QStringLiteral("cancelled")); + if (proc.exitStatus() != QProcess::NormalExit || proc.exitCode() != 0) { + if (!sidecarError.isEmpty()) + return failResult(QStringLiteral("trellis2: %1").arg(sidecarError)); + return failResult(QStringLiteral( + "trellis2: sidecar exited with code %1 (see stderr log).") + .arg(proc.exitCode())); + } + + // ---- 3. read the interchange --------------------------------------------- + Trellis2Interchange::ReadResult read = + Trellis2Interchange::read(outQtm3d); + if (!read.ok) + return failResult(QStringLiteral("trellis2: bad interchange: %1") + .arg(read.error)); + srcData = std::move(read.data); + pythonQtm3dPath = outQtm3d; + } + const Trellis2Interchange::Data& src = srcData; + + // Phase 9: preserve the full-resolution generation. + QString keptSourcePath; + if (!opts.sourceKeepDir.isEmpty()) { + QDir().mkpath(opts.sourceKeepDir); + const QString base = opts.sourceKeepBaseName.isEmpty() + ? QStringLiteral("trellis2") : opts.sourceKeepBaseName; + keptSourcePath = QDir(opts.sourceKeepDir) + .filePath(base + QStringLiteral("_source.qtm3d")); + QFile::remove(keptSourcePath); + const bool kept = !pythonQtm3dPath.isEmpty() + ? QFile::copy(pythonQtm3dPath, keptSourcePath) + : Trellis2Interchange::write(keptSourcePath, src); + if (!kept) + keptSourcePath.clear(); + } + + // ---- 4. native game-ready processing (Phase 8) ---------------------------- + if (!report(Stage::Decode, 0, 1)) + return failResult(QStringLiteral("cancelled")); + Trellis2Bake::GameReadyOptions gr; + gr.targetTriangles = opts.targetTriangles; + // "Original" (0) + texture bake: cap the density anyway. xatlas cannot + // realistically unwrap a raw multi-million-triangle dual-grid mesh (its + // chart compute is superlinear — a 4.86M-tri source burned 19 CPU-hours + // without finishing), and upstream trellis.cpp itself always decimates to + // 150k (res 512) / 300k (cascade) before unwrapping. The uncapped raw + // mesh is still preserved in the .qtm3d source for later re-baking. + if (gr.targetTriangles <= 0 && opts.bakeTexture) { + const int cap = src.resolution > 512 ? 300000 : 150000; + if (src.triangleCount > cap) + gr.targetTriangles = cap; + } + // Voxel-scale weld (the upstream reference uses 1/(res*8)): the raw + // dual-grid mesh is non-manifold at sub-voxel scale and unsimplifiable + // without it. + if (src.voxelSize > 0.0f) + gr.weldEpsilonAbsolute = src.voxelSize / 8.0f; + // Pre-smooth: voxel decodes of fuzzy subjects (fur/hair) carry sub-voxel + // micro-pits that render as dark pepper speckle and derail QEM on thin + // double-walled features (ear lace). Volume-preserving, so real shape + // survives; only sub-voxel noise flattens. + gr.taubinIterations = 5; + const Trellis2Bake::GameReadyResult processed = + Trellis2Bake::makeGameReady(src.positions, src.indices, gr); + if (!processed.ok) + return failResult(QStringLiteral("trellis2: %1").arg(processed.error)); + fprintf(stderr, + "[trellis2] game-ready: %d -> %d tris (welded %d, dropped %d comps, " + "simplify err %.4f)\n", + processed.inputTriangles, processed.outputTriangles, + processed.weldedVertices, processed.removedComponents, + processed.simplifyError); + + Trellis2Bake::SparseVolumeSampler volume; + if (src.voxelCount > 0) + volume.build(src.voxelCoords.data(), src.voxelAttrs.data(), + src.voxelCount, src.voxelSize, src.origin); + + MeshGenPredictor::Result r; + r.warning = warning; + r.sourceInterchangePath = keptSourcePath; + r.usedModel = !opts.mock; + r.bakeTripoSROrientation = false; // TRELLIS.2 is +Y-up like TripoSG + + // ---- 5. native multi-channel PBR bake (Phase 7) ---------------------------- + bool baked = false; + if (opts.bakeTexture && volume.valid()) { + Trellis2Bake::BakeOptions bo; + bo.textureSize = opts.textureSize; + bo.supersample = opts.supersample; + bo.bakeNormalMap = opts.bakeNormalMap; + if (progress) { + bo.progress = [&progress](int done, int total) { + return progress(Stage::Bake, done, total); + }; + } + Trellis2Bake::BakeResult bk = Trellis2Bake::bake( + processed.positions, processed.indices, + src.positions, src.indices, volume, bo); + if (bk.cancelled) + return failResult(QStringLiteral("cancelled")); + if (bk.ok) { + r.positions = std::move(bk.positions); + r.indices = std::move(bk.indices); + r.uvs = std::move(bk.uvs); + r.normals = std::move(bk.normals); + r.vertexCount = bk.vertexCount; + r.triangleCount = bk.triangleCount; + r.texture = std::move(bk.baseColor); + r.roughnessMap = std::move(bk.roughness); + r.metallicMap = std::move(bk.metallic); + r.normalMap = std::move(bk.normalMap); + baked = true; + } else { + if (!r.warning.isEmpty()) + r.warning += QStringLiteral(" "); + r.warning += QStringLiteral( + "texture bake failed (%1) — using per-vertex colours.") + .arg(bk.error); + } + } + + if (!baked) { + r.positions = processed.positions; + r.indices = processed.indices; + r.vertexCount = static_cast(r.positions.size() / 3); + r.triangleCount = static_cast(r.indices.size() / 3); + if (volume.valid()) { + r.colors.resize(static_cast(r.vertexCount) * 3); + for (int v = 0; v < r.vertexCount; ++v) { + float attr[6]; + volume.sample(&r.positions[static_cast(v) * 3], attr); + r.colors[static_cast(v) * 3 + 0] = attr[0]; + r.colors[static_cast(v) * 3 + 1] = attr[1]; + r.colors[static_cast(v) * 3 + 2] = attr[2]; + } + } + } + + if (r.vertexCount <= 0 || r.triangleCount <= 0) + return failResult(QStringLiteral("trellis2: generation produced an empty mesh.")); + + report(Stage::Decode, 1, 1); + r.ok = true; + return r; +} diff --git a/src/ImageTo3D/Trellis2Predictor.h b/src/ImageTo3D/Trellis2Predictor.h new file mode 100644 index 000000000..22431860f --- /dev/null +++ b/src/ImageTo3D/Trellis2Predictor.h @@ -0,0 +1,116 @@ +#ifndef TRELLIS2_PREDICTOR_H +#define TRELLIS2_PREDICTOR_H + +#include "MeshGenPredictor.h" // shared Result / Stage / ProgressFn contract + +#include +#include + +// Microsoft TRELLIS.2 image-to-3D backend — the project's highest-quality +// generation tier and the DEFAULT backend whenever its runtime is installed. +// +// TRELLIS.2 ("Native and Compact Structured Latents for 3D Generation", +// arXiv 2512.14692) — **MIT code AND MIT weights** (microsoft/TRELLIS.2 + +// HF microsoft/TRELLIS.2-4B), pinned revisions in ai/trellis2/install.py. +// Unlike the in-process ONNX backends (TripoSR/TripoSG), this one runs as an +// OUT-OF-PROCESS Python sidecar: the 4B-parameter sparse-voxel flow stack +// needs CUDA (Linux + NVIDIA GPU, >=24 GB VRAM recommended) and its custom +// sparse kernels (FlexGEMM/o-voxel/CuMesh, all MIT) have no ONNX lowering. +// +// Division of labour (Phase 5 of the integration plan): +// Python (ai/trellis2/generate.py) — inference only: image → sparse +// structure → shape/tex SLats → raw mesh + sparse PBR attribute volume, +// exported as a QTM3D interchange file (Trellis2Interchange). +// C++ (this file + Trellis2Bake) — everything that makes an asset: alpha +// matte via the project's own U²-Net (BackgroundRemover keepAlpha — the +// upstream default remover briaai/RMBG-2.0 is CC BY-NC and is never +// loaded), weld/cleanup/simplify (game-ready presets), xatlas UV unwrap, +// multi-channel PBR texture bake, Ogre build + export. +// +// **NVIDIA nvdiffrast / nvdiffrec are excluded end-to-end** (NVIDIA Source +// Code License — research/evaluation only): not installed, not imported, not +// invoked; the sidecar refuses/warns if they are present, and CI greps for +// them. Full audit: docs/trellis2-dependencies.md. +// +// Runtime discovery (no crash when absent — every surface reports a clean +// "runtime not installed" message): env QTMESH_TRELLIS2_ENV → QSettings +// ai/trellis2Env → /trellis2 (where ai/trellis2/install.py installs +// by default). The Python interpreter can be overridden with +// QTMESH_TRELLIS2_PYTHON / QSettings ai/trellis2Python. +class Trellis2Predictor { +public: + struct Options { + Options(); + // fast = TRELLIS.2 '512', balanced = '1024_cascade' (upstream + // default), high = '1536_cascade'. + QString preset = QStringLiteral("balanced"); + unsigned seed = 42; + int steps = 0; // sampler steps override (0 = upstream default) + // ---- QtMeshEditor-side asset processing -------------------------------- + // Game-ready simplification target; 0 keeps the raw density (Phase 8: + // ~10k Low / ~25k Medium / ~50k High — no exact-count promise). + int targetTriangles = 0; + bool bakeTexture = true; // false → per-vertex colours only + int textureSize = 2048; // 1024 / 2048 / 4096 + int supersample = 1; // 1 or 2 (2 = 2×2 subsamples per texel) + bool bakeNormalMap = true; // source detail normals onto the simplified target + bool removeBackground = true; // U²-Net alpha matte (skipped if the + // input already carries real alpha) + // Phase 9: persist the raw generation (QTM3D) here so textures/LODs can + // be re-baked later without re-running inference. Empty = don't keep. + QString sourceKeepDir; + QString sourceKeepBaseName; // file stem for the kept interchange + // Test hook — run the sidecar's --mock synthetic generation (no GPU). + bool mock = false; + }; + + // Compiled in unconditionally (no ONNX needed). What actually gates the + // backend is runtimeAvailable(). + static bool isAvailable(); + + // The backend has TWO interchangeable runtime flavors (#966): + // TrellisCpp — the C++/GGML trellis.cpp CLI (CUDA/Vulkan/Metal, no + // Python; invoked with --dump-post so QtMeshEditor keeps + // the game-ready + bake pipeline). Preferred when found. + // PythonSidecar — the upstream-exact Python env (ai/trellis2/). + enum class RuntimeKind { None, TrellisCpp, PythonSidecar }; + static RuntimeKind runtimeKind(); + + // ---- Python sidecar flavor ------------------------------------------------ + // Resolved runtime directory ("" when none found). + static QString runtimeDir(); + // Resolved python interpreter + generate.py ("" when unresolvable). + static QString pythonPath(); + static QString generateScriptPath(); + + // ---- trellis.cpp flavor ----------------------------------------------------- + // Resolved trellis-cli binary: env QTMESH_TRELLIS2_CLI → QSettings + // ai/trellis2Cli → PATH lookup ("" when unresolvable). + static QString trellisCliPath(); + // GGUF model dir for the CLI: env QTMESH_TRELLIS2_CLI_MODELS → QSettings + // ai/trellis2CliModels → /models. + static QString trellisCliModelsDir(); + // Binary + the minimum 512-pipeline GGUFs present. + static bool trellisCliAvailable(); + + // True when either flavor resolves — the cheap probe every surface gates on. + static bool runtimeAvailable(); + // One-line human description of the runtime state (for UI/CLI errors). + static QString runtimeDescription(); + + // Run the full TRELLIS.2 pipeline (sidecar inference + native asset + // processing). Same Result/Stage/ProgressFn contract as + // MeshGenPredictor::predict: Stage::Encode covers model load/preprocess, + // Stage::Denoise the generation, Stage::Decode extract/transfer, + // Stage::Bake the native texture bake. Returning false from `progress` + // cancels (the sidecar process is terminated). Never throws. + static MeshGenPredictor::Result predict( + const QImage& image, + const Options& opts = {}, + const MeshGenPredictor::ProgressFn& progress = {}); + +private: + static QString resolvePython(const QString& dir); +}; + +#endif // TRELLIS2_PREDICTOR_H diff --git a/src/ImageTo3D/Trellis2Predictor_test.cpp b/src/ImageTo3D/Trellis2Predictor_test.cpp new file mode 100644 index 000000000..0d8922902 --- /dev/null +++ b/src/ImageTo3D/Trellis2Predictor_test.cpp @@ -0,0 +1,85 @@ +// Unit tests for the TRELLIS.2 provider's runtime-discovery + failure paths. +// The real generation needs a Linux + NVIDIA CUDA runtime (and the mock path +// needs a Python interpreter), so these tests cover exactly what CI can: +// clean "runtime not installed" behaviour — the project's "no crash when +// unavailable" convention — and option plumbing. +#include "Trellis2Predictor.h" + +#include + +#include + +namespace { + +// Force-resolve to a nonexistent runtime for the duration of a test. +struct NoRuntimeGuard { + NoRuntimeGuard() + { + qputenv("QTMESH_TRELLIS2_ENV", "/nonexistent/qtmesh-trellis2-ut"); + qputenv("QTMESH_TRELLIS2_PYTHON", "/nonexistent/python-ut"); + // Also neutralize the trellis.cpp flavor (#966): an env override that + // points nowhere beats any PATH-installed trellis-cli. + qputenv("QTMESH_TRELLIS2_CLI", "/nonexistent/trellis-cli-ut"); + } + ~NoRuntimeGuard() + { + qunsetenv("QTMESH_TRELLIS2_ENV"); + qunsetenv("QTMESH_TRELLIS2_PYTHON"); + qunsetenv("QTMESH_TRELLIS2_CLI"); + } +}; + +} // namespace + +TEST(Trellis2PredictorTest, AlwaysCompiledIn) +{ + // Unlike the ONNX backends, availability is not a build-flag question — + // the runtime probe is the gate. + EXPECT_TRUE(Trellis2Predictor::isAvailable()); +} + +TEST(Trellis2PredictorTest, MissingRuntimeReportsCleanly) +{ + NoRuntimeGuard guard; + EXPECT_FALSE(Trellis2Predictor::runtimeAvailable()); + EXPECT_TRUE(Trellis2Predictor::pythonPath().isEmpty()); + EXPECT_TRUE(Trellis2Predictor::generateScriptPath().isEmpty()); + const QString desc = Trellis2Predictor::runtimeDescription(); + EXPECT_TRUE(desc.contains(QStringLiteral("install.py"))); +} + +TEST(Trellis2PredictorTest, PredictWithoutRuntimeFailsWithInstallHint) +{ + NoRuntimeGuard guard; + QImage img(8, 8, QImage::Format_RGB888); + img.fill(Qt::red); + const auto r = Trellis2Predictor::predict(img, {}); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(r.error.contains(QStringLiteral("runtime"))); +} + +TEST(Trellis2PredictorTest, NullImageRejected) +{ + const auto r = Trellis2Predictor::predict(QImage(), {}); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(r.error.contains(QStringLiteral("null"))); +} + +TEST(Trellis2PredictorTest, DispatchThroughMeshGenPredictor) +{ + // Backend::Trellis2 must route through the shared dispatch (both ONNX and + // non-ONNX builds) and fail with the runtime hint, never the generic + // "needs ONNX" error. + NoRuntimeGuard guard; + QImage img(8, 8, QImage::Format_RGB888); + img.fill(Qt::blue); + MeshGenPredictor::Options opts; + opts.backend = MeshGenPredictor::Backend::Trellis2; + const auto r = MeshGenPredictor::predict(img, QString(), QString(), opts); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(r.error.contains(QStringLiteral("TRELLIS.2"))); + + // And the default-backend resolver falls back to TripoSR without it. + EXPECT_EQ(MeshGenPredictor::defaultBackend(), + MeshGenPredictor::Backend::TripoSR); +} diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 16ba42e99..c08ffc41c 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -31,6 +31,7 @@ #include "CLIPipeline.h" #include "ImageTo3D/MeshGenPredictor.h" #include "ImageTo3D/TripoSGPredictor.h" +#include "ImageTo3D/Trellis2Predictor.h" #include "ImageTo3D/MeshGenBuilder.h" #include "OgreWidget.h" #include "SpaceCamera.h" @@ -2747,13 +2748,36 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) const bool upscaleTex = args.value("upscale_texture").toBool(); const bool generatePbr = args.contains("generate_pbr") ? args["generate_pbr"].toBool(true) : true; + // Default backend: TRELLIS.2 when its sidecar runtime is installed on + // this machine, else TripoSR. An explicit 'backend' arg always wins. + opts.backend = MeshGenPredictor::defaultBackend(); if (args.contains("backend")) { const QString b = args["backend"].toString().toLower(); if (b == "triposg") opts.backend = MeshGenPredictor::Backend::TripoSG; - else if (b == "triposr" || b.isEmpty()) + else if (b == "trellis2" || b == "trellis.2" || b == "trellis") + opts.backend = MeshGenPredictor::Backend::Trellis2; + else if (b == "triposr") opts.backend = MeshGenPredictor::Backend::TripoSR; - else return makeErrorResult("'backend' must be 'triposr' or 'triposg'."); - } + else if (!b.isEmpty()) + return makeErrorResult("'backend' must be 'trellis2', 'triposr' or 'triposg'."); + } + if (args.contains("seed")) { + const int s = args["seed"].toInt(42); + if (s < 0) return makeErrorResult("'seed' must be >= 0."); + opts.seed = static_cast(s); + } + if (args.contains("preset")) { + const QString p = args["preset"].toString().toLower(); + if (p != "fast" && p != "balanced" && p != "high") + return makeErrorResult("'preset' must be 'fast', 'balanced' or 'high'."); + opts.trellis2Preset = p; + } + if (args.contains("target_tris")) { + opts.targetTriangles = args["target_tris"].toInt(0); + if (opts.targetTriangles < 0 || opts.targetTriangles > 10000000) + return makeErrorResult("'target_tris' must be in [0, 10000000] (0 = original)."); + } + opts.bakeNormalMap = generatePbr && opts.bakeTexture; if (args.contains("flow_steps")) { opts.flowSteps = args["flow_steps"].toInt(25); if (opts.flowSteps < 1 || opts.flowSteps > 200) @@ -2765,13 +2789,28 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) return makeErrorResult("'guidance' must be between 0 and 30."); } + const QString backendName = + opts.backend == MeshGenPredictor::Backend::Trellis2 ? QStringLiteral("trellis2") + : opts.backend == MeshGenPredictor::Backend::TripoSG ? QStringLiteral("triposg") + : QStringLiteral("triposr"); SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), QStringLiteral("generate_mesh_from_image %1 res=%2 backend=%3") .arg(QFileInfo(imagePath).fileName()).arg(opts.sdfResolution) - .arg(opts.backend == MeshGenPredictor::Backend::TripoSG - ? QStringLiteral("triposg") : QStringLiteral("triposr"))); - - if (opts.backend == MeshGenPredictor::Backend::TripoSG) { + .arg(backendName)); + + if (opts.backend == MeshGenPredictor::Backend::Trellis2) { + if (!Trellis2Predictor::runtimeAvailable()) + return makeErrorResult(Trellis2Predictor::runtimeDescription()); + opts.removeBackground = true; // trellis2 needs an alpha matte; the + // predictor skips it when the input + // already carries one + const QString outPath = args.value("output").toString(); + if (!outPath.isEmpty()) { + // Phase 9: keep the raw full-res generation next to the export. + opts.trellis2SourceKeepDir = QFileInfo(outPath).absolutePath(); + opts.trellis2SourceKeepBaseName = QFileInfo(outPath).completeBaseName(); + } + } else if (opts.backend == MeshGenPredictor::Backend::TripoSG) { // TripoSG always runs the fp32 DiT (int8 tier dropped — degraded // geometry, no ARM speed win); 'quality' still selects the TripoSR // tier used for the colour bake. @@ -2849,6 +2888,10 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) result["vertexCount"] = res.vertexCount; result["triangleCount"] = res.triangleCount; if (!meshPath.isEmpty()) result["meshPath"] = meshPath; + result["backend"] = backendName; + // Phase 9 (trellis2): the preserved full-resolution generation. + if (!res.sourceInterchangePath.isEmpty()) + result["sourcePath"] = res.sourceInterchangePath; // Surface non-fatal degradations (bake fell back to vertex colours, …) so // the MCP caller can tell a textured result from a fallback one. if (!res.warning.isEmpty()) result["warning"] = res.warning; @@ -9844,18 +9887,24 @@ QJsonArray MCPServer::buildToolsList() props["texture_size"] = QJsonObject{{"type", "integer"}, {"description", "Baked-texture resolution 64..8192 (default 1024)."}}; props["upscale_texture"] = QJsonObject{{"type", "boolean"}, {"description", "Run Real-ESRGAN 2x on the baked diffuse before saving (default false; best-effort — keeps the un-upscaled texture if the upscale model is unavailable)."}}; props["generate_pbr"] = QJsonObject{{"type", "boolean"}, {"description", "Synthesize normal + roughness maps from the baked diffuse (#404 PBRify) and bind them into the material — the polished-surface look (default true; requires bake_texture; fails soft to diffuse-only if the models are unavailable)."}}; - props["backend"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"triposr", "triposg"}}, {"description", "Generation backend (default triposr). triposr = fast single-pass LRM with color; triposg = 1.5B rectified-flow model — higher-fidelity GEOMETRY, slower, geometry-only (no texture bake). Both MIT. TripoSG models download on first use."}}; - props["flow_steps"] = QJsonObject{{"type", "integer"}, {"description", "TripoSG rectified-flow Euler steps 1..200 (default 25; 50 = reference quality, 10 = fast preview). Ignored by triposr."}}; - props["guidance"] = QJsonObject{{"type", "number"}, {"description", "TripoSG classifier-free-guidance scale 0..30 (default 7; 0 disables CFG and halves DiT cost). Ignored by triposr."}}; + props["backend"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"trellis2", "triposr", "triposg"}}, {"description", "Generation backend. DEFAULT: trellis2 when its runtime is installed on this machine, else triposr. trellis2 = Microsoft TRELLIS.2-4B (MIT) via the Python sidecar (Linux + NVIDIA GPU) — highest quality, real PBR (base color/metallic/roughness) baked natively by QtMeshEditor WITHOUT NVIDIA nvdiffrast/nvdiffrec; triposr = fast local single-pass LRM with color; triposg = 1.5B rectified-flow model — higher-fidelity GEOMETRY, slower, geometry-only."}}; + props["flow_steps"] = QJsonObject{{"type", "integer"}, {"description", "TripoSG rectified-flow Euler steps 1..200 (default 25; 50 = reference quality, 10 = fast preview). Ignored by the other backends."}}; + props["guidance"] = QJsonObject{{"type", "number"}, {"description", "TripoSG classifier-free-guidance scale 0..30 (default 7; 0 disables CFG and halves DiT cost). Ignored by the other backends."}}; + props["seed"] = QJsonObject{{"type", "integer"}, {"description", "trellis2 only: deterministic generation seed (default 42)."}}; + props["preset"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"fast", "balanced", "high"}}, {"description", "trellis2 only: quality preset (default balanced). fast = 512 pipeline, balanced = 1024 cascade, high = 1536 cascade (more VRAM/time)."}}; + props["target_tris"] = QJsonObject{{"type", "integer"}, {"description", "ALL backends: game-ready target triangle count — weld + debris-cull + simplify, re-baking lost detail as diffuse + tangent-space normal maps (0 = keep the original density; suggested presets: 10000 low / 25000 medium / 50000 high). For trellis2 the full-res source is preserved as a .qtm3d sidecar when 'output' is given."}}; appendTool( "generate_mesh_from_image", - "AI image-to-3D mesh generation (epic #764, TripoSR via ONNX): " - "reconstruct a 3D mesh from a single image. Runs the TripoSR encoder " - "(image -> triplane) + decoder (density grid) and extracts the surface " - "with native marching cubes. Returns vertexCount/triangleCount and, when " - "'output' is given, the saved meshPath; otherwise the mesh is loaded into " - "the scene. The model downloads on first use; without it (or a non-ONNX " - "build) the call returns a clear error (no crash).", + "AI image-to-3D mesh generation (epic #764 + TRELLIS.2): reconstruct a " + "3D mesh from a single image. Backends: trellis2 (Microsoft TRELLIS.2-4B " + "sidecar — the default when installed; PBR-textured, game-ready " + "processing + native texture bake), triposr (local ONNX, fast, " + "color), triposg (local ONNX, best local geometry). Returns " + "vertexCount/triangleCount/backend and, when 'output' is given, the " + "saved meshPath (+ sourcePath for the preserved trellis2 full-res " + "generation); otherwise the mesh is loaded into the scene. Models " + "download on first use; a missing runtime/model returns a clear error " + "(no crash).", props, QJsonArray{"image_path"} ); diff --git a/src/main.cpp b/src/main.cpp index 3f6037061..a582d7423 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -154,6 +154,13 @@ int main(int argc, char *argv[]) freopen("CONOUT$", "w", stderr); } #endif + // Match the GUI's QSettings identity — without this the CLI reads a + // DIFFERENT preferences domain (com.qtmesheditor.* vs the GUI's + // com.none.*) and settings like ai/trellis2Cli written by one surface + // are invisible to the other. + QCoreApplication::setOrganizationName("QtMeshEditor"); + QCoreApplication::setOrganizationDomain("none"); + QCoreApplication::setApplicationName("QtMeshEditor"); return CLIPipeline::run(argc, argv); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1ec9769b8..b3b0feb9e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -219,6 +219,9 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenBaker.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenPredictor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/TripoSGPredictor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/Trellis2Interchange.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/Trellis2Bake.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/Trellis2Predictor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenController.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/BackgroundRemover.cpp From 5b9aca282d0ccac6fe3e582f129b91c11e07931d Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 31 Aug 2026 11:05:25 -0400 Subject: [PATCH 2/5] ui: drop the runtime-flavor helper text when TRELLIS.2 is ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ⚠ hint stays only when the runtime is missing; a working setup needs no explanatory caption under the backend picker. Co-Authored-By: Claude Fable 5 --- qml/PropertiesPanel.qml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 6f0299e63..bbbbfa739 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -1983,17 +1983,6 @@ Rectangle { wrapMode: Text.WordWrap width: parent.width - 16 } - // Which runtime flavor is active (trellis.cpp / Python sidecar). - Text { - visible: mgBackendCombo.t2Selected && mgBackendCombo.t2Ready - text: " " + MeshGenController.trellis2RuntimeHint() - + " Textures + PBR maps are baked natively by QtMeshEditor." - color: PropertiesPanelController.textColor - font.pixelSize: 10 - wrapMode: Text.WordWrap - width: parent.width - 16 - } - // ---- TRELLIS.2 options (only for the TRELLIS.2 backend) ---------- Row { spacing: 6 From e88531c06b8767a13151231884aa66018e9c565c Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 31 Aug 2026 22:56:30 -0400 Subject: [PATCH 3/5] =?UTF-8?q?fix(trellis2):=20bake=20-90=C2=B0=20X=20rot?= =?UTF-8?q?ation=20so=20generated=20models=20stand=20upright?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TRELLIS.2 decodes in a Z-up frame; in the Y-up viewer the model arrived face-down. Rotate positions + object-space normals (x,y,z)->(x,z,-y) in the predictTrellis2 wrapper — rigid, so winding/UVs/tangent-space normal map are unaffected, and the kept .qtm3d source stays in the native frame (re-bakes come back through the same path). Verified: real-photo generation renders upright from all four turntable angles. Co-Authored-By: Claude Fable 5 --- src/ImageTo3D/MeshGenPredictor.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ImageTo3D/MeshGenPredictor.cpp b/src/ImageTo3D/MeshGenPredictor.cpp index 19e0d0a57..bf64bce5c 100644 --- a/src/ImageTo3D/MeshGenPredictor.cpp +++ b/src/ImageTo3D/MeshGenPredictor.cpp @@ -132,7 +132,26 @@ MeshGenPredictor::Result predictTrellis2( t2.mock = opts.trellis2Mock; t2.sourceKeepDir = opts.trellis2SourceKeepDir; t2.sourceKeepBaseName = opts.trellis2SourceKeepBaseName; - return Trellis2Predictor::predict(image, t2, progress); + MeshGenPredictor::Result r = Trellis2Predictor::predict(image, t2, progress); + // TRELLIS.2 decodes in a Z-up frame — in the viewer's Y-up world the + // model arrives face-down. Bake a -90° X rotation into the geometry: + // (x, y, z) -> (x, z, -y). Rigid (det +1), so winding, UVs and the + // tangent-space normal map are untouched; object-space vertex normals + // rotate with the positions. The kept .qtm3d source stays in the native + // frame (re-bakes come back through this same path). + if (r.ok) { + for (size_t v = 0; v + 2 < r.positions.size(); v += 3) { + const float y = r.positions[v + 1]; + r.positions[v + 1] = r.positions[v + 2]; + r.positions[v + 2] = -y; + } + for (size_t v = 0; v + 2 < r.normals.size(); v += 3) { + const float y = r.normals[v + 1]; + r.normals[v + 1] = r.normals[v + 2]; + r.normals[v + 2] = -y; + } + } + return r; } // Game-ready pass for the LOCAL backends (TripoSR/TripoSG): weld, drop From a884e3e256631df609de07c79b2f7d81abfd70c0 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 1 Sep 2026 01:20:45 -0400 Subject: [PATCH 4/5] fix: address PR #968 review feedback + Windows CI failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows CI: the --upscale-texture block in cmdGenerate3d became reachable in non-ONNX builds (the trellis2 backend needs no ONNX) but uses the ONNX-only Real-ESRGAN upscaler — now guarded with a clear warning fallback. Codex review: - MCP: generate_mesh_from_image is now advertised and functional in ONNX-off builds for the trellis2 backend (schema ungated; the handler gates only the local TripoSR/TripoSG paths + the upscaler on ENABLE_ONNX) - MeshGenBuilder: disabling PBR now suppresses the predictor-baked roughness/metallic/normal maps too — 'no PBR' means a diffuse-only asset - Trellis2Bake: BakeOptions::sourceNormalSmoothIterations is now actually applied (new smoothNormalField pass over the welded adjacency, both bake paths). Default changed to 0: the runtime now receives the REMESHED shell, and smoothing a clean source bakes curvature disagreement instead of removing noise; raise it for raw fallback dumps. Covered by a new smoothing-reduces-tilt assertion. CodeRabbit review: - Trellis2Interchange: overflow-checked manifest arithmetic (file-controlled shape/offset/byteLength), voxel coords validated against resolution, every QFile::write checked (truncated file removed + error), non-finite PBR lanes rejected before u8 quantization - Trellis2Predictor: mock requests fail clearly instead of falling through to a real trellis.cpp generation; usedModel honours env-triggered mock; QTMESH_TRELLIS2_IMPORT re-bake works with no runtime installed - MeshGenController: TRELLIS.2 always gets the alpha matte (checkbox now governs only the Tripo backends — matches the CLI) - guard script/test: scan targets validated (missing path no longer passes vacuously), editable/VCS requirement forms (-e git+…#egg=nvdiffrast) detected, bare '-' no longer allowlists requirement flags - install.py: hard stop on non-Linux (venv layout is Linux-only; points at the trellis.cpp runtime instead) - test hygiene: env overrides set after ASSERTs so a failed assertion can't leak fake runtime paths into later tests; QTMESH_TRELLIS2_CLI also pinned - docs: fence languages, model-download timing corrected to first generation Co-Authored-By: Claude Fable 5 --- ai/trellis2/README.md | 2 +- ai/trellis2/install.py | 11 ++- docs/TRELLIS2.md | 2 +- docs/trellis2-dependencies.md | 4 +- scripts/check-trellis2-restricted-deps.sh | 24 ++++-- src/CLIPipeline.cpp | 8 ++ ...LIPipeline_cmdgenerate3d_coverage_test.cpp | 9 ++- src/ImageTo3D/MeshGenBuilder.cpp | 11 ++- src/ImageTo3D/MeshGenController.cpp | 7 +- src/ImageTo3D/Trellis2Bake.cpp | 80 ++++++++++++++++++- src/ImageTo3D/Trellis2Bake.h | 13 +-- src/ImageTo3D/Trellis2Bake_test.cpp | 18 +++++ src/ImageTo3D/Trellis2Guard_test.cpp | 13 ++- src/ImageTo3D/Trellis2Interchange.cpp | 66 ++++++++++++--- src/ImageTo3D/Trellis2Predictor.cpp | 29 ++++--- src/MCPServer.cpp | 27 ++++--- 16 files changed, 267 insertions(+), 57 deletions(-) diff --git a/ai/trellis2/README.md b/ai/trellis2/README.md index 7ddaf2dfa..bebebe28e 100644 --- a/ai/trellis2/README.md +++ b/ai/trellis2/README.md @@ -8,7 +8,7 @@ GUI's *AI: Image → 3D* panel). `generate.py` runs **Microsoft TRELLIS.2** (MIT, pinned revision) inference only: -``` +```text RGBA image (alpha matte made by QtMeshEditor's own U²-Net) ↓ generate.py DINOv3 cond → sparse structure → shape SLat → tex SLat raw geometry + sparse PBR attribute volume (base color / metallic / roughness / alpha) diff --git a/ai/trellis2/install.py b/ai/trellis2/install.py index 3c531f1de..eb2233b45 100644 --- a/ai/trellis2/install.py +++ b/ai/trellis2/install.py @@ -159,8 +159,15 @@ def main() -> None: args = ap.parse_args() if not sys.platform.startswith("linux"): - log("WARNING: TRELLIS.2 upstream supports Linux + NVIDIA GPUs only; " - "continuing, but generation will not work on this platform.") + # Hard stop: upstream's CUDA extensions only build on Linux, and the + # venv layout below assumes env/bin/python (Windows venvs use + # env\Scripts\python.exe) — continuing would fail with a confusing + # filesystem error instead of this message. On macOS use the + # trellis.cpp runtime instead (docs/TRELLIS2.md). + raise SystemExit( + "TRELLIS.2's Python sidecar supports Linux + NVIDIA GPUs only. " + "On macOS/Windows install the trellis.cpp runtime instead — see " + "docs/TRELLIS2.md.") if sys.version_info < (3, 10): raise SystemExit("Python >= 3.10 required") if shutil.which("git") is None: diff --git a/docs/TRELLIS2.md b/docs/TRELLIS2.md index 38849df0f..cdd7bb438 100644 --- a/docs/TRELLIS2.md +++ b/docs/TRELLIS2.md @@ -14,7 +14,7 @@ next to the local ONNX backends: **TRELLIS.2 generates; QtMeshEditor makes the asset.** -``` +```text input image ── U²-Net alpha matte (QtMeshEditor, Apache-2.0) ──► RGBA ──► ai/trellis2/generate.py (inference ONLY: DINOv3 cond → sparse structure → shape SLat → tex SLat → raw mesh + sparse PBR attribute volume) diff --git a/docs/trellis2-dependencies.md b/docs/trellis2-dependencies.md index a68cae3ef..58f6193bd 100644 --- a/docs/trellis2-dependencies.md +++ b/docs/trellis2-dependencies.md @@ -67,8 +67,8 @@ backend specifically (all other QtMeshEditor features work without any of this). | Weights | Purpose | License | Bundled? | Commercial use | |---|---|---|---|---| -| `microsoft/TRELLIS.2-4B` (rev `af44b45f…`) | flow models + shape/tex VAEs | MIT | no — HF download on install | ✅ | -| `microsoft/TRELLIS-image-large` `ss_dec_conv3d_16l8_fp16` only (rev `25e0d31f…`) | sparse-structure decoder referenced by `pipeline.json` | MIT | no — HF download | ✅ | +| `microsoft/TRELLIS.2-4B` (rev `af44b45f…`) | flow models + shape/tex VAEs | MIT | no — HF download on first generation | ✅ | +| `microsoft/TRELLIS-image-large` `ss_dec_conv3d_16l8_fp16` only (rev `25e0d31f…`) | sparse-structure decoder referenced by `pipeline.json` | MIT | no — HF download on first generation | ✅ | | `facebook/dinov3-vitl16-pretrain-lvd1689m` | image conditioning encoder | **DINOv3 License** (custom Meta) | no — **gated** HF download by the user | ✅ with conditions (§4) | | `briaai/RMBG-2.0` | upstream default background remover (named in `pipeline.json`) | **CC BY-NC 4.0** ❌ | **never downloaded or loaded** — bypassed (§3) | ❌ non-commercial | | `ZhengPeng7/BiRefNet` | MIT alternative background remover | MIT | not used (QtMeshEditor does its own bg removal) | ✅ | diff --git a/scripts/check-trellis2-restricted-deps.sh b/scripts/check-trellis2-restricted-deps.sh index 017882149..43c26a1af 100755 --- a/scripts/check-trellis2-restricted-deps.sh +++ b/scripts/check-trellis2-restricted-deps.sh @@ -13,21 +13,35 @@ fail=0 note() { echo "check-trellis2-restricted-deps: $*"; } +# Every scan target must exist and be readable — a missing path would make +# grep exit 2, which the `if` conditions below would treat exactly like +# "no match" and the gate would pass vacuously. +for target in ai/trellis2 src/ImageTo3D ai/trellis2/requirements.txt ai/trellis2/install.py src/ImageTo3D/Trellis2Predictor.cpp src/ImageTo3D/Trellis2Bake.cpp src/ImageTo3D/Trellis2Interchange.cpp src/ImageTo3D/Trellis2Predictor.h src/ImageTo3D/Trellis2Bake.h src/ImageTo3D/Trellis2Interchange.h; do + if [ ! -r "$target" ]; then + note "FAIL: scan target missing/unreadable: $target" + fail=1 + fi +done +[ "$fail" -ne 0 ] && { note "prohibited-dependency check FAILED"; exit 1; } + # 1. No import/require form ANYWHERE in the sidecar or the C++ integration. # (import nvdiffrast / from nvdiffrast import / import nvdiffrec…) # The guard test itself is an allowed location — it QUOTES the import forms # in order to detect them (same carve-out the spec gives this script). if grep -RInE '(^|[^a-zA-Z_])(import|from)[[:space:]]+nvdiff(rast|rec)' \ --exclude='Trellis2Guard_test.cpp' \ - ai/trellis2 src/ImageTo3D 2>/dev/null; then + ai/trellis2 src/ImageTo3D; then note "FAIL: an import of a prohibited NVIDIA library was introduced." fail=1 fi # 2. Dependency manifests must not list them (or the PyPI `cumesh` trap — an # unrelated, unlicensed package; CuMesh is built from the pinned checkout). -if grep -RInE '^[[:space:]]*(nvdiffrast|nvdiffrec|cumesh)([=<>![:space:];[]|$)' \ - ai/trellis2/requirements.txt 2>/dev/null; then +# Ban the names ANYWHERE in a non-comment line — editable/VCS forms +# ("-e git+…#egg=nvdiffrast", "nvdiffrast @ git+…") don't start with the +# package name. +if grep -v '^[[:space:]]*#' ai/trellis2/requirements.txt \ + | grep -inE 'nvdiffrast|nvdiffrec|cumesh'; then note "FAIL: a prohibited/trap package appears in requirements.txt." fail=1 fi @@ -39,14 +53,14 @@ if grep -InE 'nvdiff(rast|rec)' \ src/ImageTo3D/Trellis2Interchange.cpp \ src/ImageTo3D/Trellis2Predictor.h \ src/ImageTo3D/Trellis2Bake.h \ - src/ImageTo3D/Trellis2Interchange.h 2>/dev/null \ + src/ImageTo3D/Trellis2Interchange.h \ | grep -vE ':[0-9]+:[[:space:]]*(//|\*)' ; then note "FAIL: non-comment reference to a prohibited NVIDIA library in the C++ integration." fail=1 fi # 4. install.py must not clone/install them either (git URLs, pip specs). -if grep -InE 'nvdiffrast\.git|nvdiffrec\.git|pip.*nvdiff' ai/trellis2/install.py 2>/dev/null; then +if grep -InE 'nvdiffrast\.git|nvdiffrec\.git|pip.*nvdiff' ai/trellis2/install.py; then note "FAIL: install.py fetches a prohibited NVIDIA library." fail=1 fi diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index c79e10e57..0f4484570 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -10757,6 +10757,7 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) "upscale (was the bake disabled or did it fall back?)." << Qt::endl; } if (upscaleTex && !res.uvs.empty() && !res.texture.isNull()) { +#ifdef ENABLE_ONNX const QString upModel = AIAssistManager::instance()->ensureUpscaleModel(2); if (upModel.isEmpty()) { err() << "Warning: upscale model unavailable — keeping the " @@ -10770,6 +10771,13 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) err() << "Warning: texture upscale failed (" << ur.error << ") — keeping the un-upscaled texture." << Qt::endl; } +#else + // Reachable on non-ONNX builds via the TRELLIS.2 backend (which has + // no ONNX dependency) — the Real-ESRGAN upscaler is ONNX-only. + err() << "Warning: --upscale-texture requires an ONNX build " + "(rebuild with -DENABLE_ONNX=ON) — keeping the un-upscaled " + "texture." << Qt::endl; +#endif } // Baked texture (+ synthesized PBR maps) land next to the exported mesh diff --git a/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp b/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp index d606558e0..ea547f293 100644 --- a/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp +++ b/src/ImageTo3D/CLIPipeline_cmdgenerate3d_coverage_test.cpp @@ -146,17 +146,22 @@ TEST(CLIPipelineCmdGenerate3dCoverage, Trellis2BackendAcceptedButUnknownRejected // trellis2 is a valid backend; with a real image but a deliberately // nonexistent runtime the command must fail at RUNTIME (1) with the // install hint — never crash, never a usage error. - qputenv("QTMESH_TRELLIS2_ENV", "/nonexistent/qtmesh-trellis2-cli-ut"); - qputenv("QTMESH_TRELLIS2_PYTHON", "/nonexistent/python-cli-ut"); + // All ASSERTs run BEFORE the env override: an ASSERT returns from the + // test body immediately, and env vars set before a failed ASSERT would + // leak into every later test in this process. QTemporaryDir tmp; ASSERT_TRUE(tmp.isValid()); const QString png = QDir(tmp.path()).filePath("in.png"); QImage img(16, 16, QImage::Format_RGB888); img.fill(Qt::red); ASSERT_TRUE(img.save(png, "PNG")); + qputenv("QTMESH_TRELLIS2_ENV", "/nonexistent/qtmesh-trellis2-cli-ut"); + qputenv("QTMESH_TRELLIS2_PYTHON", "/nonexistent/python-cli-ut"); + qputenv("QTMESH_TRELLIS2_CLI", "/nonexistent/trellis-cli-cli-ut"); const QByteArray pngBytes = png.toLocal8Bit(); Gen3dArgv ok({"generate3d", pngBytes.constData(), "--backend", "trellis2"}); EXPECT_EQ(CLIPipeline::cmdGenerate3d(ok.argc(), ok.argv()), 1); qunsetenv("QTMESH_TRELLIS2_ENV"); qunsetenv("QTMESH_TRELLIS2_PYTHON"); + qunsetenv("QTMESH_TRELLIS2_CLI"); } diff --git a/src/ImageTo3D/MeshGenBuilder.cpp b/src/ImageTo3D/MeshGenBuilder.cpp index 2278604a5..3aa70b398 100644 --- a/src/ImageTo3D/MeshGenBuilder.cpp +++ b/src/ImageTo3D/MeshGenBuilder.cpp @@ -326,9 +326,14 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, unique + QStringLiteral("_%1.png").arg(QLatin1String(suffix))); return img.save(p, "PNG") ? p : QString(); }; - normalPath = saveMap(result.normalMap, "normal"); - roughnessPath = saveMap(result.roughnessMap, "roughness"); - metallicPath = saveMap(result.metallicMap, "metallic"); + // Honour the disabled-PBR option for the REAL baked maps too — + // "no PBR" must mean a diffuse-only asset, not "no synthesized + // maps but the generated ones still bind". + if (opts.generatePbrMaps) { + normalPath = saveMap(result.normalMap, "normal"); + roughnessPath = saveMap(result.roughnessMap, "roughness"); + metallicPath = saveMap(result.metallicMap, "metallic"); + } // Optional PBR stage (#404): synthesize normal + roughness from the // baked diffuse BEFORE the resource location is (re)indexed so the diff --git a/src/ImageTo3D/MeshGenController.cpp b/src/ImageTo3D/MeshGenController.cpp index 4bdde6738..11c320367 100644 --- a/src/ImageTo3D/MeshGenController.cpp +++ b/src/ImageTo3D/MeshGenController.cpp @@ -421,8 +421,11 @@ void MeshGenController::generate(const QString& imagePath, int resolution, opts.vertexColor = true; // TripoSR removal already ran above; TripoSG's white-background and // TRELLIS.2's keep-alpha matte removal happen inside the predictor - // dispatch. - opts.removeBackground = rembg && (useSG || useT2); + // dispatch. TRELLIS.2 ALWAYS gets the matte (the CLI forces it too): + // its preprocess needs an alpha channel to keep the non-commercial + // rembg model unused, so the GUI checkbox only governs the Tripo + // backends. + opts.removeBackground = useT2 || (rembg && useSG); opts.smoothMesh = wantSmooth; opts.refineSurface = wantRefine; opts.bakeTexture = wantBake; diff --git a/src/ImageTo3D/Trellis2Bake.cpp b/src/ImageTo3D/Trellis2Bake.cpp index 61fa4bd3c..c657e8da2 100644 --- a/src/ImageTo3D/Trellis2Bake.cpp +++ b/src/ImageTo3D/Trellis2Bake.cpp @@ -135,6 +135,76 @@ std::vector smoothNormalsWelded(const std::vector& positions, return n; } +// Laplacian-smooth a per-vertex normal FIELD over the position-welded vertex +// adjacency (renormalizing each pass). Raw dual-grid surfaces carry +// voxel-scale normal noise; baked as a detail normal it reads as glittery +// specular speckle. A few averaging passes flatten the noise while the +// underlying geometry (and therefore real relief) is untouched. +std::vector smoothNormalField(std::vector normals, + const std::vector& positions, + const std::vector& indices, + int iterations) +{ + if (iterations <= 0 || normals.size() != positions.size()) + return normals; + const size_t nv = positions.size() / 3; + // Weld by bit-identical position so seam-split vertices smooth together. + struct PosKey { + uint32_t a, b, c; + bool operator==(const PosKey& o) const + { return a == o.a && b == o.b && c == o.c; } + }; + struct PosKeyHash { + size_t operator()(const PosKey& k) const + { + uint64_t h = k.a; + h = h * 0x9E3779B97F4A7C15ull + k.b; + h = h * 0x9E3779B97F4A7C15ull + k.c; + return static_cast(h ^ (h >> 32)); + } + }; + std::unordered_map canonOf; + canonOf.reserve(nv * 2); + std::vector canon(nv); + for (size_t v = 0; v < nv; ++v) { + PosKey k; + std::memcpy(&k.a, &positions[v * 3 + 0], 4); + std::memcpy(&k.b, &positions[v * 3 + 1], 4); + std::memcpy(&k.c, &positions[v * 3 + 2], 4); + canon[v] = canonOf.emplace(k, static_cast(v)).first->second; + } + // Canonical edge list (deduped implicitly by symmetric accumulation). + std::vector acc; + for (int it = 0; it < iterations; ++it) { + acc.assign(positions.size(), 0.0f); + for (size_t t = 0; t + 2 < indices.size(); t += 3) { + for (int k = 0; k < 3; ++k) { + const uint32_t a = canon[indices[t + k]]; + const uint32_t b = canon[indices[t + (k + 1) % 3]]; + if (a == b) + continue; + for (int c = 0; c < 3; ++c) { + acc[a * 3 + c] += normals[b * 3 + c]; + acc[b * 3 + c] += normals[a * 3 + c]; + } + } + } + for (size_t v = 0; v < nv; ++v) { + const uint32_t cv = canon[v]; + float nn[3] = {normals[cv * 3 + 0] + acc[cv * 3 + 0], + normals[cv * 3 + 1] + acc[cv * 3 + 1], + normals[cv * 3 + 2] + acc[cv * 3 + 2]}; + const float l = len3(nn); + if (l > 1e-20f) { + normals[v * 3 + 0] = nn[0] / l; + normals[v * 3 + 1] = nn[1] / l; + normals[v * 3 + 2] = nn[2] / l; + } + } + } + return normals; +} + // ---- sparse uniform grid over source triangles for closest-point queries --- class TriangleGrid { public: @@ -1013,8 +1083,9 @@ BakeResult bake(const std::vector& targetPositions, &origNormals[static_cast(xref[v]) * 3], sizeof(float) * 3); } - const std::vector sNormals = - smoothNormals(sourcePositions, sourceIndices); + const std::vector sNormals = smoothNormalField( + smoothNormals(sourcePositions, sourceIndices), + sourcePositions, sourceIndices, opts.sourceNormalSmoothIterations); std::vector tTangent; // xyzw per vertex (w = handedness) if (opts.bakeNormalMap) { std::vector tan1(r.positions.size(), 0.0f); @@ -1495,8 +1566,9 @@ NormalBakeResult bakeDetailNormal(const std::vector& targetPositions, // accumulated per split vertex (UV seams SHOULD split the tangent basis). const std::vector tNormals = smoothNormalsWelded(targetPositions, targetIndices); - const std::vector sNormals = - smoothNormalsWelded(sourcePositions, sourceIndices); + const std::vector sNormals = smoothNormalField( + smoothNormalsWelded(sourcePositions, sourceIndices), + sourcePositions, sourceIndices, opts.sourceNormalSmoothIterations); std::vector tan1(targetPositions.size(), 0.0f); std::vector tan2(targetPositions.size(), 0.0f); for (size_t t = 0; t + 2 < targetIndices.size(); t += 3) { diff --git a/src/ImageTo3D/Trellis2Bake.h b/src/ImageTo3D/Trellis2Bake.h index 60c40700a..30c6386ff 100644 --- a/src/ImageTo3D/Trellis2Bake.h +++ b/src/ImageTo3D/Trellis2Bake.h @@ -130,11 +130,14 @@ struct BakeOptions { int supersample = 1; // 1 or 2 (2 = 2×2 subsamples per texel) bool bakeNormalMap = true; // bake source detail normals (for simplified targets) // Laplacian smoothing iterations applied to the SOURCE normal field - // before it feeds the detail-normal bake. Raw dual-grid surfaces carry - // voxel-scale normal noise that otherwise bakes into a glittery normal - // map (white specular speckle); ~8 iterations flattens the noise while - // keeping shape-scale relief. 0 disables. - int sourceNormalSmoothIterations = 8; + // before it feeds the detail-normal bake (0 = off, the default). Raw + // dual-grid dumps carried voxel-scale normal noise that baked into a + // glittery normal map; the runtime now receives the REMESHED shell, so + // by default the field is used as-is — smoothing a clean source would + // instead bake curvature disagreement (rounded source vs one-ring + // target normals) into the map. Raise this only when baking from a raw + // (un-remeshed) dual-grid source. + int sourceNormalSmoothIterations = 0; // done/total covered texels; return false to cancel. std::function progress; }; diff --git a/src/ImageTo3D/Trellis2Bake_test.cpp b/src/ImageTo3D/Trellis2Bake_test.cpp index 54bd22ea8..6aa1c9788 100644 --- a/src/ImageTo3D/Trellis2Bake_test.cpp +++ b/src/ImageTo3D/Trellis2Bake_test.cpp @@ -331,6 +331,24 @@ TEST(Trellis2BakeTest, DetailNormalEncodesSourceRelief) ++tilted; } EXPECT_GT(tilted, 20); + + // And the (opt-in) source-normal field smoothing actually smooths: the + // same bake with smoothing passes must tilt strictly fewer texels — + // this fixture's single-edge-scale ridge is exactly the "noise" scale + // the option exists to flatten on raw dual-grid sources. + Trellis2Bake::BakeOptions bo; + bo.sourceNormalSmoothIterations = 8; + const auto rs = Trellis2Bake::bakeDetailNormal(tpos, tidx, tuvs, 64, 64, + spos, sidx, bo); + ASSERT_TRUE(rs.ok) << rs.error.toStdString(); + int tiltedSmoothed = 0; + for (int y = 4; y < 60; y += 4) + for (int x = 4; x < 60; x += 4) { + const uchar* p = rs.normalMap.constScanLine(y) + size_t(x) * 3; + if (std::abs(int(p[0]) - 128) > 8) + ++tiltedSmoothed; + } + EXPECT_LT(tiltedSmoothed, tilted); } TEST(Trellis2BakeTest, DetailNormalRejectsBadInput) diff --git a/src/ImageTo3D/Trellis2Guard_test.cpp b/src/ImageTo3D/Trellis2Guard_test.cpp index 34a92dcbc..bb962a688 100644 --- a/src/ImageTo3D/Trellis2Guard_test.cpp +++ b/src/ImageTo3D/Trellis2Guard_test.cpp @@ -34,8 +34,11 @@ bool lineIsAllowlisted(const QString& line) { const QString t = line.trimmed(); // Python/requirements comments and Markdown prose are documentation. + // NB: only a Markdown BULLET ("- text") is prose — a bare '-' prefix + // would also allowlist pip requirement flags like + // "-e git+…/nvdiffrast.git#egg=nvdiffrast". if (t.startsWith(QLatin1Char('#')) || t.startsWith(QLatin1Char('*')) - || t.startsWith(QLatin1Char('-')) || t.startsWith(QLatin1Char('|')) + || t.startsWith(QLatin1String("- ")) || t.startsWith(QLatin1Char('|')) || t.startsWith(QLatin1Char('>'))) return true; // The explicit guard/report constructs in generate.py. @@ -118,6 +121,14 @@ TEST(Trellis2GuardTest, RequirementsNeverListRestrictedOrTrapPackages) // The PyPI package named `cumesh` is an unrelated, unlicensed project — // CuMesh must be built from the pinned JeffreyXiang/CuMesh checkout. EXPECT_NE(pkg, QStringLiteral("cumesh")) << line.toStdString(); + // Editable/VCS requirement forms ("-e git+…#egg=nvdiffrast", + // "nvdiffrast @ git+…") hide the name from the pkg-prefix parse — + // ban the names ANYWHERE in a non-comment requirement line. + const QString low = line.toLower(); + EXPECT_FALSE(low.contains(QLatin1String("nvdiffrast")) + || low.contains(QLatin1String("nvdiffrec")) + || low.contains(QLatin1String("cumesh"))) + << line.toStdString(); } } diff --git a/src/ImageTo3D/Trellis2Interchange.cpp b/src/ImageTo3D/Trellis2Interchange.cpp index 2b41bd572..f8c12a90b 100644 --- a/src/ImageTo3D/Trellis2Interchange.cpp +++ b/src/ImageTo3D/Trellis2Interchange.cpp @@ -6,6 +6,7 @@ #include #include +#include namespace Trellis2Interchange { @@ -55,10 +56,14 @@ qint64 elementSize(const QString& dtype) qint64 elementCount(const ArrayRef& ref) { + // shape values are FILE-CONTROLLED: multiply with overflow checks so a + // crafted manifest can't wrap the count and slip past checkedBlob(). qint64 n = 1; for (qint64 d : ref.shape) { if (d < 0) return -1; + if (d != 0 && n > std::numeric_limits::max() / d) + return -1; n *= d; } return n; @@ -75,12 +80,20 @@ const uint8_t* checkedBlob(const QByteArray& blob, qint64 blobBase, return nullptr; } const qint64 count = elementCount(ref); - if (count < 0 || count * esize != ref.byteLength) { + if (count < 0 || count > std::numeric_limits::max() / esize + || count * esize != ref.byteLength) { *error = QStringLiteral("array size mismatch (dtype %1)").arg(ref.dtype); return nullptr; } + // offset/byteLength are file-controlled too — reject any range whose + // arithmetic would overflow before it can be bounds-checked. + if (ref.offset > std::numeric_limits::max() - blobBase) { + *error = QStringLiteral("array exceeds file bounds"); + return nullptr; + } const qint64 start = blobBase + ref.offset; - if (start < 0 || start + ref.byteLength > blob.size()) { + if (start < 0 || ref.byteLength > blob.size() + || start > blob.size() - ref.byteLength) { *error = QStringLiteral("array exceeds file bounds"); return nullptr; } @@ -194,6 +207,21 @@ ReadResult read(const QString& path) for (size_t i = 0; i < d.voxelCoords.size(); ++i) d.voxelCoords[i] = s[i]; } + // Coordinates index the res³ grid — a coordinate outside + // [0, resolution) would make every volume sampler mis-key. + if (d.resolution <= 0) { + r.error = QStringLiteral( + "'voxel_coords' present without a positive 'resolution'"); + return r; + } + for (uint32_t c : d.voxelCoords) { + if (c >= static_cast(d.resolution)) { + r.error = QStringLiteral( + "voxel coordinate %1 outside grid (resolution %2)") + .arg(c).arg(d.resolution); + return r; + } + } } } @@ -312,30 +340,45 @@ bool write(const QString& path, const Data& data, QString* error) QFile f(path); if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) return failWith(QStringLiteral("cannot write %1").arg(path)); - f.write(kMagic, 8); + // A short write (disk full, quota, unplugged volume) must not leave a + // truncated file that reads as "valid until it isn't" — check every + // write and fail loudly. + auto put = [&f](const char* p, qint64 n) { + return f.write(p, n) == n; + }; + bool ok = put(kMagic, 8); const uint32_t version = kVersion; const uint32_t jsonLen = static_cast(json.size()); - f.write(reinterpret_cast(&version), 4); - f.write(reinterpret_cast(&jsonLen), 4); - f.write(json); + ok = ok && put(reinterpret_cast(&version), 4); + ok = ok && put(reinterpret_cast(&jsonLen), 4); + ok = ok && f.write(json) == json.size(); const qint64 headerLen = 16 + json.size(); const qint64 blobBase = align16(headerLen); static const char zeros[16] = {}; - f.write(zeros, blobBase - headerLen); + ok = ok && put(zeros, blobBase - headerLen); qint64 pos = 0; for (const Entry& e : entries) { + if (!ok) + break; const qint64 wanted = static_cast(arrays.value(QLatin1String(e.name)) .toObject() .value(QStringLiteral("offset")) .toDouble()); if (wanted > pos) { - f.write(zeros, wanted - pos); + ok = put(zeros, wanted - pos); pos = wanted; } - f.write(reinterpret_cast(e.ptr), e.bytes); + ok = ok && put(reinterpret_cast(e.ptr), e.bytes); pos += e.bytes; } + ok = ok && f.flush(); + if (!ok) { + f.close(); + QFile::remove(path); + return failWith(QStringLiteral("short write to %1 (%2)") + .arg(path, f.errorString())); + } return true; } @@ -416,7 +459,10 @@ ReadResult readTrellisCppDump(const QString& path) const float* ap = reinterpret_cast(p); for (size_t i = 0; i < d.voxelAttrs.size(); ++i) { const float v = ap[i]; - const float cl = v < 0.0f ? 0.0f : (v > 1.0f ? 1.0f : v); + // NaN fails both comparisons and float->u8 of NaN is UB — + // treat non-finite lanes as 0. + const float cl = !std::isfinite(v) ? 0.0f + : (v < 0.0f ? 0.0f : (v > 1.0f ? 1.0f : v)); d.voxelAttrs[i] = static_cast(cl * 255.0f + 0.5f); } } diff --git a/src/ImageTo3D/Trellis2Predictor.cpp b/src/ImageTo3D/Trellis2Predictor.cpp index ae9405114..1b535ac52 100644 --- a/src/ImageTo3D/Trellis2Predictor.cpp +++ b/src/ImageTo3D/Trellis2Predictor.cpp @@ -191,13 +191,25 @@ MeshGenPredictor::Result Trellis2Predictor::predict( return failResult(QStringLiteral("trellis2: input image is null.")); RuntimeKind kind = runtimeKind(); // --mock is a Python-sidecar feature (synthetic generation for GPU-less - // plumbing tests) — route mock runs there even when trellis.cpp is found. - if ((opts.mock || qEnvironmentVariableIsSet("QTMESH_TRELLIS2_MOCK")) - && kind == RuntimeKind::TrellisCpp - && !pythonPath().isEmpty() && !generateScriptPath().isEmpty()) { - kind = RuntimeKind::PythonSidecar; + // plumbing tests) — route mock runs there even when trellis.cpp is found, + // and NEVER silently fall through to a real multi-minute generation when + // the sidecar isn't there to serve the mock. + const bool mockRun = + opts.mock || qEnvironmentVariableIsSet("QTMESH_TRELLIS2_MOCK"); + if (mockRun && kind != RuntimeKind::None) { + if (!pythonPath().isEmpty() && !generateScriptPath().isEmpty()) + kind = RuntimeKind::PythonSidecar; + else if (kind == RuntimeKind::TrellisCpp) + return failResult(QStringLiteral( + "trellis2: mock generation needs the Python sidecar " + "(ai/trellis2/generate.py + a python with numpy/Pillow) — " + "refusing to run a real trellis.cpp generation for a mock " + "request.")); } - if (kind == RuntimeKind::None) + // The QTMESH_TRELLIS2_IMPORT re-bake seam (Phase 9) skips inference + // entirely — it must work with NO runtime installed. + const QString importPath = qEnvironmentVariable("QTMESH_TRELLIS2_IMPORT"); + if (kind == RuntimeKind::None && importPath.isEmpty()) return failResult(runtimeDescription()); auto report = [&progress](Stage s, int done, int total) -> bool { @@ -223,7 +235,7 @@ MeshGenPredictor::Result Trellis2Predictor::predict( }(); QString warning; bool matteReady = inputHasMatte; - if (!inputHasMatte && opts.removeBackground && !opts.mock) { + if (!inputHasMatte && opts.removeBackground && !mockRun) { if (!report(Stage::Encode, 0, 1)) return failResult(QStringLiteral("cancelled")); if (BackgroundRemover::isAvailable()) { @@ -272,7 +284,6 @@ MeshGenPredictor::Result Trellis2Predictor::predict( // skips inference entirely and re-runs the native pipeline (game-ready + // bake) on a previously preserved generation — re-bake textures / // re-target LODs without paying the model again. - const QString importPath = qEnvironmentVariable("QTMESH_TRELLIS2_IMPORT"); if (!importPath.isEmpty()) { Trellis2Interchange::ReadResult rr = Trellis2Interchange::read(importPath); if (!rr.ok) @@ -545,7 +556,7 @@ MeshGenPredictor::Result Trellis2Predictor::predict( MeshGenPredictor::Result r; r.warning = warning; r.sourceInterchangePath = keptSourcePath; - r.usedModel = !opts.mock; + r.usedModel = !mockRun; r.bakeTripoSROrientation = false; // TRELLIS.2 is +Y-up like TripoSG // ---- 5. native multi-channel PBR bake (Phase 7) ---------------------------- diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index c08ffc41c..34906b174 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -2712,12 +2712,9 @@ QJsonObject MCPServer::toolGeneratePbrMaps(const QJsonObject &args) QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) { -#ifndef ENABLE_ONNX - Q_UNUSED(args); - return makeErrorResult( - "This build was compiled without AI image-to-3D generation " - "(rebuild with -DENABLE_ONNX=ON)."); -#else + // NOT blanket-gated on ENABLE_ONNX: the TRELLIS.2 backend runs through an + // external runtime (trellis.cpp / Python sidecar) and works in ONNX-off + // builds — only the local TripoSR/TripoSG paths require the ONNX build. const QString imagePath = args.value("image_path").toString(); if (imagePath.trimmed().isEmpty()) return makeErrorResult("'image_path' is required."); @@ -2798,6 +2795,13 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) .arg(QFileInfo(imagePath).fileName()).arg(opts.sdfResolution) .arg(backendName)); +#ifndef ENABLE_ONNX + if (opts.backend != MeshGenPredictor::Backend::Trellis2) + return makeErrorResult( + "This build was compiled without local AI image-to-3D generation " + "(rebuild with -DENABLE_ONNX=ON, or install the TRELLIS.2 runtime " + "and use backend 'trellis2')."); +#endif if (opts.backend == MeshGenPredictor::Backend::Trellis2) { if (!Trellis2Predictor::runtimeAvailable()) return makeErrorResult(Trellis2Predictor::runtimeDescription()); @@ -2846,6 +2850,7 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) // Optional Real-ESRGAN 2x on the baked diffuse (best-effort; keeps the // un-upscaled texture on any failure — same policy as the CLI). +#ifdef ENABLE_ONNX if (upscaleTex && !res.uvs.empty() && !res.texture.isNull()) { const QString upModel = AIAssistManager::instance()->ensureUpscaleModel(2); if (!upModel.isEmpty()) { @@ -2855,6 +2860,9 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) res.texture = ur.image; } } +#else + Q_UNUSED(upscaleTex); // Real-ESRGAN is ONNX-only; best-effort, skipped. +#endif // Baked texture (+ synthesized PBR maps): land next to the export target // when one is given so the references survive outside the app; else the @@ -2896,7 +2904,6 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) // the MCP caller can tell a textured result from a fallback one. if (!res.warning.isEmpty()) result["warning"] = res.warning; return result; -#endif } QJsonObject MCPServer::toolUpscaleTexture(const QJsonObject &args) @@ -9871,8 +9878,9 @@ QJsonArray MCPServer::buildToolsList() ); } -#ifdef ENABLE_ONNX - // generate_mesh_from_image (#764) — only advertised when ONNX is compiled in. + // generate_mesh_from_image (#764) — always advertised: the TRELLIS.2 + // backend runs through an external runtime and needs no ONNX build (the + // handler gates the local TripoSR/TripoSG paths on ENABLE_ONNX itself). { QJsonObject props; props["image_path"] = QJsonObject{{"type", "string"}, {"description", "Absolute path to the source image (a single object, ideally background-removed). Required."}}; @@ -9909,7 +9917,6 @@ QJsonArray MCPServer::buildToolsList() QJsonArray{"image_path"} ); } -#endif // ENABLE_ONNX // save_scene { From 72659937af53947341cc0b4ff00b178c37b1d07c Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 1 Sep 2026 14:29:17 -0400 Subject: [PATCH 5/5] fix(bake): two-phase smoothNormalField update; chore: bump version to 3.36.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smoothNormalField wrote normalized results back into normals[] while later weld-group members still read their canonical slot — split vertices received the POST-update canonical value (canon[v] is the group's smallest index, so it updates first) and diverged from their canonical vertex, compounding per iteration. Resolve every canonical slot from pre-iteration values into a scratch buffer, then broadcast to all group members (review follow-up). Version 3.36.0 for the post-merge release; doc refs synced via scripts/sync-doc-versions-from-cmake.sh. Co-Authored-By: Claude Fable 5 --- CMakeLists.txt | 2 +- README.md | 22 +++++++++---------- src/ImageTo3D/Trellis2Bake.cpp | 29 +++++++++++++++++++------ website/src/hooks/useQtmeshActionRef.js | 2 +- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b26eaf7e..fc5f2bd03 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ cmake_minimum_required(VERSION 3.24.0) cmake_policy(SET CMP0005 NEW) cmake_policy(SET CMP0048 NEW) # manages project version -project(QtMeshEditor VERSION 3.35.0 LANGUAGES C CXX) +project(QtMeshEditor VERSION 3.36.0 LANGUAGES C CXX) message(STATUS "Building QtMeshEditor version ${PROJECT_VERSION}") set(QTMESHEDITOR_VERSION_STRING "\"${PROJECT_VERSION}\"") diff --git a/README.md b/README.md index d8caf584d..4fc7661dd 100755 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Available on the [GitHub Actions Marketplace](https://github.com/marketplace/act **Versioning** - **Always follow the latest GitHub release** — use the Marketplace floating tag `fernandotonon/QtMeshEditor@v1` (same pattern as the [Marketplace example](https://github.com/marketplace/actions/qtmesheditor)). The composite action defaults to `image-tag: latest`, so the Docker CLI tracks the newest published `ghcr.io/fernandotonon/qtmesh` image. -- **Reproducible builds** — pin the action and the container to the same semver as this repository’s `project(QtMeshEditor VERSION …)` in `CMakeLists.txt` (currently **3.35.0**). After bumping the version in CMake, run `./scripts/sync-doc-versions-from-cmake.sh` to refresh the pinned refs in `README.md` and the docs site fallback; CI enforces the match with `./scripts/sync-doc-versions-from-cmake.sh --check`. +- **Reproducible builds** — pin the action and the container to the same semver as this repository’s `project(QtMeshEditor VERSION …)` in `CMakeLists.txt` (currently **3.36.0**). After bumping the version in CMake, run `./scripts/sync-doc-versions-from-cmake.sh` to refresh the pinned refs in `README.md` and the docs site fallback; CI enforces the match with `./scripts/sync-doc-versions-from-cmake.sh --check`. Pinned workflow template (action + `ghcr.io` image aligned): @@ -48,10 +48,10 @@ jobs: - uses: actions/checkout@v4 - name: Run QtMesh scan - uses: fernandotonon/QtMeshEditor@3.35.0 + uses: fernandotonon/QtMeshEditor@3.36.0 with: command: scan - image-tag: "3.35.0" + image-tag: "3.36.0" env: QTMESH_CLOUD_TOKEN: ${{ secrets.QTMESH_CLOUD_TOKEN }} ``` @@ -76,37 +76,37 @@ Release tags are listed on the [releases page](https://github.com/fernandotonon/ ```yaml # Validate a specific mesh -- uses: fernandotonon/QtMeshEditor@3.35.0 +- uses: fernandotonon/QtMeshEditor@3.36.0 with: command: validate input-file: ./models/character.fbx - image-tag: "3.35.0" + image-tag: "3.36.0" # Convert FBX → glTF -- uses: fernandotonon/QtMeshEditor@3.35.0 +- uses: fernandotonon/QtMeshEditor@3.36.0 with: command: convert input-file: ./models/character.fbx output-file: ./output/character.gltf2 - image-tag: "3.35.0" + image-tag: "3.36.0" # Resample Mixamo animations (200+ keyframes → 30) -- uses: fernandotonon/QtMeshEditor@3.35.0 +- uses: fernandotonon/QtMeshEditor@3.36.0 with: command: anim input-file: ./animations/dance.fbx output-file: ./output/dance_optimized.fbx options: --resample 30 - image-tag: "3.35.0" + image-tag: "3.36.0" # Get mesh info as JSON -- uses: fernandotonon/QtMeshEditor@3.35.0 +- uses: fernandotonon/QtMeshEditor@3.36.0 id: info with: command: info input-file: ./models/character.fbx options: --json - image-tag: "3.35.0" + image-tag: "3.36.0" # Docker (alternative — :latest tracks newest image; pin :3.4.0 to match semver action ref) # The image is multi-arch (linux/amd64 + linux/arm64), so it runs natively on diff --git a/src/ImageTo3D/Trellis2Bake.cpp b/src/ImageTo3D/Trellis2Bake.cpp index c657e8da2..15ef8e48b 100644 --- a/src/ImageTo3D/Trellis2Bake.cpp +++ b/src/ImageTo3D/Trellis2Bake.cpp @@ -189,18 +189,33 @@ std::vector smoothNormalField(std::vector normals, } } } + // Two-phase update: resolve every canonical slot from the + // PRE-iteration values into a scratch buffer first, then broadcast. + // Writing normals[] in place while later weld-group members still + // read their canonical slot would hand them the already-normalized + // post-update value (canon[v] is the smallest index of the group, + // so it is always updated first) — split vertices would diverge + // from their canonical vertex, compounding per iteration. + std::vector next(positions.size()); for (size_t v = 0; v < nv; ++v) { - const uint32_t cv = canon[v]; - float nn[3] = {normals[cv * 3 + 0] + acc[cv * 3 + 0], - normals[cv * 3 + 1] + acc[cv * 3 + 1], - normals[cv * 3 + 2] + acc[cv * 3 + 2]}; + if (canon[v] != v) + continue; // canonical slots only + float nn[3] = {normals[v * 3 + 0] + acc[v * 3 + 0], + normals[v * 3 + 1] + acc[v * 3 + 1], + normals[v * 3 + 2] + acc[v * 3 + 2]}; const float l = len3(nn); if (l > 1e-20f) { - normals[v * 3 + 0] = nn[0] / l; - normals[v * 3 + 1] = nn[1] / l; - normals[v * 3 + 2] = nn[2] / l; + nn[0] /= l; nn[1] /= l; nn[2] /= l; + } else { + nn[0] = normals[v * 3 + 0]; + nn[1] = normals[v * 3 + 1]; + nn[2] = normals[v * 3 + 2]; } + std::memcpy(&next[v * 3], nn, sizeof(nn)); } + for (size_t v = 0; v < nv; ++v) + std::memcpy(&normals[v * 3], &next[canon[v] * 3], + sizeof(float) * 3); } return normals; } diff --git a/website/src/hooks/useQtmeshActionRef.js b/website/src/hooks/useQtmeshActionRef.js index 2a6f1c4d3..811d52094 100644 --- a/website/src/hooks/useQtmeshActionRef.js +++ b/website/src/hooks/useQtmeshActionRef.js @@ -1,7 +1,7 @@ import { useEffect, useState } from 'react'; const QTMESH_RELEASES_LATEST_API = 'https://api.github.com/repos/fernandotonon/QtMeshEditor/releases/latest'; -const QTMESH_ACTION_REF_FALLBACK = 'fernandotonon/QtMeshEditor@3.35.0'; +const QTMESH_ACTION_REF_FALLBACK = 'fernandotonon/QtMeshEditor@3.36.0'; const CACHE_KEY = 'qtmesh.actionRef.cache.v1'; const CACHE_TTL_MS = 6 * 60 * 60 * 1000;