From 2296a4fe0fb95060605e8909743dcc1b88f421a0 Mon Sep 17 00:00:00 2001 From: Adam Eivy Date: Thu, 3 Sep 2026 04:39:18 +0000 Subject: [PATCH] feat: export a generated 3D model as USDZ so it opens in AR on your phone (#5756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3D page could only ever spin a generated mesh inside a browser canvas. It now offers "Export for AR (.usdz)", which converts the model you are looking at into the one format that drops it into your actual room at real scale — tap and it is on your floor, no app install and no upload to anyone else. On a desktop the same panel shows a QR code pointing at the install's own URL, so a model generated at your desk reaches your phone by scanning it. The conversion runs in the browser via three's USDZExporter, which is already in the installed three.js — no new dependency, no USD toolchain, and no re-decoding a GLB the viewer has already parsed. The bytes are persisted as a sibling artifact of the record rather than handed back as a blob URL (AR Quick Look does not reliably open a blob, and a blob does not survive a reload), so revisiting a model reuses the stored export instead of re-converting it. - POST/GET /api/image-to-3d/models/:id/usdz store and re-serve the artifact. The GET is served `inline` as `model/vnd.usdz+zip` — Safari will not engage AR Quick Look without that exact header pair — via streamAttachment, which gains an opt-in `disposition` for it. - Uploaded bytes are validated as untrusted input: ready record, non-empty, under a 64 MB cap, and actually a zip archive. - A successful re-render clears the stored export and its record pointer: the file would describe geometry the render just replaced. A FAILED render leaves it alone, since model.glb is untouched and the export still matches. - The AR affordance is feature-detected, never user-agent sniffed. Only Safari on iOS/iPadOS gets the `` handoff; everywhere else the panel offers a plain download plus the QR bridge, so no browser is shown a "View in AR" button that cannot do anything. - Meshes above the AR triangle budget still export, but say so and name the count, pointing at a lower Quality tier rather than shipping a file that opens to an empty room. Claude-Session: https://claude.ai/code/session_01Uz59AsJawa1Djqm2T9Fb8t --- client/public/ar-quick-look.svg | 10 + client/src/components/media/ArExportPanel.jsx | 176 ++++++++++++++++++ .../components/media/ArExportPanel.test.jsx | 147 +++++++++++++++ client/src/components/media/GlbViewer.jsx | 17 +- .../src/components/media/GlbViewer.test.jsx | 15 ++ client/src/lib/README.md | 1 + client/src/lib/index.js | 1 + client/src/lib/usdzExport.js | 103 ++++++++++ client/src/lib/usdzExport.test.js | 53 ++++++ client/src/pages/Media3DDetail.jsx | 8 + client/src/pages/Media3DDetail.test.jsx | 23 ++- client/src/services/README.md | 2 +- client/src/services/apiImageTo3d.js | 19 ++ server/lib/apiRouteCatalog.generated.json | 20 +- server/lib/streamAttachment.js | 12 +- .../routes/imageTo3d.genericDispatch.test.js | 3 + server/routes/imageTo3d.js | 46 ++++- server/routes/imageTo3d.test.js | 63 +++++++ server/services/imageTo3d/db.js | 5 + server/services/imageTo3d/models.js | 102 +++++++++- server/services/imageTo3d/models.test.js | 96 +++++++++- 21 files changed, 906 insertions(+), 16 deletions(-) create mode 100644 client/public/ar-quick-look.svg create mode 100644 client/src/components/media/ArExportPanel.jsx create mode 100644 client/src/components/media/ArExportPanel.test.jsx create mode 100644 client/src/lib/usdzExport.js create mode 100644 client/src/lib/usdzExport.test.js diff --git a/client/public/ar-quick-look.svg b/client/public/ar-quick-look.svg new file mode 100644 index 0000000000..efc3b2bfd5 --- /dev/null +++ b/client/public/ar-quick-look.svg @@ -0,0 +1,10 @@ + + Augmented reality + + + + + + + + diff --git a/client/src/components/media/ArExportPanel.jsx b/client/src/components/media/ArExportPanel.jsx new file mode 100644 index 0000000000..9c5ce9dbd1 --- /dev/null +++ b/client/src/components/media/ArExportPanel.jsx @@ -0,0 +1,176 @@ +import { useCallback, useMemo, useState } from 'react'; +import { AlertTriangle, Box, Loader2, QrCode, Smartphone } from 'lucide-react'; +import { imageTo3dUsdzUrl, uploadImageTo3dUsdz } from '../../services/api'; +import { + AR_TRIANGLE_BUDGET, + countSceneTriangles, + exportSceneToUsdz, + supportsArQuickLook, +} from '../../lib/usdzExport.js'; +import { generateQrCodeSvg } from '../../lib/qrCode'; +import useMounted from '../../hooks/useMounted'; +import { formatBytes } from '../../utils/formatters'; +import toast from '../ui/Toast'; + +/** + * "Export for AR" on `/3d/:id` — the step that takes a generated mesh off the + * screen and into the user's actual room (#5756). + * + * The conversion happens HERE, in the browser, because the viewer has already + * parsed the GLB and decoded its textures; the server only stores the result. The + * bytes are persisted rather than handed back as a blob URL: AR Quick Look does + * not reliably open a blob, and a blob does not survive a reload — so a record + * exported once is re-served on every later visit instead of re-exported. + * + * The affordance is deliberately NOT the same on every device. Only Safari on + * iOS/iPadOS implements the `` handoff, so that is the only place a + * "View in AR" label is honest; everywhere else this is a plain `.usdz` download + * plus a QR code, which is the bridge from "I generated this at my desk" to "it is + * now on my floor". + */ +export default function ArExportPanel({ record, scene, onRecordChange }) { + const mountedRef = useMounted(); + const [busy, setBusy] = useState(false); + + // Resolved once per mount rather than per render: it is a static browser + // capability, and re-probing it on every keystroke elsewhere on the page is + // pointless DOM work. + const canQuickLook = useMemo(() => supportsArQuickLook(), []); + const usdzUrl = imageTo3dUsdzUrl(record.id); + // AR Quick Look needs a real URL and the QR code needs an absolute one — the + // install's own origin as this browser reached it, which is already the + // Tailscale HTTPS host when the page was opened that way. + const absoluteUsdzUrl = typeof window === 'undefined' + ? usdzUrl + : new URL(usdzUrl, window.location.origin).toString(); + + const exported = Boolean(record.usdzPath); + const canExport = record.status === 'ready' && Boolean(record.assetPath); + + const handleExport = useCallback(async () => { + if (busy || !scene) return; + setBusy(true); + // Yield a frame so the spinner actually paints: `parseAsync` is async but its + // work is CPU-bound and long enough on a textured mesh to freeze a button that + // never got a chance to re-render. + await new Promise((resolve) => { requestAnimationFrame(() => resolve()); }); + + const triangles = countSceneTriangles(scene); + if (triangles > AR_TRIANGLE_BUDGET) { + // Say so rather than shipping a file that opens to a blank room. Not a + // refusal — the export still runs — but the count and the remedy are named, + // because the only real fix is a lighter render, not a lighter export. + toast( + `This mesh is ${triangles.toLocaleString()} triangles — above the ` + + `${AR_TRIANGLE_BUDGET.toLocaleString()} that opens comfortably in AR. ` + + 'Re-render at a lower Quality tier for a lighter AR file.', + { icon: '⚠️' }, + ); + } + + const bytes = await exportSceneToUsdz(scene).catch((err) => { + toast.error(err?.message || 'Could not convert this model to USDZ.'); + return null; + }); + if (!bytes) { + if (mountedRef.current) setBusy(false); + return; + } + + const next = await uploadImageTo3dUsdz(record.id, bytes, { silent: true }).catch((err) => { + toast.error(err?.message || 'Could not save the AR export.'); + return null; + }); + if (!mountedRef.current) return; + setBusy(false); + if (next) { + onRecordChange(next); + toast.success(`AR export ready (${formatBytes(bytes.byteLength)}).`); + } + }, [busy, scene, record.id, onRecordChange, mountedRef]); + + if (!canExport) return null; + + return ( +
+
+
+ +

Augmented reality

+
+ +
+ + {!exported && ( +

+ Converts the mesh you are looking at to USDZ so it can be placed at real scale in + your room — no app install, no upload to anyone else. +

+ )} + + {exported && canQuickLook && ( +
+ {/* AR Quick Look engages off an `` whose ONLY child is an + `` — Safari treats that image as the poster it badges and taps + through. A text node inside the anchor breaks the handoff, so the + label is a sibling and the anchor carries its own accessible name. */} + + View in AR + +
+

View in AR

+

Opens in your room at real scale.

+
+
+ )} + + {exported && !canQuickLook && ( +
+
+ + + AR placement needs Safari on an iPhone or iPad. Scan this from your phone to + open the model in your room, or download the file below. + +
+
+
+ + + Download AR model (.usdz) + +
+
+ )} + + {exported && ( +

+ + Re-rendering this model clears its AR export — the file would no longer match the mesh. +

+ )} +
+ ); +} diff --git a/client/src/components/media/ArExportPanel.test.jsx b/client/src/components/media/ArExportPanel.test.jsx new file mode 100644 index 0000000000..b0353c0a2a --- /dev/null +++ b/client/src/components/media/ArExportPanel.test.jsx @@ -0,0 +1,147 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { Group } from 'three'; +import ArExportPanel from './ArExportPanel'; + +const uploadImageTo3dUsdz = vi.fn(); +vi.mock('../../services/api', () => ({ + imageTo3dUsdzUrl: (id) => `/api/image-to-3d/models/${id}/usdz`, + uploadImageTo3dUsdz: (...a) => uploadImageTo3dUsdz(...a), +})); + +// The real exporter pulls three's USDZ writer and a zip library and walks GPU +// textures — none of which jsdom can run. What this suite covers is the contract +// AROUND it: what gets exported, what gets persisted, and which affordance each +// browser is offered. `supportsArQuickLook` is the switch that decides the last +// one, so it is the only thing stubbed per-test. +const usdz = vi.hoisted(() => ({ quickLook: false, bytes: null, error: null, triangles: 1000 })); +vi.mock('../../lib/usdzExport.js', async (importOriginal) => ({ + ...(await importOriginal()), + supportsArQuickLook: () => usdz.quickLook, + countSceneTriangles: () => usdz.triangles, + exportSceneToUsdz: vi.fn(async () => { + if (usdz.error) throw usdz.error; + return usdz.bytes; + }), +})); +vi.mock('../../lib/qrCode', () => ({ + generateQrCodeSvg: (text) => ``, +})); +const toast = vi.hoisted(() => Object.assign(vi.fn(), { error: vi.fn(), success: vi.fn() })); +vi.mock('../ui/Toast', () => ({ default: toast })); + +const record = (over = {}) => ({ + id: 'image3d-1', + name: 'Example Beacon', + status: 'ready', + assetPath: '/data/image-to-3d/image3d-1/model.glb', + usdzPath: null, + ...over, +}); + +beforeEach(() => { + vi.clearAllMocks(); + usdz.quickLook = false; + usdz.error = null; + usdz.triangles = 1000; + usdz.bytes = new ArrayBuffer(2048); +}); + +describe('ArExportPanel', () => { + // A record with no mesh has nothing to convert — offering the action anyway + // produces a button whose only outcome is a server 409. + it('renders nothing until the record has a rendered mesh', () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + // The scene arrives asynchronously from the viewer; exporting before it lands + // would throw inside the exporter instead of just waiting. + it('disables the export until the viewer hands over the loaded scene', () => { + render(); + expect(screen.getByRole('button', { name: /export for ar/i })).toBeDisabled(); + }); + + it('exports the loaded scene and persists the bytes', async () => { + const onRecordChange = vi.fn(); + const exported = record({ usdzPath: '/data/image-to-3d/image3d-1/model.usdz' }); + uploadImageTo3dUsdz.mockResolvedValue(exported); + render(); + + fireEvent.click(screen.getByRole('button', { name: /export for ar/i })); + await waitFor(() => expect(uploadImageTo3dUsdz).toHaveBeenCalled()); + const [id, bytes, options] = uploadImageTo3dUsdz.mock.calls[0]; + expect(id).toBe('image3d-1'); + expect(bytes).toBe(usdz.bytes); + // The component owns its own error toast, so the request must not raise a + // second one from apiCore. + expect(options).toMatchObject({ silent: true }); + expect(onRecordChange).toHaveBeenCalledWith(exported); + }); + + // A mesh far above the AR budget still exports — but silently handing back a + // file that opens to an empty room is the failure this warning exists to + // prevent, and the count is the only thing the user can act on. + it('warns with the triangle count when the mesh is above the AR budget', async () => { + usdz.triangles = 900_000; + uploadImageTo3dUsdz.mockResolvedValue(record({ usdzPath: '/x.usdz' })); + render(); + + fireEvent.click(screen.getByRole('button', { name: /export for ar/i })); + await waitFor(() => expect(uploadImageTo3dUsdz).toHaveBeenCalled()); + expect(toast).toHaveBeenCalledWith(expect.stringContaining('900,000'), expect.anything()); + }); + + it('surfaces a conversion failure without persisting anything', async () => { + usdz.error = new Error('USDZ does not support negative scales'); + render(); + + fireEvent.click(screen.getByRole('button', { name: /export for ar/i })); + await waitFor(() => expect(toast.error).toHaveBeenCalledWith(expect.stringContaining('negative scales'))); + expect(uploadImageTo3dUsdz).not.toHaveBeenCalled(); + }); + + // The stored artifact is the whole reason the bytes are persisted rather than + // handed back as a blob: revisiting the record must reuse it, never silently + // re-run a multi-second conversion. + it('reuses a stored export instead of re-exporting on load', async () => { + render( + , + ); + expect(uploadImageTo3dUsdz).not.toHaveBeenCalled(); + expect(screen.getByRole('button', { name: /re-export for ar/i })).toBeInTheDocument(); + }); + + // Labelling a desktop button "View in AR" is a promise the browser cannot keep, + // so the affordance has to differ — a QR bridge to the phone plus a plain + // download, and NO AR anchor. + it('offers a QR bridge and a download where AR Quick Look is unavailable', () => { + render( + , + ); + expect(screen.queryByRole('link', { name: /view in ar/i })).toBeNull(); + expect(screen.getByRole('link', { name: /download ar model/i })) + .toHaveAttribute('href', '/api/image-to-3d/models/image3d-1/usdz'); + // Absolute, because the QR is scanned by a DIFFERENT device — a relative path + // would resolve against whatever the phone happens to have open. + expect(screen.getByLabelText('QR code linking to the AR model').innerHTML) + .toContain('http://localhost:3000/api/image-to-3d/models/image3d-1/usdz'); + }); + + it('renders the rel="ar" anchor on a browser that implements AR Quick Look', () => { + usdz.quickLook = true; + render( + , + ); + const anchor = screen.getByRole('link', { name: 'View in AR' }); + expect(anchor).toHaveAttribute('rel', 'ar'); + expect(anchor).toHaveAttribute('href', '/api/image-to-3d/models/image3d-1/usdz'); + // Safari only engages the handoff when the anchor's only child is an ; + // a text node inside it silently turns this back into a download. + expect(anchor.children).toHaveLength(1); + expect(anchor.firstElementChild.tagName).toBe('IMG'); + expect(screen.queryByLabelText('QR code linking to the AR model')).toBeNull(); + }); +}); diff --git a/client/src/components/media/GlbViewer.jsx b/client/src/components/media/GlbViewer.jsx index 89ba94f4d3..3f55205f0a 100644 --- a/client/src/components/media/GlbViewer.jsx +++ b/client/src/components/media/GlbViewer.jsx @@ -54,7 +54,7 @@ export function cloneGlbSceneWithOpaqueMaterials(scene) { return clone; } -function GlbModel({ src, forceOpaque }) { +function GlbModel({ src, forceOpaque, onSceneLoaded }) { // `useGLTF` keys drei's global cache on the URL, so a new generation (a new // `src`) parses fresh while revisiting the same mesh reuses the cache — no // manual cache-clear needed (clearing on unmount would force a full multi-MB @@ -74,6 +74,15 @@ function GlbModel({ src, forceOpaque }) { }); }; }, [forceOpaque, renderedScene]); + // Hand the DISPLAYED graph (post force-opaque clone) to the parent, so a + // consumer that re-serializes it — the AR/USDZ export — ships what the user is + // actually looking at rather than a differently-materialed original. Cleared on + // unmount/src change because the effect above disposes this clone's materials: + // a retained handle would then serialize a scene whose textures are gone. + useEffect(() => { + onSceneLoaded?.(renderedScene); + return () => onSceneLoaded?.(null); + }, [onSceneLoaded, renderedScene]); return ; } @@ -163,6 +172,10 @@ export default function GlbViewer({ className = '', forceOpaque = false, initialBackground = DEFAULT_BACKGROUND, + // Called with the loaded three.js object graph once the GLB parses, and with + // `null` when it unloads. Must be referentially stable (a `useState` setter or a + // `useCallback`) — an inline arrow re-fires the effect on every render. + onSceneLoaded, }) { const backgroundInputId = useId(); const controlsPanelId = useId(); @@ -270,7 +283,7 @@ export default function GlbViewer({ - + diff --git a/client/src/components/media/GlbViewer.test.jsx b/client/src/components/media/GlbViewer.test.jsx index db8680d8a2..360e05d770 100644 --- a/client/src/components/media/GlbViewer.test.jsx +++ b/client/src/components/media/GlbViewer.test.jsx @@ -342,6 +342,21 @@ describe('GlbViewer HDRI failures', () => { expect(logged).toHaveBeenCalledWith(expect.stringContaining('💥 React Error'), expect.anything()); }); + // The AR/USDZ export re-serializes the graph the viewer loaded, so the handle + // has to reach the parent — and has to be RETRACTED when the mesh unloads, + // because the force-opaque cleanup disposes that clone's materials. A retained + // handle would export a scene whose textures are gone. + it('hands the loaded scene to onSceneLoaded and clears it on unmount', () => { + const onSceneLoaded = vi.fn(); + const { unmount } = render( + , + ); + expect(onSceneLoaded).toHaveBeenCalledWith(expect.any(Object)); + onSceneLoaded.mockClear(); + unmount(); + expect(onSceneLoaded).toHaveBeenCalledWith(null); + }); + // The other half of the split: the mesh's own failure must still surface. it('still shows the failure panel when the MESH is what failed', () => { gltf.error = new Error('Could not load /data/image-to-3d/abc/model.glb: 404 Not Found'); diff --git a/client/src/lib/README.md b/client/src/lib/README.md index 96028b85a0..dec2f80b41 100644 --- a/client/src/lib/README.md +++ b/client/src/lib/README.md @@ -219,3 +219,4 @@ grep -i "what you want to do" client/src/lib/README.md | `universeMarkdownFilename.cases.js` | Shared client/server filename contract cases used to keep the browser download name and server attachment name in lockstep. | | `qrCode.js` | Deterministic SVG QR code generator for scoped mobile session join links (#5383). | | `riggingReasons.js` | Client mirror of the character-rigging readiness reason labels in `server/services/rigging/readiness.js`: `RIGGING_UNAVAILABLE_REASONS` (reason code -> user-facing label), `RIGGING_REASON_FALLBACK`, and `riggingReasonLabel(code, fallback?)`. Render these anywhere a `GET /api/rigging/readiness` reason is shown; never re-declare the codes. Parity + code coverage enforced by `server/services/rigging/unavailableReasons.parity.test.js`. | +| `usdzExport.js` | AR Quick Look (USDZ) export for the 3D viewer (#5756): `exportSceneToUsdz(scene, { maxTextureSize })` lazy-loads three's `USDZExporter` and serializes the already-decoded scene to bytes, `countSceneTriangles(object3d)` measures an object graph against `AR_TRIANGLE_BUDGET`, and `supportsArQuickLook()` feature-detects the `` handoff (Safari on iOS/iPadOS only — never sniff the user agent) so a desktop is offered a download instead of a button that does nothing. `AR_MAX_TEXTURE_SIZE` bounds the file: USDZ stores every texel raw, with no Draco/meshopt equivalent. | diff --git a/client/src/lib/index.js b/client/src/lib/index.js index 161b98a5ee..7d530516c9 100644 --- a/client/src/lib/index.js +++ b/client/src/lib/index.js @@ -188,3 +188,4 @@ export * from './wrSceneCursor.js'; export * from './writingGuide.js'; export * from './qrCode.js'; export * from './riggingReasons.js'; +export * from './usdzExport.js'; diff --git a/client/src/lib/usdzExport.js b/client/src/lib/usdzExport.js new file mode 100644 index 0000000000..36377d7390 --- /dev/null +++ b/client/src/lib/usdzExport.js @@ -0,0 +1,103 @@ +/** + * USDZ (AR Quick Look) export helpers for the 3D viewer (#5756). + * + * The conversion runs in the BROWSER: by the time the user can ask for it, the + * viewer has already parsed the GLB and decoded every texture, so three's + * `USDZExporter` only has to re-serialize what is in memory. PortOS ships no USD + * toolchain, and a server-side converter would have to redo all of that decode + * work in Node to produce a file the page already holds. + * + * Pure and React-free on purpose — the panel that calls this owns the UI state. + */ + +/** + * Triangles above which an AR export is worth warning about. + * + * Not a hard refusal: AR Quick Look has no published limit, it degrades (long + * load, then a blank room) rather than erroring, and the threshold depends on the + * device. The viewer-grade GLB the exporter reads is already the DECIMATED mesh — + * the million-face `model.obj` sidecar is never loaded here — so a record over + * this budget is one whose Quality tier was pushed high, and the honest remedy is + * to say so and name the count rather than to silently ship a file the user's + * phone will choke on. + */ +export const AR_TRIANGLE_BUDGET = 500_000; + +/** + * Texture edge the exporter downsamples to. + * + * USDZ has no Draco/meshopt equivalent — every vertex and every texel is stored + * raw — so textures, not geometry, are what make an export unopenable on a phone. + * 1024 is three's own default and keeps a typical export in single-digit + * megabytes. + */ +export const AR_MAX_TEXTURE_SIZE = 1024; + +/** + * Count the triangles a three.js object graph would render. + * + * Indexed and non-indexed geometry differ in where the count lives (`index` vs + * the position attribute), and a `Points`/`Line` child has neither — so this reads + * the geometry rather than assuming a `Mesh`-only tree. + */ +export function countSceneTriangles(object3d) { + let triangles = 0; + object3d?.traverse?.((child) => { + const geometry = child.isMesh ? child.geometry : null; + if (!geometry) return; + const vertices = geometry.index ? geometry.index.count : (geometry.attributes?.position?.count ?? 0); + triangles += Math.floor(vertices / 3); + }); + return triangles; +} + +/** + * Whether THIS browser can hand a `.usdz` to AR Quick Look. + * + * Feature-detected via `relList.supports('ar')` rather than sniffed from the user + * agent: only Safari on iOS/iPadOS/visionOS implements the `rel="ar"` handoff, and + * every other browser — including Chrome on the very same iPhone — must be offered + * a plain download instead. Labelling a desktop button "View in AR" when nothing + * will happen is the failure mode this exists to prevent. + * + * The try/catch is load-bearing, not defensive habit: `DOMTokenList.supports()` is + * SPECIFIED to throw `TypeError` when the attribute has no supported-tokens list, + * which is what every engine without AR Quick Look does for `rel`. Callers run this + * during render, so an uncaught throw here unmounts the whole route rather than + * hiding one button. + */ +export function supportsArQuickLook() { + if (typeof document === 'undefined') return false; + const relList = document.createElement('a').relList; + try { + return Boolean(relList?.supports?.('ar')); + } catch { + return false; + } +} + +/** + * Serialize a loaded scene to USDZ bytes. + * + * The exporter module is imported lazily: it is ~35 KB plus its zip library, and + * nothing outside this one action needs it. `vite.chunkGroups.js` folds all of + * `three` into the `vendor-three` chunk, which the 3D page already loads for the + * viewer — so the deferral keeps this off every other route rather than splitting + * it out of that chunk. Note the scene handed in is the drei cache's own object + * graph — it is read, never mutated or disposed. + * + * @param {import('three').Object3D} scene + * @param {{ maxTextureSize?: number }} [options] + * @returns {Promise} + */ +export async function exportSceneToUsdz(scene, { maxTextureSize = AR_MAX_TEXTURE_SIZE } = {}) { + if (!scene) throw new Error('The 3D model is still loading — try again in a moment.'); + const { USDZExporter } = await import('three/examples/jsm/exporters/USDZExporter.js'); + const exporter = new USDZExporter(); + return exporter.parseAsync(scene, { + maxTextureSize, + // Horizontal plane anchoring: these models are objects, so AR Quick Look + // should drop them on the floor/table rather than a wall. + ar: { anchoring: { type: 'plane' }, planeAnchoring: { alignment: 'horizontal' } }, + }); +} diff --git a/client/src/lib/usdzExport.test.js b/client/src/lib/usdzExport.test.js new file mode 100644 index 0000000000..56c79662c5 --- /dev/null +++ b/client/src/lib/usdzExport.test.js @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { BufferAttribute, BufferGeometry, Group, Mesh, MeshStandardMaterial, Points } from 'three'; +import { countSceneTriangles, supportsArQuickLook } from './usdzExport.js'; + +const geometry = (vertexCount, { indexed = false } = {}) => { + const geo = new BufferGeometry(); + geo.setAttribute('position', new BufferAttribute(new Float32Array(vertexCount * 3), 3)); + if (indexed) geo.setIndex(new BufferAttribute(new Uint16Array(vertexCount), 1)); + return geo; +}; + +describe('countSceneTriangles', () => { + // Indexed and non-indexed geometry keep the triangle count in DIFFERENT places + // (`index.count` vs the position attribute), and a GLB from the render pipeline + // may be either — reading only one silently reports the wrong budget for half of + // every user's models, which is exactly the warning this number drives. + it('reads the count from the index when the geometry is indexed', () => { + const group = new Group(); + group.add(new Mesh(geometry(12, { indexed: true }), new MeshStandardMaterial())); + expect(countSceneTriangles(group)).toBe(4); + }); + + it('falls back to the position attribute for non-indexed geometry', () => { + const group = new Group(); + group.add(new Mesh(geometry(9), new MeshStandardMaterial())); + expect(countSceneTriangles(group)).toBe(3); + }); + + // A GLB can carry a Points/Line child (and always carries plain Object3D + // groups). Those have no triangles, and treating their vertex count as + // triangles would inflate the budget check into a false warning. + it('ignores non-mesh children and sums the meshes', () => { + const group = new Group(); + group.add(new Group()); + group.add(new Points(geometry(300))); + group.add(new Mesh(geometry(6), new MeshStandardMaterial())); + group.add(new Mesh(geometry(30, { indexed: true }), new MeshStandardMaterial())); + expect(countSceneTriangles(group)).toBe(12); + }); + + it('reports zero rather than throwing for a missing scene', () => { + expect(countSceneTriangles(null)).toBe(0); + }); +}); + +describe('supportsArQuickLook', () => { + // jsdom's relList reports no `ar` support, which is the correct answer for + // every non-Safari browser — the panel must offer a download and a QR code + // there instead of a "View in AR" button that does nothing. + it('is false where the rel="ar" handoff is unimplemented', () => { + expect(supportsArQuickLook()).toBe(false); + }); +}); diff --git a/client/src/pages/Media3DDetail.jsx b/client/src/pages/Media3DDetail.jsx index 5bd41ed3e1..47848a45b1 100644 --- a/client/src/pages/Media3DDetail.jsx +++ b/client/src/pages/Media3DDetail.jsx @@ -10,6 +10,7 @@ import MediaImage from '../components/MediaImage'; import InlineConfirmRow from '../components/ui/InlineConfirmRow'; import ImageTo3dRenderOptions from '../components/media/ImageTo3dRenderOptions'; import RigPanel from '../components/media/RigPanel'; +import ArExportPanel from '../components/media/ArExportPanel'; import { fieldsFromRun, renderOptionsBody, runWantsTransparency, SUBJECT_SCALE_DEFAULT, } from '../lib/imageTo3dRenderOptions'; @@ -33,6 +34,10 @@ export default function Media3DDetail() { const [notFound, setNotFound] = useState(false); const [busy, setBusy] = useState(false); const [confirmingDelete, setConfirmingDelete] = useState(false); + // The three.js graph GlbViewer has loaded, handed up so the AR panel can + // re-serialize the very scene on screen to USDZ. `null` until the GLB parses + // (and again when it unloads) — the export button stays disabled until then. + const [loadedScene, setLoadedScene] = useState(null); // Per-run knobs, seeded from the latest run once per id (NOT on every poll // tick — that would clobber in-progress edits). Seed stays blank by design: // see fieldsFromRun. @@ -254,6 +259,7 @@ export default function Media3DDetail() { src={meshSrc} downloadHref={imageTo3dAssetUrl(record.id)} forceOpaque={!renderedTransparent} + onSceneLoaded={setLoadedScene} /> {/* The viewer loads the decimated GLB because that is what a browser can render; the decoder's full mesh is an order of magnitude @@ -280,6 +286,8 @@ export default function Media3DDetail() { + + ); diff --git a/client/src/pages/Media3DDetail.test.jsx b/client/src/pages/Media3DDetail.test.jsx index 5ea37aee4f..d03d544cf1 100644 --- a/client/src/pages/Media3DDetail.test.jsx +++ b/client/src/pages/Media3DDetail.test.jsx @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { useEffect } from 'react'; import { render, screen, fireEvent, waitFor, within } from '@testing-library/react'; import { MemoryRouter, Routes, Route } from 'react-router'; import Media3DDetail from './Media3DDetail'; @@ -12,13 +13,24 @@ vi.mock('../services/api', () => ({ deleteImageTo3dModel: (...a) => deleteImageTo3dModel(...a), imageTo3dAssetUrl: (id) => `/api/image-to-3d/models/${id}/asset`, imageTo3dFullMeshUrl: (id) => `/api/image-to-3d/models/${id}/full-mesh`, + imageTo3dUsdzUrl: (id) => `/api/image-to-3d/models/${id}/usdz`, })); -// GlbViewer wraps a WebGL canvas jsdom can't render — stub to a marker echoing src. +// GlbViewer wraps a WebGL canvas jsdom can't render — stub to a marker echoing +// src. It also hands the loaded three.js graph up via `onSceneLoaded`, so the +// stub fires that with a marker: the AR export button is disabled until it +// arrives, and dropping that prop is an invisible regression otherwise. vi.mock('../components/media/GlbViewer', () => ({ - default: ({ src, forceOpaque }) => ( -
{src}
- ), + default: ({ src, forceOpaque, onSceneLoaded }) => { + useEffect(() => { onSceneLoaded?.({ marker: 'loaded-scene' }); }, [onSceneLoaded]); + return
{src}
; + }, +})); +// The AR panel owns its own export/upload flow (covered by ArExportPanel.test.jsx); +// stubbing it keeps this suite about the page — but it echoes whether the scene +// reached it, which is the page's half of the contract. +vi.mock('../components/media/ArExportPanel', () => ({ + default: ({ scene }) =>
, })); vi.mock('../components/MediaImage', () => ({ default: ({ alt, src }) => {alt} })); // The rig panel owns its own readiness fetch + feature gate (covered by @@ -61,6 +73,9 @@ describe('Media3DDetail', () => { ); expect(screen.getByTestId('glb-viewer')).toHaveAttribute('data-force-opaque', 'true'); expect(screen.getByAltText('Source image')).toBeInTheDocument(); + // The viewer's loaded scene has to reach the AR panel or its export button + // stays permanently disabled — a prop chain no other assertion touches. + expect(await screen.findByTestId('ar-export-panel')).toHaveAttribute('data-has-scene', 'true'); }); it('surfaces the render error for a failed record', async () => { diff --git a/client/src/services/README.md b/client/src/services/README.md index 84de8aefd5..790c846f6f 100644 --- a/client/src/services/README.md +++ b/client/src/services/README.md @@ -109,7 +109,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire | `apiSprites.js` | Sprite Manager records, asset library, production-set import (#2895), reference workflow: create/generate/lock (#2896), directional walk and per-track generation/approval, animation-type definition CRUD (#3153), trim/postprocess, and per-run source-frame listing for the Loop Trimmer's re-derive (#2980), and animation render-provider readiness (#4876). | | `apiShell.js` | Shell sessions over HTTP: hand a photo (plus a message) to the agent TUI running in a session. Keystrokes/output stay on the `shell:*` socket protocol. | | `apiThreejsModels.js` | Procedural Three.js model workspaces: gallery-image generation, refinement, source export, deletion, and the subject-family checklist options. | -| `apiImageTo3d.js` | Image-to-3D (`/3d`): selectable targets (TRELLIS.2) with host availability/install status, and per-image model records — create/list/get/generate/delete + GLB asset URL and the full-resolution OBJ download URL. | +| `apiImageTo3d.js` | Image-to-3D (`/3d`): selectable targets (TRELLIS.2) with host availability/install status, and per-image model records — create/list/get/generate/delete + GLB asset URL, the full-resolution OBJ download URL, and the AR Quick Look USDZ upload/download pair. | | `apiPipeline.js` | Pipeline (issues + stages + canon). | | `apiUniverseBuilder.js` | Universe Builder (generate + edit + commit). | | `apiAuthors.js` | Author personas (name, writing style, bio, headshot description/style). | diff --git a/client/src/services/apiImageTo3d.js b/client/src/services/apiImageTo3d.js index 8df31513e3..890b30da85 100644 --- a/client/src/services/apiImageTo3d.js +++ b/client/src/services/apiImageTo3d.js @@ -51,3 +51,22 @@ export const imageTo3dAssetUrl = (id) => // 404 — not a sign the record is broken). The GLB stays the thing the viewer loads. export const imageTo3dFullMeshUrl = (id) => `/api/image-to-3d/models/${encodeURIComponent(id)}/full-mesh`; + +// The stored AR Quick Look artifact, served `inline` as `model/vnd.usdz+zip` — +// the exact header pair Safari requires before it will open an `` +// target in AR. Deliberately NOT the record's static `usdzPath`: that mount +// leaves the content type to mime lookup, and this contract is the feature. +export const imageTo3dUsdzUrl = (id) => + `/api/image-to-3d/models/${encodeURIComponent(id)}/usdz`; + +// Persist a USDZ the VIEWER produced (three's USDZExporter over the scene it has +// already decoded — PortOS ships no USD toolchain). Raw bytes, not JSON: the +// explicit content type overrides apiCore's `application/json` default so the +// server's `express.raw` parser claims the body. +export const uploadImageTo3dUsdz = (id, bytes, options) => + request(`/image-to-3d/models/${encodeURIComponent(id)}/usdz`, { + method: 'POST', + body: bytes, + headers: { 'Content-Type': 'model/vnd.usdz+zip' }, + ...options, + }); diff --git a/server/lib/apiRouteCatalog.generated.json b/server/lib/apiRouteCatalog.generated.json index 0eae864374..d78e584336 100644 --- a/server/lib/apiRouteCatalog.generated.json +++ b/server/lib/apiRouteCatalog.generated.json @@ -7917,6 +7917,22 @@ "server/routes/imageTo3d.js" ] }, + { + "method": "GET", + "path": "/api/image-to-3d/models/:id/usdz", + "mountPath": "/api/image-to-3d", + "sources": [ + "server/routes/imageTo3d.js" + ] + }, + { + "method": "POST", + "path": "/api/image-to-3d/models/:id/usdz", + "mountPath": "/api/image-to-3d", + "sources": [ + "server/routes/imageTo3d.js" + ] + }, { "method": "GET", "path": "/api/image-to-3d/targets", @@ -17440,8 +17456,8 @@ ], "stats": { "mounts": 146, - "operations": 2160, - "declarations": 2168, + "operations": 2162, + "declarations": 2170, "sourceFiles": 229 } } diff --git a/server/lib/streamAttachment.js b/server/lib/streamAttachment.js index 05c4ae4b7e..4b1a256e04 100644 --- a/server/lib/streamAttachment.js +++ b/server/lib/streamAttachment.js @@ -20,7 +20,7 @@ function teardown(stream) { * after the readiness check) get the JSON error envelope; mid-stream failures * tear the socket down, since sendErrorResponse no-ops once headers are sent. * - * Call sites: routes/imageTo3d.js (GLB + full-mesh), routes/backup.js + * Call sites: routes/imageTo3d.js (GLB + full-mesh + USDZ), routes/backup.js * (snapshot tarball). * * @param {import('express').Response} res @@ -32,8 +32,14 @@ function teardown(stream) { * @param {import('./errorHandler.js').ServerError} opts.failure - the error * returned when the stream fails before any bytes are written. * @param {string} opts.label - short subject for the warning log. + * @param {'attachment'|'inline'} [opts.disposition] - defaults to `attachment`. + * Pass `inline` for a format whose whole point is that the OS handler opens it + * in place rather than landing in Downloads (USDZ / AR Quick Look): Safari will + * not engage Quick Look on an `attachment` response. */ -export function streamAttachment(res, stream, { filename, contentType, failure, label }) { +export function streamAttachment(res, stream, { + filename, contentType, failure, label, disposition = 'attachment', +}) { // The route awaited settings/stat before getting here, so the client may have // already gone — in which case res's 'close' fired before the listener below // was installed and nothing would ever tear the stream down. @@ -47,7 +53,7 @@ export function streamAttachment(res, stream, { filename, contentType, failure, // slugged it: a quote or newline in a record-derived name would otherwise // break out of the header value. const safeName = String(filename).replace(/[^\w.\-]+/g, '_') || 'download'; - res.set('Content-Disposition', `attachment; filename="${safeName}"`); + res.set('Content-Disposition', `${disposition}; filename="${safeName}"`); // Attachments are never meant to be sniffed into an executable type. res.set('X-Content-Type-Options', 'nosniff'); diff --git a/server/routes/imageTo3d.genericDispatch.test.js b/server/routes/imageTo3d.genericDispatch.test.js index 3ba637e006..a16c123a83 100644 --- a/server/routes/imageTo3d.genericDispatch.test.js +++ b/server/routes/imageTo3d.genericDispatch.test.js @@ -54,6 +54,9 @@ vi.mock('../services/imageTo3d/models.js', () => ({ startGeneration: vi.fn(), deleteModel: vi.fn(), getModelAsset: vi.fn(), + // Read at module scope by the USDZ body parser, so — unlike the handler-only + // exports above — omitting it fails the IMPORT, not a test. + USDZ_MAX_BYTES: 64 * 1024 * 1024, })); import { hfChildEnv } from '../services/hfToken.js'; diff --git a/server/routes/imageTo3d.js b/server/routes/imageTo3d.js index 4dbc573f3f..159ff86b03 100644 --- a/server/routes/imageTo3d.js +++ b/server/routes/imageTo3d.js @@ -1,4 +1,4 @@ -import { Router } from 'express'; +import { Router, raw } from 'express'; import { createReadStream } from 'node:fs'; import { z } from 'zod'; import { asyncHandler, ServerError } from '../lib/errorHandler.js'; @@ -14,6 +14,9 @@ import { deleteModel, getModelAsset, getModelFullMesh, + getModelUsdz, + saveModelUsdz, + USDZ_MAX_BYTES, } from '../services/imageTo3d/models.js'; import { RENDER_STEPS_MIN, RENDER_STEPS_MAX, RENDER_SEED_MAX, DETAIL_TIERS, ALPHA_MODES, @@ -309,6 +312,47 @@ router.get('/models/:id/full-mesh', asyncHandler(async (req, res) => { }); })); +// ── AR Quick Look (USDZ) ────────────────────────────────────────────────── +// The conversion runs in the VIEWER (three's USDZExporter over the scene it has +// already parsed and decoded), not on the server — PortOS ships no USD toolchain +// and would otherwise have to re-decode the GLB and its textures in Node to +// produce a file the browser already holds in memory. The server's job is to +// persist the result: a blob URL is not reliably openable by AR Quick Look and +// does not survive a reload, so the bytes are stored as a sibling artifact and +// re-served on every later visit instead of being re-exported. + +/** + * A raw USDZ body. `express.json()` is mounted app-wide but only claims + * `application/json`, so these content types reach the route unparsed. The limit + * is enforced here (a 413 from the parser) AND in `saveModelUsdz` — the parser + * guards memory before the body is buffered, the service guards the invariant for + * any other caller. + */ +const usdzBody = raw({ + type: ['model/vnd.usdz+zip', 'application/octet-stream'], + limit: USDZ_MAX_BYTES, +}); + +router.post('/models/:id/usdz', usdzBody, asyncHandler(async (req, res) => { + // Body-shape validation is byte-level (non-empty, under cap, zip magic), not a + // Zod object schema — there is no JSON here to describe. + const model = await saveModelUsdz(req.params.id, Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0)); + res.status(201).json(withRenderSupport(model)); +})); + +// Served `inline`, unlike the GLB/OBJ downloads: AR Quick Look will not engage on +// an attachment response, and it needs the exact `model/vnd.usdz+zip` type. +router.get('/models/:id/usdz', asyncHandler(async (req, res) => { + const { path, filename } = await getModelUsdz(req.params.id); + streamAttachment(res, createReadStream(path), { + filename, + contentType: 'model/vnd.usdz+zip', + disposition: 'inline', + failure: new ServerError('USDZ file not found', { status: 404, code: 'ASSET_MISSING' }), + label: 'Image-to-3D AR export', + }); +})); + router.get('/models/:id', asyncHandler(async (req, res) => { const model = await getModel(req.params.id); if (!model) throw new ServerError('Image-to-3D model not found', { status: 404, code: 'NOT_FOUND' }); diff --git a/server/routes/imageTo3d.test.js b/server/routes/imageTo3d.test.js index 65886cc627..bc277bd7d0 100644 --- a/server/routes/imageTo3d.test.js +++ b/server/routes/imageTo3d.test.js @@ -78,6 +78,9 @@ vi.mock('../services/imageTo3d/models.js', () => ({ deleteModel: vi.fn(), getModelAsset: vi.fn(), getModelFullMesh: vi.fn(), + getModelUsdz: vi.fn(), + saveModelUsdz: vi.fn(), + USDZ_MAX_BYTES: 64 * 1024 * 1024, })); import * as targets from '../services/imageTo3d/targets.js'; @@ -565,6 +568,66 @@ describe('image-to-3d model records', () => { expect(res.body?.error?.code || res.body?.code).toBe('FULL_MESH_MISSING'); }); + // ── AR Quick Look (USDZ) ──────────────────────────────────────────────── + // The bytes are produced in the browser, so the route's whole job is to accept a + // raw body past the app-wide JSON parser, persist it, and re-serve it with the + // exact content type + disposition AR Quick Look requires. + + // A minimal stored-zip header — USDZ is an uncompressed zip, and the service + // gates on that magic rather than trusting the request's content type. + const ZIP_BYTES = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x00, 0x00]); + + it('POST /models/:id/usdz accepts a raw USDZ body past the JSON parser', async () => { + models.saveModelUsdz.mockResolvedValue({ + id: 'image3d-1', status: 'ready', usdzPath: '/data/image-to-3d/image3d-1/model.usdz', + }); + const res = await request(makeApp()) + .post('/api/image-to-3d/models/image3d-1/usdz') + .set('Content-Type', 'model/vnd.usdz+zip') + .send(ZIP_BYTES); + expect(res.status).toBe(201); + expect(res.body.usdzPath).toBe('/data/image-to-3d/image3d-1/model.usdz'); + const [id, body] = models.saveModelUsdz.mock.calls.at(-1); + expect(id).toBe('image3d-1'); + expect(Buffer.from(body).equals(ZIP_BYTES)).toBe(true); + }); + + it('POST /models/:id/usdz surfaces the service refusal for a non-USDZ payload', async () => { + const { ServerError } = await import('../lib/errorHandler.js'); + models.saveModelUsdz.mockRejectedValue( + new ServerError('Payload is not a USDZ archive', { status: 400, code: 'USDZ_INVALID' }), + ); + const res = await request(makeApp()) + .post('/api/image-to-3d/models/image3d-1/usdz') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from('not a zip')); + expect(res.status).toBe(400); + expect(res.body?.error?.code || res.body?.code).toBe('USDZ_INVALID'); + }); + + it('GET /models/:id/usdz serves inline with the AR Quick Look content type', async () => { + // `inline`, not `attachment`: Safari will not engage AR Quick Look on an + // attachment response, so this header pair IS the feature. + const tmp = join(tmpdir(), `it-usdz-${process.pid}.usdz`); + await writeFile(tmp, ZIP_BYTES); + models.getModelUsdz.mockResolvedValue({ path: tmp, filename: 'beacon.usdz' }); + const res = await request(makeApp()).get('/api/image-to-3d/models/image3d-1/usdz'); + expect(res.status).toBe(200); + expect(res.headers['content-type']).toMatch(/model\/vnd\.usdz\+zip/); + expect(res.headers['content-disposition']).toMatch(/^inline; filename="beacon\.usdz"$/); + await rm(tmp, { force: true }); + }); + + it('GET /models/:id/usdz 404s when the model was never exported for AR', async () => { + const { ServerError } = await import('../lib/errorHandler.js'); + models.getModelUsdz.mockRejectedValue( + new ServerError('not exported', { status: 404, code: 'USDZ_MISSING' }), + ); + const res = await request(makeApp()).get('/api/image-to-3d/models/image3d-1/usdz'); + expect(res.status).toBe(404); + expect(res.body?.error?.code || res.body?.code).toBe('USDZ_MISSING'); + }); + it('routes /full-mesh to its own handler with the record id', async () => { // Deliberately NOT claiming this proves route ordering: Express's `:id` matches a // single path segment, so `/models/x/full-mesh` can never match `/models/:id` diff --git a/server/services/imageTo3d/db.js b/server/services/imageTo3d/db.js index a1775bb2f7..b05eb9102c 100644 --- a/server/services/imageTo3d/db.js +++ b/server/services/imageTo3d/db.js @@ -72,6 +72,11 @@ export async function createModel(input) { // predates rigging has records with the key ABSENT, so readers must treat absent and // `null` the same and never assume the field exists. rig: null, + // The stored AR Quick Look export's served path, or `null` when the model has + // never been exported for AR (#5756). Same absent-vs-null contract as `rig`: + // every record created before this feature has the key MISSING, so read it as + // `record.usdzPath` truthiness and never assume it exists. + usdzPath: null, error: null, generationOperationId: null, runs: [], diff --git a/server/services/imageTo3d/models.js b/server/services/imageTo3d/models.js index 337fc4d30f..40c01eb3df 100644 --- a/server/services/imageTo3d/models.js +++ b/server/services/imageTo3d/models.js @@ -17,7 +17,7 @@ import { randomUUID } from 'crypto'; import { join } from 'node:path'; -import { rm, access } from 'node:fs/promises'; +import { rm, access, writeFile } from 'node:fs/promises'; import { ServerError } from '../../lib/errorHandler.js'; import { PATHS, resolveGalleryImage, ensureDir } from '../../lib/fileUtils.js'; import { claimHeavyLocalJob } from '../../lib/heavyJobClaim.js'; @@ -69,6 +69,28 @@ const fullMeshDiskPath = (id) => join(recordDir(id), 'model.obj'); // strand an orphan beside the new one in every existing record directory. const preparedSourcePath = (id) => join(recordDir(id), 'source-keyed.png'); +/** + * The value stored on `record.usdzPath` once an AR export exists — the static + * `/data` mount's path for it, and the record's "has been exported" marker. + * `null` (or, on records predating this feature, ABSENT — readers must treat the + * two the same, exactly like `rig`) means nobody has exported it yet. + * + * The 3D page still points its AR anchor at `GET /api/image-to-3d/models/:id/usdz` + * rather than at this path: AR Quick Look needs `model/vnd.usdz+zip` served + * `inline`, and only that route guarantees the pair. + */ +const usdzUrl = (id) => `/data/image-to-3d/${id}/model.usdz`; +/** + * The AR Quick Look artifact, exported by the viewer from the SAME `model.glb` + * the 3D page loads and stored beside it. + * + * Deliberately NOT in backup's DEFAULT_EXCLUDES: it is a few megabytes (the + * viewer-grade GLB with 1024px textures, not the gigabyte `model.obj` sidecar), + * and re-deriving it needs a browser session with the model open — so it is + * cheaper to keep than to reproduce, exactly like the published `rig/` pair. + */ +const usdzDiskPath = (id) => join(recordDir(id), 'model.usdz'); + /** * Remove a record's render directory (the exported GLB + its folder). Used to * clean the orphaned mesh a killed/deleted render may have left on disk. `force` @@ -250,6 +272,10 @@ async function executeRender({ id, operationId, adapter, sourcePath, caps, optio ...current, status: 'ready', assetPath: assetUrl(id), + // A new mesh invalidates the AR export derived from the OLD one. Cleared on + // success only: a FAILED render leaves model.glb untouched, so its USDZ is + // still a faithful copy of what the viewer shows and must survive. + usdzPath: null, error: null, generationOperationId: null, generatedAt: completedAt, @@ -260,6 +286,8 @@ async function executeRender({ id, operationId, adapter, sourcePath, caps, optio }), }; }, { includeDeleted: true }); + await rm(usdzDiskPath(id), { force: true }) + .catch((err) => console.error(`❌ Image-to-3D stale USDZ cleanup failed for ${id}: ${err.message}`)); console.log(`🧊 Image-to-3D mesh ready: ${id}`); } catch (error) { console.error(`❌ Image-to-3D render failed for ${id}: ${cleanError(error)}`); @@ -449,6 +477,78 @@ export async function getModelFullMesh(id, { exists = pathExists } = {}) { return { path, filename: `${slugifyForFilename(model.name)}-full.obj` }; } +/** + * The largest USDZ the AR export route will accept. + * + * The viewer exports from the SAME viewer-grade GLB the page already rendered, so a + * legitimate payload is single-digit megabytes; the cap exists so a wrong/hostile + * body can't fill the record directory, not to shape a real export. + */ +export const USDZ_MAX_BYTES = 64 * 1024 * 1024; + +/** USDZ is a plain (stored, uncompressed) zip archive — every one starts `PK\x03\x04`. */ +const isZipArchive = (bytes) => bytes.length >= 4 + && bytes[0] === 0x50 && bytes[1] === 0x4b && bytes[2] === 0x03 && bytes[3] === 0x04; + +/** + * Store the viewer's USDZ export for a ready record, beside its GLB. + * + * The bytes come from the CLIENT (three's USDZExporter over the already-loaded + * scene) rather than from a server-side converter, so they are validated here as + * untrusted input: ready record, non-empty, under the cap, and actually a zip. + * Re-exporting simply overwrites — the file is derived, so there is no version to + * preserve. + */ +export async function saveModelUsdz(id, bytes) { + const model = await store.getModel(id); + if (!model) throw new ServerError('Image-to-3D model not found', { status: 404, code: 'NOT_FOUND' }); + if (model.status !== 'ready' || !model.assetPath) { + throw new ServerError('This model has no generated mesh yet', { status: 409, code: 'MODEL_NOT_READY' }); + } + if (!bytes?.length) { + throw new ServerError('USDZ payload is empty', { status: 400, code: 'USDZ_INVALID' }); + } + if (bytes.length > USDZ_MAX_BYTES) { + throw new ServerError( + `USDZ payload exceeds the ${Math.round(USDZ_MAX_BYTES / (1024 * 1024))} MB limit`, + { status: 413, code: 'USDZ_TOO_LARGE' }, + ); + } + if (!isZipArchive(bytes)) { + throw new ServerError('Payload is not a USDZ archive', { status: 400, code: 'USDZ_INVALID' }); + } + await ensureDir(recordDir(id)); + await writeFile(usdzDiskPath(id), bytes); + console.log(`🥽 Image-to-3D stored AR export for ${id} (${bytes.length} bytes)`); + return store.mutateModel(id, (current) => ({ + ...current, + usdzPath: usdzUrl(id), + usdzGeneratedAt: new Date().toISOString(), + })); +} + +/** + * Resolve a record's stored USDZ for download. + * + * Like `getModelFullMesh`, its absence is not an error state of the RECORD — a + * model nobody has exported for AR yet is perfectly healthy — so it probes disk + * rather than trusting `usdzPath` alone. That also covers the reverse skew: a + * record whose file was pruned out from under it still 404s instead of streaming a + * missing path. + */ +export async function getModelUsdz(id, { exists = pathExists } = {}) { + const model = await store.getModel(id); + if (!model) throw new ServerError('Image-to-3D model not found', { status: 404, code: 'NOT_FOUND' }); + const path = usdzDiskPath(id); + if (!await exists(path)) { + throw new ServerError( + 'This model has not been exported for AR yet. Open it in the 3D viewer and export it.', + { status: 404, code: 'USDZ_MISSING' }, + ); + } + return { path, filename: `${slugifyForFilename(model.name)}.usdz` }; +} + export async function recoverInterruptedModels() { const result = await store.recoverInterruptedModels(); if (result.recovered > 0) { diff --git a/server/services/imageTo3d/models.test.js b/server/services/imageTo3d/models.test.js index 1ca63c62ca..f8b42d3f21 100644 --- a/server/services/imageTo3d/models.test.js +++ b/server/services/imageTo3d/models.test.js @@ -4,6 +4,7 @@ import { posixPath } from '../../lib/testHelper.js'; vi.mock('node:fs/promises', async (importOriginal) => ({ ...(await importOriginal()), rm: vi.fn(() => Promise.resolve()), + writeFile: vi.fn(() => Promise.resolve()), })); vi.mock('../../lib/fileUtils.js', () => ({ @@ -65,7 +66,7 @@ vi.mock('./db.js', () => ({ recoverInterruptedModels: vi.fn(), })); -import { rm } from 'node:fs/promises'; +import { rm, writeFile } from 'node:fs/promises'; import { ensureDir } from '../../lib/fileUtils.js'; import { resolveTarget, renderOptionSupportFor } from './targets.js'; import { isTrellis2Installed, runTrellis2Generate } from './trellis2.js'; @@ -73,7 +74,8 @@ import { claimHeavyLocalJob } from '../../lib/heavyJobClaim.js'; import { prepareSourceImage } from './sourceKeying.js'; import * as store from './db.js'; import { - createModel, startGeneration, getModelAsset, getModelFullMesh, recoverInterruptedModels, deleteModel, + createModel, startGeneration, getModelAsset, getModelFullMesh, getModelUsdz, saveModelUsdz, + USDZ_MAX_BYTES, recoverInterruptedModels, deleteModel, } from './models.js'; const draftRecord = () => ({ @@ -165,6 +167,30 @@ describe('image-to-3D model orchestration', () => { expect(current.runs.at(-1)).toMatchObject({ status: 'completed', percent: 100 }); }); + // A new mesh makes the AR export a lie — it describes the geometry the render + // just replaced — so a successful re-render must drop both the record's pointer + // and the file. Cleared on SUCCESS only: a failed render leaves model.glb + // untouched, so its USDZ still matches and must survive. + it('clears a stale AR export when a re-render succeeds', async () => { + let current = { + ...draftRecord(), + status: 'ready', + assetPath: '/data/image-to-3d/image3d-example/model.glb', + usdzPath: '/data/image-to-3d/image3d-example/model.usdz', + }; + store.getModel.mockImplementation(async () => current); + store.mutateModel.mockImplementation(async (_id, mutate) => { + const next = mutate(current); + if (next) current = next; + return current; + }); + + await startGeneration('image3d-example'); + await vi.waitFor(() => expect(current.status).toBe('ready')); + expect(current.usdzPath).toBeNull(); + expect(rm.mock.calls.some(([path]) => posixPath(path).endsWith('/image3d-example/model.usdz'))).toBe(true); + }); + it('marks the record failed when the render throws', async () => { let current = draftRecord(); store.createModel.mockImplementation(async () => current); @@ -332,6 +358,72 @@ describe('image-to-3D model orchestration', () => { .rejects.toMatchObject({ status: 404, code: 'NOT_FOUND' }); }); }); + + // The AR export is produced by the BROWSER and posted back, so these bytes are + // untrusted input rather than something this process generated — the guards + // below are the whole reason the store is a service function and not a bare + // writeFile in the route. + describe('AR (USDZ) artifact', () => { + const ready = () => ({ + ...draftRecord(), + status: 'ready', + name: 'My Beacon', + assetPath: '/data/image-to-3d/image3d-example/model.glb', + }); + const zip = (extra = 0) => Buffer.concat([ + Buffer.from([0x50, 0x4b, 0x03, 0x04]), + Buffer.alloc(extra), + ]); + + it('stores the export beside the GLB and records its served path', async () => { + store.getModel.mockResolvedValueOnce(ready()); + store.mutateModel.mockImplementationOnce(async (_id, mutate) => mutate(ready())); + const next = await saveModelUsdz('image3d-example', zip(64)); + expect(posixPath(writeFile.mock.calls[0][0])) + .toMatch(/image-to-3d\/image3d-example\/model\.usdz$/); + expect(next.usdzPath).toBe('/data/image-to-3d/image3d-example/model.usdz'); + expect(next.usdzGeneratedAt).toEqual(expect.any(String)); + }); + + it('refuses a payload that is not a zip archive', async () => { + // USDZ is a stored zip; anything else would be served to AR Quick Look as a + // valid-looking file that silently fails to open on the device. + store.getModel.mockResolvedValueOnce(ready()); + await expect(saveModelUsdz('image3d-example', Buffer.from('not a usdz'))) + .rejects.toMatchObject({ status: 400, code: 'USDZ_INVALID' }); + expect(writeFile).not.toHaveBeenCalled(); + }); + + it('refuses an empty body and one past the size cap', async () => { + store.getModel.mockResolvedValue(ready()); + await expect(saveModelUsdz('image3d-example', Buffer.alloc(0))) + .rejects.toMatchObject({ status: 400, code: 'USDZ_INVALID' }); + await expect(saveModelUsdz('image3d-example', zip(USDZ_MAX_BYTES))) + .rejects.toMatchObject({ status: 413, code: 'USDZ_TOO_LARGE' }); + expect(writeFile).not.toHaveBeenCalled(); + }); + + it('refuses to store an export for a record with no rendered mesh', async () => { + store.getModel.mockResolvedValueOnce({ ...draftRecord(), status: 'generating' }); + await expect(saveModelUsdz('image3d-example', zip())) + .rejects.toMatchObject({ status: 409, code: 'MODEL_NOT_READY' }); + }); + + it('404s a record that has never been exported for AR', async () => { + // Unlike the GLB, an absent USDZ says nothing about the record's health — + // it just means nobody has opened this model in the viewer and exported it. + store.getModel.mockResolvedValueOnce(ready()); + await expect(getModelUsdz('image3d-example', { exists: async () => false })) + .rejects.toMatchObject({ status: 404, code: 'USDZ_MISSING' }); + }); + + it('serves the stored export with a slugged filename', async () => { + store.getModel.mockResolvedValueOnce(ready()); + const artifact = await getModelUsdz('image3d-example', { exists: async () => true }); + expect(posixPath(artifact.path)).toMatch(/image-to-3d\/image3d-example\/model\.usdz$/); + expect(artifact.filename).toBe('my-beacon.usdz'); + }); + }); }); describe('render options and source keying', () => {