diff --git a/docs/webgpu_playground.md b/docs/webgpu_playground.md new file mode 100644 index 0000000000..6e8b0c03f7 --- /dev/null +++ b/docs/webgpu_playground.md @@ -0,0 +1,67 @@ +# On-device WebGPU playground + +The **WebGPU playground** is an opt-in page that turns a phone, tablet or laptop +browser into an on-device inference target. Open it from any device that can +reach your inference server, and it: + +1. checks the device for WebGPU and reads the GPU adapter: vendor and + architecture, fp16 shader support and buffer limits; +2. measures the GPU with two short compute shaders: memory bandwidth (a `vec4` + copy over a large buffer) and fp32 throughput (a chain of fused + multiply-adds); +3. estimates the decode speed of each model from the measured bandwidth; +4. downloads a small instruction-tuned language model once and chats with it, + streaming tokens and reporting the decode speed measured on that GPU. + +Everything after the page load runs in the browser, on the device's own GPU. +The server only hands out one static HTML page: prompts and answers stay on the +device, and the playground adds no inference endpoints. + +## Enable it + +The playground is off by default. Set `ENABLE_WEBGPU_PLAYGROUND=True` and open +`/webgpu` on the server: + +```bash +docker run -p 9001:9001 \ + -e ENABLE_WEBGPU_PLAYGROUND=True \ + roboflow/roboflow-inference-server-cpu +``` + +Then browse to `http://:9001/webgpu` from the phone. WebGPU +needs a secure context: `localhost` works as is; from another device, serve the +page over HTTPS (see the server's HTTPS configuration) or a tunnel. + +## Supported browsers + +| Device | Browser | +|---|---| +| Android | Chrome 121 or later | +| iPhone, iPad | Safari on iOS / iPadOS 26 or later (earlier versions: enable the WebGPU feature flag) | +| Desktop | Chrome, Edge, or Safari 26 or later | + +On a GPU without `shader-f16`, the page loads the fp32 build of each model. On a +software renderer (SwiftShader, lavapipe, WARP) the page still runs, but the +CPU executes the shaders: it marks the adapter as software and shows no speed +estimate, only the speed it measures after each answer. + +## Models + +The models are prebuilt [WebLLM](https://github.com/mlc-ai/web-llm) builds, +4-bit, from the `mlc-ai` organisation on Hugging Face: + +| Model | GPU memory (fp16 / fp32 build) | Notes | +|---|---|---| +| SmolLM2 360M Instruct | 376 / 580 MB | the default: smallest, quickest to download | +| Qwen2.5 0.5B Instruct | 945 / 1,060 MB | best answers for its size | +| Llama 3.2 1B Instruct | 879 / 1,129 MB | for newer phones | + +A model downloads once and stays in the browser's cache; the page has a button +that clears it. + +## Network access + +The page itself is served by the inference server. When a visitor starts a +model, the browser fetches WebLLM (`@mlc-ai/web-llm@0.2.78`, Apache-2.0) from +jsDelivr and the model weights from Hugging Face. On an air-gapped network the +GPU check and measurement still work; loading a model needs those two hosts. diff --git a/inference/core/env.py b/inference/core/env.py index 26e51244eb..ba6767a407 100644 --- a/inference/core/env.py +++ b/inference/core/env.py @@ -801,6 +801,11 @@ # confirmed the route responds, so this flag does not need to be flipped just # to keep that link from 404ing. ENABLE_BUILDER = str2bool(os.getenv("ENABLE_BUILDER", False)) +# Serves the on-device WebGPU playground at /webgpu: a page that measures the +# visitor's GPU (phone, tablet or laptop) and runs a small language model in the +# browser via WebLLM. The server only hands out the page; prompts and outputs +# stay on the device. Off by default. +ENABLE_WEBGPU_PLAYGROUND = str2bool(os.getenv("ENABLE_WEBGPU_PLAYGROUND", False)) # Origin allowed to make cross-origin calls into the builder API/CORS # middleware (see http_api.py) and embedded into editor.html for the builder # UI to call back out to. Defaults to this deployment's Roboflow app URL diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 270a7e13d8..93c124bdb8 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -190,6 +190,7 @@ ENABLE_CUDA_MEMORY_RECLAMATION_WATCHDOG, ENABLE_DASHBOARD, ENABLE_STREAM_API, + ENABLE_WEBGPU_PLAYGROUND, ENABLE_WORKFLOWS_PROFILING, GCP_SERVERLESS, GET_MODEL_REGISTRY_ENABLED, @@ -4779,6 +4780,15 @@ def notebook_start(browserless: bool = False): # Attach all routes from builder to the /build prefix app.include_router(builder_router, prefix="/build", tags=["builder"]) + if ENABLE_WEBGPU_PLAYGROUND: + from inference.core.interfaces.http.webgpu.routes import ( + mount_webgpu_playground, + ) + + # On-device WebGPU playground: one static page; models run in the + # visitor's browser, so this adds no inference endpoints. + mount_webgpu_playground(app, prefix="/webgpu") + if LEGACY_ROUTE_ENABLED: # Legacy object detection inference path for backwards compatibility @app.get( diff --git a/inference/core/interfaces/http/webgpu/__init__.py b/inference/core/interfaces/http/webgpu/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/inference/core/interfaces/http/webgpu/playground.html b/inference/core/interfaces/http/webgpu/playground.html new file mode 100644 index 0000000000..02c73f0dde --- /dev/null +++ b/inference/core/interfaces/http/webgpu/playground.html @@ -0,0 +1,296 @@ + + + + + + +On-device WebGPU · Roboflow Inference + + + + +
+ Roboflow Inference · On-device +

Run a model on this device.

+

Measure this GPU, then chat with a small language model running entirely on it, in this tab. Prompts and answers stay on the device.

+ +

1WebGPU on this device

+
Checking…
+ +
+ +

2Measure the GPU

+
+
Memory bandwidth
–GB/s
+
Compute (fp32)
–TFLOP/s
+
+

Two short compute shaders: a vec4 copy over a large buffer, and a chain of fused multiply-adds. Token generation is memory-bound, so bandwidth sets the pace.

+ +
+ +

3Choose a model

+
+

Estimates appear after the measurement: bandwidth ÷ the weights read per token.

+ + + +
+ + + +

Storage

+

Downloaded models stay in this browser's cache, so the next start is quick. Clearing frees the space.

+ + +
+ +
Runs on WebLLM and WebGPU. The server only sends this page.
+
+ + + + diff --git a/inference/core/interfaces/http/webgpu/routes.py b/inference/core/interfaces/http/webgpu/routes.py new file mode 100644 index 0000000000..7f2e1e907b --- /dev/null +++ b/inference/core/interfaces/http/webgpu/routes.py @@ -0,0 +1,52 @@ +"""On-device WebGPU playground: an opt-in page for phones, tablets and laptops. + +Mounted by ``inference/core/interfaces/http/http_api.py`` under ``/webgpu`` only +when ``ENABLE_WEBGPU_PLAYGROUND`` is truthy (default: False). The page measures +the visitor's GPU with WebGPU compute shaders and runs a small language model in +the browser through WebLLM, so prompts and outputs stay on the device: the +server only serves one static HTML file. +""" + +from pathlib import Path + +from fastapi import FastAPI +from starlette.responses import HTMLResponse + +PLAYGROUND_PAGE = Path(__file__).parent / "playground.html" + + +async def webgpu_playground() -> HTMLResponse: + """Serve the on-device WebGPU playground page. + + Returns: + HTMLResponse: the self-contained playground page. It loads WebLLM from + jsDelivr and model weights from Hugging Face in the browser, only when + the visitor starts a model. + """ + return HTMLResponse( + PLAYGROUND_PAGE.read_text(encoding="utf-8"), + headers={"Cache-Control": "no-cache"}, + ) + + +def mount_webgpu_playground(app: FastAPI, prefix: str = "/webgpu") -> None: + """Register the playground page at ``prefix`` and ``prefix + "/"``. + + The routes go on the app directly rather than through ``include_router``: + FastAPI 0.141 wraps an included router in an object without ``.path``, and + prometheus-fastapi-instrumentator 6.0.0 reads ``route.path`` on every route + while it names a request, so a page behind ``include_router`` answers 500 + while metrics are on (the default). + + Args: + app: the inference server's FastAPI app. + prefix: the URL path of the page. + """ + for path in (prefix, f"{prefix}/"): + app.add_api_route( + path, + webgpu_playground, + methods=["GET"], + include_in_schema=False, + tags=["webgpu"], + ) diff --git a/mkdocs.yml b/mkdocs.yml index 4585649931..1f1b1978c8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,7 @@ nav: - "Models": "https://docs.roboflow.com/models/supported-models" - "Workflows": "https://docs.roboflow.com/workflows" - "Classifier Embeddings": workflows/classifier_embeddings.md + - "On-device WebGPU": webgpu_playground.md - "Reference": "https://docs.roboflow.com/reference/inference/choosing-the-right-tool" - "Changelog": "https://github.com/roboflow/inference/releases" diff --git a/tests/inference/unit_tests/core/interfaces/http/test_webgpu_playground.py b/tests/inference/unit_tests/core/interfaces/http/test_webgpu_playground.py new file mode 100644 index 0000000000..08e8f6b236 --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/test_webgpu_playground.py @@ -0,0 +1,85 @@ +"""Tests for the opt-in on-device WebGPU playground served at /webgpu. + +http_api.py mounts the page only when ENABLE_WEBGPU_PLAYGROUND is truthy +(default: False). As in test_builder_availability.py, these tests exercise the +mount on a bare FastAPI app instead of importing the whole http_api module +(which pulls in the full model-serving stack). +""" + +import re + +import pytest +from fastapi import FastAPI +from fastapi.routing import APIRoute +from prometheus_fastapi_instrumentator import Instrumentator +from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND +from starlette.testclient import TestClient + +from inference.core.interfaces.http.webgpu.routes import ( + PLAYGROUND_PAGE, + mount_webgpu_playground, +) + + +@pytest.fixture +def webgpu_app() -> FastAPI: + app = FastAPI() + mount_webgpu_playground(app, prefix="/webgpu") + return app + + +def test_playground_absent_when_not_mounted() -> None: + """Mirrors the `if ENABLE_WEBGPU_PLAYGROUND:` branch taking the False path.""" + client = TestClient(FastAPI()) + + assert client.get("/webgpu").status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.parametrize("path", ["/webgpu", "/webgpu/"]) +def test_playground_served_when_mounted(webgpu_app: FastAPI, path: str) -> None: + client = TestClient(webgpu_app) + + response = client.get(path, follow_redirects=False) + + assert response.status_code == HTTP_200_OK + assert "text/html" in response.headers["content-type"] + assert response.headers["cache-control"] == "no-cache" + assert "navigator.gpu" in response.text + assert "isSecureContext" in response.text + + +def test_playground_served_with_metrics_instrumentation(webgpu_app: FastAPI) -> None: + """The server instruments every app with prometheus-fastapi-instrumentator + (METRICS_ENABLED defaults to True); the page must still answer 200.""" + Instrumentator().instrument(webgpu_app) + client = TestClient(webgpu_app) + + assert client.get("/webgpu").status_code == HTTP_200_OK + + +def test_playground_adds_no_api_routes(webgpu_app: FastAPI) -> None: + """The page runs models in the browser: the mount serves HTML and nothing else.""" + routes = { + (route.path, frozenset(route.methods)) + for route in webgpu_app.routes + if isinstance(route, APIRoute) + } + + assert routes == { + ("/webgpu", frozenset({"GET"})), + ("/webgpu/", frozenset({"GET"})), + } + assert not [p for p in webgpu_app.openapi()["paths"] if p.startswith("/webgpu")] + + +def test_playground_loads_only_the_pinned_webllm_build() -> None: + """The only third-party script is WebLLM, pinned to an exact version.""" + page = PLAYGROUND_PAGE.read_text(encoding="utf-8") + + script_sources = re.findall(r"]*\bsrc=", page) + module_imports = set(re.findall(r"https://cdn\.jsdelivr\.net/npm/[^'\"]+", page)) + + assert script_sources == [] + assert module_imports == { + "https://cdn.jsdelivr.net/npm/@mlc-ai/web-llm@0.2.78/+esm" + }