From 3fd9d8183828f6960e860201b0cd53778500f840 Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 22:57:07 -0700
Subject: [PATCH 1/6] feat(config): plumb endpoint fields from LLM config to
 ModelEndpoint

Add api_base, api_key, max_tokens, timeout_seconds and is_multimodal to
the LLM config schema (artemis/config/llm.py) and forward them in
_resolve_endpoint() (artemis/services/llm.py) for primary and fallback
nodes, so local OpenAI-compatible endpoints (Ollama/vLLM/custom) can be
configured per agent. LLM.validate_provider no longer requires cloud API
keys for ollama/vllm/custom providers.

Refs #1, refs #2
---
 artemis/config/llm.py   |  9 +++++++++
 artemis/services/llm.py | 15 ++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/artemis/config/llm.py b/artemis/config/llm.py
index 36915dd9..fc19992f 100644
--- a/artemis/config/llm.py
+++ b/artemis/config/llm.py
@@ -80,9 +80,18 @@ class LLM(BaseModel):
     reasoning_effort: Literal["none", "low", "medium", "high"] | None = None
     include_thoughts: bool | None = None
     enable_grounding: bool | None = None
+    api_base: str | None = None
+    api_key: str | None = None
+    max_tokens: int | None = None
+    timeout_seconds: float | None = None
+    is_multimodal: bool | None = None
 
     def validate_provider(self, name: str) -> None:
         """Ensure the required API key or credentials exist in settings for this provider."""
+        # Local / self-hosted OpenAI-compatible providers do not use cloud API
+        # keys; connectivity is configured via api_base instead.
+        if self.provider in ("ollama", "vllm", "custom"):
+            return
         if self.provider == "openai":
             if not settings.OPENAI_API_KEY:
                 raise Exception(f"{name} requires OPENAI_API_KEY in .env")
diff --git a/artemis/services/llm.py b/artemis/services/llm.py
index 784e971f..c24afedc 100644
--- a/artemis/services/llm.py
+++ b/artemis/services/llm.py
@@ -1064,16 +1064,29 @@ def _get_val(obj, attr, expected_type):
     provider_val = getattr(cfg, "provider", "google")
     model_val = getattr(cfg, "model", "gemini-2.5-flash")
 
+    # Endpoint-level knobs (api_base, api_key, max_tokens, timeout_seconds,
+    # is_multimodal) only exist on LLM configs that opt in to them; forward
+    # them to the ModelEndpoint so local/self-hosted endpoints (Ollama, vLLM,
+    # custom OpenAI-compatible servers) can be reached. Applies to fallback
+    # endpoints too, since cfg has already been swapped to the fallback above.
+    is_multimodal_val = _get_val(cfg, "is_multimodal", bool)
+
     return ModelEndpoint(
         provider=ModelProvider.from_string(provider_val),
         model_name=str(model_val),
         temperature=_get_val(cfg, "temperature", (int, float)) or 0.0,
-        timeout_seconds=_get_val(cfg, "timeout", (int, float)) or 60.0,
+        max_tokens=_get_val(cfg, "max_tokens", int),
+        timeout_seconds=_get_val(cfg, "timeout_seconds", (int, float))
+        or _get_val(cfg, "timeout", (int, float))
+        or 60.0,
+        api_base=_get_val(cfg, "api_base", str),
+        api_key=_get_val(cfg, "api_key", str),
         thinking_budget=_get_val(cfg, "thinking_budget", int),
         thinking_level=_get_val(cfg, "thinking_level", str),
         reasoning_effort=_get_val(cfg, "reasoning_effort", str),
         include_thoughts=_get_val(cfg, "include_thoughts", bool),
         enable_grounding=_get_val(cfg, "enable_grounding", bool) or False,
+        **({"is_multimodal": is_multimodal_val} if is_multimodal_val is not None else {}),
     )
 
 

From a3ef5dd809056e74628ce6a39d7ccdd154499423 Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 22:57:28 -0700
Subject: [PATCH 2/6] fix(config): repair local-ollama preset and string-safe
 JSONC parsing

The local-ollama preset used provider 'openai' with no api_base, which
failed credential validation without OPENAI_API_KEY. Switch it to
provider 'ollama' with api_base http://localhost:11434/v1 and a local VLM
default (qwen2.5vl:7b), in both config/artemis.jsonc and the bundled
artemis/resources/config/artemis.jsonc.

That requires strip_json_comments (artemis/utils/file.py) to ignore '//'
and '/* */' inside string literals; the previous regex truncated URL
values such as 'http://localhost:11434/v1' and broke config loading.

Also document local-endpoint env fallbacks in .env.example.

Refs #2
---
 .env.example                           |  8 ++++++
 artemis/resources/config/artemis.jsonc | 14 +++++++--
 artemis/utils/file.py                  | 39 +++++++++++++++++++++++---
 config/artemis.jsonc                   | 14 +++++++--
 4 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/.env.example b/.env.example
index 8b23d0a5..03d609f9 100644
--- a/.env.example
+++ b/.env.example
@@ -12,6 +12,14 @@ ANTHROPIC_API_KEY=
 OPEN_ROUTER_API_KEY=
 XAI_API_KEY=
 
+# Local / Self-Hosted Endpoints (Optional - Ollama, vLLM, or any
+# OpenAI-compatible server; no cloud API key required for these providers).
+# Prefer per-model "api_base" / "api_key" fields in config/artemis.jsonc
+# (see the "local-ollama" preset); these env vars are the fallback defaults.
+# OPENAI_BASE_URL=http://localhost:11434/v1
+# ARTEMIS_OLLAMA_HOST=http://localhost:11434
+# ARTEMIS_VLLM_HOST=http://localhost:8000/v1
+
 # Google Cloud Vision OCR (Optional - for advanced OCR processing)
 OCR_API_KEY=
 
diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc
index 517d7384..fb2264e1 100644
--- a/artemis/resources/config/artemis.jsonc
+++ b/artemis/resources/config/artemis.jsonc
@@ -41,10 +41,18 @@
       "model": "gemini-3.5-flash-lite",
       "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" }
     },
+    // 🏠 Local On-Device Tier: Ollama-served OpenAI-compatible VLM endpoint
+    // (no cloud API key required). Point api_base at your local/edge server
+    // and pick any multimodal model you have pulled, e.g. `ollama pull qwen2.5vl:7b`.
     "local-ollama": {
-      "provider": "openai",
-      "model": "llama3.2-vision",
-      "fallback": { "provider": "openai", "model": "llama3.2-vision" }
+      "provider": "ollama",
+      "model": "qwen2.5vl:7b",
+      "api_base": "http://localhost:11434/v1",
+      "fallback": {
+        "provider": "ollama",
+        "model": "qwen2.5vl:7b",
+        "api_base": "http://localhost:11434/v1"
+      }
     }
   },
 
diff --git a/artemis/utils/file.py b/artemis/utils/file.py
index 7b8d97d6..13b68fd1 100644
--- a/artemis/utils/file.py
+++ b/artemis/utils/file.py
@@ -13,14 +13,45 @@
 # limitations under the License.
 
 import json
-import re
 from typing import IO
 
 
 def strip_json_comments(text: str) -> str:
-    text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE)
-    text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
-    return text
+    """Remove ``//`` and ``/* */`` comments without touching string literals.
+
+    A naive regex strips ``//`` inside strings too, which corrupts values like
+    ``"http://localhost:11434/v1"``; this scanner tracks string state instead.
+    """
+    out: list[str] = []
+    i = 0
+    n = len(text)
+    in_string = False
+    while i < n:
+        ch = text[i]
+        if in_string:
+            out.append(ch)
+            if ch == "\\" and i + 1 < n:
+                out.append(text[i + 1])
+                i += 2
+                continue
+            if ch == '"':
+                in_string = False
+            i += 1
+        elif ch == '"':
+            in_string = True
+            out.append(ch)
+            i += 1
+        elif ch == "/" and i + 1 < n and text[i + 1] == "/":
+            # Line comment: skip to (but keep) the newline.
+            while i < n and text[i] not in "\r\n":
+                i += 1
+        elif ch == "/" and i + 1 < n and text[i + 1] == "*":
+            end = text.find("*/", i + 2)
+            i = n if end == -1 else end + 2
+        else:
+            out.append(ch)
+            i += 1
+    return "".join(out)
 
 
 def load_jsonc(file: IO) -> dict:
diff --git a/config/artemis.jsonc b/config/artemis.jsonc
index 517d7384..fb2264e1 100644
--- a/config/artemis.jsonc
+++ b/config/artemis.jsonc
@@ -41,10 +41,18 @@
       "model": "gemini-3.5-flash-lite",
       "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" }
     },
+    // 🏠 Local On-Device Tier: Ollama-served OpenAI-compatible VLM endpoint
+    // (no cloud API key required). Point api_base at your local/edge server
+    // and pick any multimodal model you have pulled, e.g. `ollama pull qwen2.5vl:7b`.
     "local-ollama": {
-      "provider": "openai",
-      "model": "llama3.2-vision",
-      "fallback": { "provider": "openai", "model": "llama3.2-vision" }
+      "provider": "ollama",
+      "model": "qwen2.5vl:7b",
+      "api_base": "http://localhost:11434/v1",
+      "fallback": {
+        "provider": "ollama",
+        "model": "qwen2.5vl:7b",
+        "api_base": "http://localhost:11434/v1"
+      }
     }
   },
 

From 219bff88113e28b1b4b2a3708928c4301c8b468c Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 22:57:28 -0700
Subject: [PATCH 3/6] test(llm): add provider-surface unit tests for the model
 router

Cover ModelProvider.from_string alias parsing (incl. ollama/vllm/custom),
ModelFactory.create_model construction for local endpoints with
langchain_openai.ChatOpenAI mocked (no network), _resolve_endpoint
passthrough of api_base/api_key/max_tokens/timeout_seconds/is_multimodal
for primary and fallback nodes, local-provider credential validation,
string-safe JSONC comment stripping, and the local-ollama presets.

Refs #6
---
 tests/unit/test_llm_router.py | 278 ++++++++++++++++++++++++++++++++++
 1 file changed, 278 insertions(+)
 create mode 100644 tests/unit/test_llm_router.py

diff --git a/tests/unit/test_llm_router.py b/tests/unit/test_llm_router.py
new file mode 100644
index 00000000..0d71b0d6
--- /dev/null
+++ b/tests/unit/test_llm_router.py
@@ -0,0 +1,278 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the multi-provider model router and endpoint resolution.
+
+Covers ModelProvider alias parsing, ModelFactory construction of local
+OpenAI-compatible endpoints (Ollama/vLLM/custom), and the config-to-endpoint
+passthrough in ``_resolve_endpoint`` (api_base / api_key / max_tokens).
+"""
+
+from types import SimpleNamespace
+from unittest.mock import patch
+import json
+
+import pytest
+
+from artemis.config.llm import LLM, LLMWithFallback
+from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
+from artemis.services import llm as llm_service
+from artemis.services.llm import _resolve_endpoint
+
+
+class TestModelProviderFromString:
+    def test_local_and_custom_providers_parse(self):
+        assert ModelProvider.from_string("ollama") is ModelProvider.OLLAMA
+        assert ModelProvider.from_string("vllm") is ModelProvider.VLLM
+        assert ModelProvider.from_string("custom") is ModelProvider.CUSTOM
+
+    def test_cloud_aliases_parse(self):
+        assert ModelProvider.from_string("google") is ModelProvider.GOOGLE
+        assert ModelProvider.from_string("gemini") is ModelProvider.GOOGLE
+        assert ModelProvider.from_string("vertexai") is ModelProvider.VERTEX_AI
+        assert ModelProvider.from_string("vertex") is ModelProvider.VERTEX_AI
+        assert ModelProvider.from_string("openai") is ModelProvider.OPENAI
+        assert ModelProvider.from_string("anthropic") is ModelProvider.ANTHROPIC
+        assert ModelProvider.from_string("claude") is ModelProvider.ANTHROPIC
+        assert ModelProvider.from_string("openrouter") is ModelProvider.OPENROUTER
+        assert ModelProvider.from_string("xai") is ModelProvider.XAI
+        assert ModelProvider.from_string("grok") is ModelProvider.XAI
+
+    def test_normalization_is_case_and_separator_insensitive(self):
+        assert ModelProvider.from_string(" Ollama ") is ModelProvider.OLLAMA
+        assert ModelProvider.from_string("V-LLM") is ModelProvider.VLLM
+        assert ModelProvider.from_string("Vertex_AI") is ModelProvider.VERTEX_AI
+        assert ModelProvider.from_string("OPENAI") is ModelProvider.OPENAI
+
+    def test_none_and_empty_mean_default_google(self):
+        assert ModelProvider.from_string(None) is ModelProvider.GOOGLE
+        assert ModelProvider.from_string("") is ModelProvider.GOOGLE
+        assert ModelProvider.from_string("   ") is ModelProvider.GOOGLE
+
+    def test_enum_passes_through(self):
+        assert ModelProvider.from_string(ModelProvider.OLLAMA) is ModelProvider.OLLAMA
+
+    def test_unknown_provider_raises(self):
+        with pytest.raises(ValueError, match="Unknown LLM provider"):
+            ModelProvider.from_string("sentient-toaster")
+
+
+class TestModelFactoryLocalEndpoints:
+    """create_model() for OLLAMA/VLLM/CUSTOM builds a langchain_openai ChatOpenAI."""
+
+    @pytest.fixture(autouse=True)
+    def _clean_env(self, monkeypatch):
+        monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+
+    def _build(self, endpoint: ModelEndpoint):
+        with patch("langchain_openai.ChatOpenAI") as mock_cls:
+            mock_cls.return_value = object()
+            result = ModelFactory.create_model(endpoint)
+        assert mock_cls.call_count == 1
+        return result, mock_cls.call_args.kwargs
+
+    def test_ollama_endpoint_uses_configured_api_base(self):
+        endpoint = ModelEndpoint(
+            provider=ModelProvider.OLLAMA,
+            model_name="qwen2.5vl:7b",
+            api_base="http://localhost:11434/v1",
+            max_tokens=2048,
+            timeout_seconds=120.0,
+        )
+        _, kwargs = self._build(endpoint)
+        assert kwargs["model"] == "qwen2.5vl:7b"
+        assert kwargs["base_url"] == "http://localhost:11434/v1"
+        assert kwargs["max_tokens"] == 2048
+        assert kwargs["timeout"] == 120.0
+        # Local servers accept any non-empty key; "EMPTY" is the default.
+        assert kwargs["api_key"] == "EMPTY"
+
+    def test_custom_endpoint_falls_back_to_localhost_default(self):
+        endpoint = ModelEndpoint(provider=ModelProvider.CUSTOM, model_name="edge-vlm")
+        _, kwargs = self._build(endpoint)
+        assert kwargs["base_url"] == "http://localhost:8000/v1"
+
+    def test_endpoint_api_key_takes_precedence(self):
+        endpoint = ModelEndpoint(
+            provider=ModelProvider.VLLM,
+            model_name="qwen2.5vl-7b-awq",
+            api_base="http://edge-box:8000/v1",
+            api_key="sk-local-secret",
+        )
+        _, kwargs = self._build(endpoint)
+        assert kwargs["api_key"] == "sk-local-secret"
+        assert kwargs["base_url"] == "http://edge-box:8000/v1"
+
+    def test_none_values_are_not_forwarded(self):
+        endpoint = ModelEndpoint(provider=ModelProvider.OLLAMA, model_name="m")
+        _, kwargs = self._build(endpoint)
+        assert "max_tokens" not in kwargs
+        assert "reasoning_effort" not in kwargs
+
+    def test_reasoning_effort_forwarded_when_set(self):
+        endpoint = ModelEndpoint(
+            provider=ModelProvider.CUSTOM, model_name="m", reasoning_effort="low"
+        )
+        _, kwargs = self._build(endpoint)
+        assert kwargs["reasoning_effort"] == "low"
+
+
+class TestResolveEndpointPassthrough:
+    """_resolve_endpoint forwards endpoint fields from LLM config to ModelEndpoint."""
+
+    @staticmethod
+    def _ctx(cfg: LLMWithFallback) -> SimpleNamespace:
+        return SimpleNamespace(llm_config=SimpleNamespace(get_agent=lambda name: cfg))
+
+    def test_endpoint_fields_pass_through(self):
+        cfg = LLMWithFallback(
+            provider="ollama",
+            model="qwen2.5vl:7b",
+            api_base="http://localhost:11434/v1",
+            api_key="sk-local",
+            max_tokens=4096,
+            timeout_seconds=180.0,
+            is_multimodal=True,
+            fallback=LLM(provider="ollama", model="qwen2.5vl:3b"),
+        )
+        endpoint = _resolve_endpoint(self._ctx(cfg), "planner")
+        assert endpoint.provider is ModelProvider.OLLAMA
+        assert endpoint.model_name == "qwen2.5vl:7b"
+        assert endpoint.api_base == "http://localhost:11434/v1"
+        assert endpoint.api_key == "sk-local"
+        assert endpoint.max_tokens == 4096
+        assert endpoint.timeout_seconds == 180.0
+        assert endpoint.is_multimodal is True
+
+    def test_fallback_endpoint_gets_same_treatment(self):
+        cfg = LLMWithFallback(
+            provider="google",
+            model="gemini-3.8-flash",
+            fallback=LLM(
+                provider="custom",
+                model="edge-vlm",
+                api_base="http://192.168.1.10:8000/v1",
+                api_key="sk-edge",
+                max_tokens=1024,
+                timeout_seconds=30.0,
+                is_multimodal=False,
+            ),
+        )
+        endpoint = _resolve_endpoint(self._ctx(cfg), "planner", use_fallback=True)
+        assert endpoint.provider is ModelProvider.CUSTOM
+        assert endpoint.model_name == "edge-vlm"
+        assert endpoint.api_base == "http://192.168.1.10:8000/v1"
+        assert endpoint.api_key == "sk-edge"
+        assert endpoint.max_tokens == 1024
+        assert endpoint.timeout_seconds == 30.0
+        assert endpoint.is_multimodal is False
+
+    def test_unset_fields_keep_endpoint_defaults(self):
+        cfg = LLMWithFallback(
+            provider="google",
+            model="gemini-3.8-flash",
+            fallback=LLM(provider="google", model="gemini-3.7-flash"),
+        )
+        endpoint = _resolve_endpoint(self._ctx(cfg), "planner")
+        assert endpoint.api_base is None
+        assert endpoint.api_key is None
+        assert endpoint.max_tokens is None
+        assert endpoint.timeout_seconds == 60.0
+        assert endpoint.is_multimodal is True
+
+    def test_legacy_timeout_field_still_honored(self):
+        cfg = LLMWithFallback(
+            provider="ollama",
+            model="qwen2.5vl:7b",
+            timeout=42.0,
+            fallback=LLM(provider="ollama", model="qwen2.5vl:3b"),
+        )
+        endpoint = _resolve_endpoint(self._ctx(cfg), "planner")
+        assert endpoint.timeout_seconds == 42.0
+
+    def test_timeout_seconds_takes_precedence_over_legacy_timeout(self):
+        cfg = LLMWithFallback(
+            provider="ollama",
+            model="qwen2.5vl:7b",
+            timeout=42.0,
+            timeout_seconds=90.0,
+            fallback=LLM(provider="ollama", model="qwen2.5vl:3b"),
+        )
+        endpoint = _resolve_endpoint(self._ctx(cfg), "planner")
+        assert endpoint.timeout_seconds == 90.0
+
+
+class TestValidateProviderLocalEndpoints:
+    """Local providers must not demand cloud API keys."""
+
+    def test_local_providers_skip_cloud_key_requirement(self):
+        for provider in ("ollama", "vllm", "custom"):
+            LLM(provider=provider, model="any-model").validate_provider("TestNode")
+
+    def test_openai_still_requires_key(self, monkeypatch):
+        monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", None, raising=False)
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        with pytest.raises(Exception, match="OPENAI_API_KEY"):
+            LLM(provider="openai", model="gpt-4o").validate_provider("TestNode")
+
+
+class TestStripJsonComments:
+    """strip_json_comments must not eat URLs inside string literals."""
+
+    def test_urls_inside_strings_survive(self):
+        from artemis.utils.file import strip_json_comments
+
+        text = '{"api_base": "http://localhost:11434/v1"} // trailing comment'
+        assert json.loads(strip_json_comments(text)) == {"api_base": "http://localhost:11434/v1"}
+
+    def test_line_and_block_comments_are_removed(self):
+        from artemis.utils.file import strip_json_comments
+
+        text = (
+            '{\n// line comment\n"a": 1, /* block // comment */ "b": "x // y",\n'
+            '"c": "escaped \\" // not a comment"\n}'
+        )
+        assert json.loads(strip_json_comments(text)) == {
+            "a": 1,
+            "b": "x // y",
+            "c": 'escaped " // not a comment',
+        }
+
+
+class TestLocalOllamaPreset:
+    """The bundled local-ollama preset must parse and target a local endpoint."""
+
+    @staticmethod
+    def _preset(path):
+        from artemis.utils.file import load_jsonc
+
+        with open(path, encoding="utf-8") as f:
+            return load_jsonc(f)["presets"]["local-ollama"]
+
+    def test_repo_config_preset(self):
+        from artemis.config.paths import ROOT_DIR
+
+        preset = self._preset(ROOT_DIR / "config" / "artemis.jsonc")
+        assert preset["provider"] == "ollama"
+        assert preset["api_base"] == "http://localhost:11434/v1"
+        assert preset["fallback"]["provider"] == "ollama"
+
+    def test_bundled_resource_preset(self):
+        from artemis.config.paths import ROOT_DIR
+
+        preset = self._preset(ROOT_DIR / "artemis" / "resources" / "config" / "artemis.jsonc")
+        assert preset["provider"] == "ollama"
+        assert preset["api_base"] == "http://localhost:11434/v1"

From 971f0e8a6b41970dcb2755615ae66b562d65a487 Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 22:57:28 -0700
Subject: [PATCH 4/6] docs(design): on-device lightweight VLM support design
 doc

Add docs/design/on-device-vlm.md covering deployment shapes (host-side
Ollama/llama.cpp/vLLM, LAN edge box, future on-SoC), the new endpoint
config schema with a hybrid local-flash + cloud-pro JSONC example,
perception pipeline next steps (#3/#4/#5), the privacy model, and a
latency measurement plan. Link it from the roadmap bullets in README.md
and README_CN.md.

Refs #7
---
 README.md                    |   2 +-
 README_CN.md                 |   2 +-
 docs/design/on-device-vlm.md | 174 +++++++++++++++++++++++++++++++++++
 3 files changed, 176 insertions(+), 2 deletions(-)
 create mode 100644 docs/design/on-device-vlm.md

diff --git a/README.md b/README.md
index d1883605..06984311 100644
--- a/README.md
+++ b/README.md
@@ -301,7 +301,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir
 
 - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio.
 - [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators.
-- [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation.
+- [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. See [docs/design/on-device-vlm.md](docs/design/on-device-vlm.md).
 - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling.
 
 ## Community & Contributing
diff --git a/README_CN.md b/README_CN.md
index 7c425c69..75969940 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -299,7 +299,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求：
 
 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。
 - [ ] **iOS 跨平台支持**：将视觉感知与自动化执行引擎拓展至 iOS 真机与模拟器。
-- [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。
+- [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。设计文档见 [docs/design/on-device-vlm.md](docs/design/on-device-vlm.md)。
 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。
 
 ## 社区与贡献
diff --git a/docs/design/on-device-vlm.md b/docs/design/on-device-vlm.md
new file mode 100644
index 00000000..1916e1c9
--- /dev/null
+++ b/docs/design/on-device-vlm.md
@@ -0,0 +1,174 @@
+<!--
+Copyright 2026 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# On-Device Lightweight VLM Support — Design
+
+Status: Draft
+Tracks: roadmap item **On-Device Lightweight VLMs** (`README.md`), issues #1, #2, #3, #4, #5, #6, #7.
+
+## 1. Goal
+
+Run ARTEMIS perception and control loops against **local, lightweight vision-language
+models** (3B–8B class, e.g. `qwen2.5vl:7b`) instead of — or alongside — cloud APIs,
+for low latency, offline operation, and privacy-first automation.
+
+This document describes the endpoint plumbing landed in issues #1/#2 and the design
+for what comes next.
+
+## 2. Deployment Shapes
+
+All shapes expose an **OpenAI-compatible HTTP API**; ARTEMIS never talks to model
+runtimes directly. The only difference is where the server lives.
+
+| Shape | Server | `api_base` example | Notes |
+|---|---|---|---|
+| Host-side | Ollama on the dev machine | `http://localhost:11434/v1` | Zero-config default; the `local-ollama` preset in `config/artemis.jsonc` |
+| Host-side | llama.cpp `llama-server` | `http://localhost:8080/v1` | GGUF quantized VLMs; provider `custom` |
+| Host-side | vLLM | `http://localhost:8000/v1` | GPU hosts; provider `vllm` |
+| LAN edge box | Ollama/vLLM on a home server | `http://192.168.1.10:8000/v1` | Shares one GPU across workstations; set `api_key` if the box is shared |
+| On-SoC (future) | NPU/GPU runtime on the phone itself, fronted by a localhost shim app | `http://127.0.0.1:<port>/v1` | Requires the perception-pipeline work in issues #3–#5 (small context, tight latency budget) |
+
+## 3. Endpoint Configuration Schema
+
+Endpoint knobs now live on the `LLM` / `LLMWithFallback` config schema
+(`artemis/config/llm.py`) and are forwarded to the router's `ModelEndpoint`
+(`artemis/llm/router.py`) by `_resolve_endpoint()` (`artemis/services/llm.py`),
+for primary **and** fallback nodes alike:
+
+| Field | Type | Default | Meaning |
+|---|---|---|---|
+| `provider` | string | — | `ollama`, `vllm`, `custom`, plus the existing cloud providers. Aliases are normalized by `ModelProvider.from_string`. |
+| `model` | string | — | Model identifier as the server knows it (e.g. `qwen2.5vl:7b`). |
+| `api_base` | string \| null | env `OPENAI_BASE_URL`, else `http://localhost:8000/v1` | OpenAI-compatible base URL. |
+| `api_key` | string \| null | `"EMPTY"` for local providers | Only needed for shared/authenticated servers. |
+| `max_tokens` | int \| null | server default | Completion cap; keep small for on-device models. |
+| `timeout_seconds` | float \| null | `60.0` | Per-request timeout; raise for slow quantized models. |
+| `is_multimodal` | bool \| null | `true` | Whether the endpoint accepts image inputs; set `false` for text-only local models so perception nodes are not routed to them. |
+
+`LLM.validate_provider()` no longer demands cloud API keys for `ollama` / `vllm` /
+`custom` — connectivity is a runtime property of `api_base`, not a credential.
+
+### Example: hybrid local-flash + cloud-pro setup
+
+```jsonc
+// config/artemis.jsonc
+{
+  // Cheap, private, low-latency default tier served by Ollama.
+  "default": {
+    "provider": "ollama",
+    "model": "qwen2.5vl:7b",
+    "api_base": "http://localhost:11434/v1",
+    "max_tokens": 2048,
+    "timeout_seconds": 120,
+    "fallback": {
+      // Cloud safety net when the local server is down or unsure.
+      "provider": "google",
+      "model": "gemini-3.8-flash"
+    }
+  },
+  "nodes": {
+    // Keep the hard reasoning on a cloud pro model.
+    "planner": {
+      "provider": "google",
+      "model": "gemini-3.8-pro",
+      "fallback": { "provider": "ollama", "model": "qwen2.5vl:7b" }
+    },
+    // Coordinate grounding stays on a specialized ER model for now (see #3).
+    "object_detector": {
+      "provider": "google",
+      "model": "gemini-robotics-er-2-preview"
+    },
+    // High-frequency lightweight judges are ideal for the local tier.
+    "hopper": {
+      "provider": "ollama",
+      "model": "qwen2.5vl:3b",
+      "api_base": "http://localhost:11434/v1"
+    }
+  }
+}
+```
+
+A ready-made starting point ships as the `local-ollama` preset in
+`config/artemis.jsonc`; environment fallbacks are documented in `.env.example`
+(`OPENAI_BASE_URL`, commented).
+
+## 4. Perception Pipeline — Next Steps
+
+Endpoint plumbing alone does not make a 7B VLM a good UI agent. The following
+work items close the gap (tracked as separate issues):
+
+- **Screenshot resize policy (#3).** Local VLMs have small effective vision
+  resolutions and token budgets. Add a deterministic resize/tile stage before
+  the screenshot enters the prompt: cap the long edge, keep the aspect ratio,
+  and record the scale factor alongside the image so coordinates can be mapped
+  back. Candidate home: the screenshot acquisition path in
+  `artemis/drivers/` / the Explorer input builders in `artemis/agents/`.
+- **Coordinate adapter (#3).** Every `[x, y]` the model emits must be scaled
+  back through the recorded factor before hitting the controller
+  (`artemis/controllers/`). Until this lands, keep `object_detector` /
+  `explorer` on cloud ER models (they are fine-tuned for sub-pixel grounding;
+  see the `object_detector` note in `config/artemis.jsonc`).
+- **Small-context memory (#4).** The default transcript budget
+  (`agent.memory.transcript.context_budget_tokens`, currently tuned for
+  1M-token cloud contexts) must scale down for 8k–32k local contexts: lower
+  `start_ratio`/`soft_ratio`, lean harder on `image_scrub_depth` and the
+  chunking/recall layers so history fits.
+- **Structured-output discipline (#5).** Small models degrade on long tool
+  schemas. Prefer the structured-output path (`artemis/llm/structured.py`)
+  with minimal schemas per node, and disable `include_thoughts`-style
+  reasoning traces the endpoint cannot honor (`reasoning_effort` is already
+  forwarded for vLLM/custom endpoints).
+- **Capability gating (#5).** Use `is_multimodal: false` to keep text-only
+  local models away from perception nodes; `_resolve_endpoint` already carries
+  the flag to `ModelEndpoint`.
+
+## 5. Privacy Model
+
+- With a fully local tier, **screenshots, UI hierarchies, and task text never
+  leave the machine**; the LAN edge-box shape extends the trust boundary to the
+  local network only.
+- Hybrid setups leak data by design at the fallback boundary. Rules of thumb:
+  - Fallbacks to cloud providers should be opt-in per node for
+    privacy-sensitive tasks; set the fallback to another local endpoint to stay
+    fully offline.
+  - `api_key` values belong in `.env` or a secrets manager, not in committed
+    config files. `ModelEndpoint.cache_key()` already hashes the key, so keys
+    never appear in cache indexes or logs.
+  - Telemetry (`artemis/telemetry/`) must not include prompt or image payloads
+    for local endpoints; audit before enabling.
+
+## 6. Latency Measurement Plan
+
+On-device VLMs only pay off if step latency beats the cloud path. Measure, don't
+assume:
+
+1. **Instrumentation.** Reuse the existing per-call accounting in
+   `artemis/services/token_meter.py` (prompt sizes, cache-hit ratios) and the
+   trace pipeline (`artemis/data_engine/trace.py`) to tag every call with
+   provider, model, and `api_base`, so local vs. cloud turns are separable in
+   the same session.
+2. **Metrics.** Per node: time-to-first-token, total step latency (observe →
+   think → act), tokens/sec, and fallback rate (how often the local tier gave
+   up and the cloud fallback fired — via the circuit-breaker/fallback counters
+   in `artemis/llm/reliability.py`).
+3. **Benchmark harness.** Run a fixed task suite in Flash profile against (a)
+   cloud default, (b) `local-ollama` preset, (c) hybrid config from §3, on the
+   same device. Success bar for the roadmap item: median step latency of the
+   local tier ≤ cloud tier on routine tasks, with task success within an agreed
+   delta.
+4. **Gates.** CI keeps provider-surface coverage via
+   `tests/unit/test_llm_router.py`; device-level latency benchmarks run
+   manually (marked `android`/`manual`) rather than in CI.

From 29389186cf672781d53176791f376c468f38224f Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 23:05:01 -0700
Subject: [PATCH 5/6] style(playground): auto-fix ruff 0.16.3 lint failures
 blocking CI

Pre-existing on main: 11 lint errors (datetime.UTC alias style) fail the
Lint Python step under the ruff version CI resolves today. Applied
ruff check --fix + format so the pipeline goes green again.
---
 playground/backend_manager/app/auth/jwt_handler.py        | 8 +++-----
 playground/backend_manager/app/auth/otp_service.py        | 6 +++---
 .../backend_manager/app/services/bigquery_service.py      | 2 +-
 playground/backend_manager/app/services/docker_service.py | 2 +-
 .../backend_manager/app/services/session_manager.py       | 8 ++++----
 5 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/playground/backend_manager/app/auth/jwt_handler.py b/playground/backend_manager/app/auth/jwt_handler.py
index f87991fa..a23e7dee 100644
--- a/playground/backend_manager/app/auth/jwt_handler.py
+++ b/playground/backend_manager/app/auth/jwt_handler.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone, UTC
 from fastapi import Depends, HTTPException, Security, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from jose import JWTError, jwt
@@ -23,13 +23,11 @@
 
 def create_access_token(user_id: str, extra_claims: dict | None = None) -> str:
     """Generate a signed JWT access token."""
-    expire = datetime.now(timezone.utc) + timedelta(
-        minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES
-    )
+    expire = datetime.now(UTC) + timedelta(minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES)
     to_encode = {
         "sub": user_id,
         "exp": expire,
-        "iat": datetime.now(timezone.utc),
+        "iat": datetime.now(UTC),
         "iss": "artemis-backend-manager",
     }
     if extra_claims:
diff --git a/playground/backend_manager/app/auth/otp_service.py b/playground/backend_manager/app/auth/otp_service.py
index bb1f0d62..750dfdcb 100644
--- a/playground/backend_manager/app/auth/otp_service.py
+++ b/playground/backend_manager/app/auth/otp_service.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone, UTC
 import logging
 import secrets
 from app.config import settings
@@ -38,7 +38,7 @@ def generate_otp(self, identifier: str) -> str:
         else:
             code = f"{secrets.randbelow(900000) + 100000}"
 
-        expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.OTP_EXPIRE_SECONDS)
+        expires_at = datetime.now(UTC) + timedelta(seconds=settings.OTP_EXPIRE_SECONDS)
         self._store[ident] = {
             "code": code,
             "expires_at": expires_at,
@@ -65,7 +65,7 @@ def verify_otp(self, identifier: str, code: str) -> bool:
             logger.warning(f"[OTP Service] No active OTP found for {ident}")
             return False
 
-        if datetime.now(timezone.utc) > record["expires_at"]:
+        if datetime.now(UTC) > record["expires_at"]:
             logger.warning(f"[OTP Service] OTP for {ident} has expired")
             self._store.pop(ident, None)
             return False
diff --git a/playground/backend_manager/app/services/bigquery_service.py b/playground/backend_manager/app/services/bigquery_service.py
index 8588ec16..8e8f64a1 100644
--- a/playground/backend_manager/app/services/bigquery_service.py
+++ b/playground/backend_manager/app/services/bigquery_service.py
@@ -34,7 +34,7 @@ class BigQueryMappingService:
 
     def __init__(self):
         self._local_cache: dict[str, SessionRecord] = {}
-        self._client: Optional["bigquery.Client"] = None
+        self._client: bigquery.Client | None = None
         self._table_ref: str = (
             f"{settings.GCP_PROJECT_ID}.{settings.BQ_DATASET}.{settings.BQ_TABLE}"
         )
diff --git a/playground/backend_manager/app/services/docker_service.py b/playground/backend_manager/app/services/docker_service.py
index 85d040e5..11512153 100644
--- a/playground/backend_manager/app/services/docker_service.py
+++ b/playground/backend_manager/app/services/docker_service.py
@@ -33,7 +33,7 @@ class DockerManagerService:
     """Manages creation, monitoring, and deletion of ephemeral Artemis session containers via Docker Socket."""
 
     def __init__(self):
-        self._client: Optional["docker.DockerClient"] = None
+        self._client: docker.DockerClient | None = None
         if DOCKER_AVAILABLE:
             try:
                 self._client = docker.DockerClient(base_url=settings.DOCKER_SOCKET_PATH)
diff --git a/playground/backend_manager/app/services/session_manager.py b/playground/backend_manager/app/services/session_manager.py
index 17bf6d64..c95ef296 100644
--- a/playground/backend_manager/app/services/session_manager.py
+++ b/playground/backend_manager/app/services/session_manager.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import asyncio
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone, UTC
 import logging
 import uuid
 from app.config import settings
@@ -49,7 +49,7 @@ async def _reaper_loop(self):
         while True:
             try:
                 await asyncio.sleep(settings.REAPER_CHECK_INTERVAL_SECONDS)
-                now = datetime.now(timezone.utc)
+                now = datetime.now(UTC)
                 active_sessions = await bigquery_service.list_all_active_sessions()
 
                 for s in active_sessions:
@@ -78,7 +78,7 @@ async def _reaper_loop(self):
     async def create_session(self, user_id: str, request: CreateSessionRequest) -> SessionResponse:
         """Provision a new Cuttlefish emulator + Artemis container and link them via ADB."""
         session_id = str(uuid.uuid4())
-        now = datetime.now(timezone.utc)
+        now = datetime.now(UTC)
         expires_at = now + timedelta(minutes=request.ttl_minutes)
 
         logger.info(
@@ -164,7 +164,7 @@ async def heartbeat(self, session_id: str, user_id: str) -> HeartbeatResponse |
         if not record or record.user_id != user_id or record.status == SessionStatus.TERMINATED:
             return None
 
-        now = datetime.now(timezone.utc)
+        now = datetime.now(UTC)
         record.last_heartbeat_at = now
         record.expires_at = now + timedelta(minutes=settings.SESSION_TTL_MINUTES)
         await bigquery_service.save_session(record)

From e0522193b146a4ab613f22a7956acf6e487a8fd6 Mon Sep 17 00:00:00 2001
From: Ranjithkumar Ragavan <43761047+RanjithRagavan@users.noreply.github.com>
Date: Fri, 18 Sep 2026 04:31:24 -0700
Subject: [PATCH 6/6] docs(design): reference upstream issue #131 instead of
 local tracker

---
 docs/design/on-device-vlm.md | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/design/on-device-vlm.md b/docs/design/on-device-vlm.md
index 1916e1c9..c96cef65 100644
--- a/docs/design/on-device-vlm.md
+++ b/docs/design/on-device-vlm.md
@@ -17,7 +17,7 @@ limitations under the License.
 # On-Device Lightweight VLM Support — Design
 
 Status: Draft
-Tracks: roadmap item **On-Device Lightweight VLMs** (`README.md`), issues #1, #2, #3, #4, #5, #6, #7.
+Tracks: roadmap item **On-Device Lightweight VLMs** (`README.md`); see google/artemis issue #131.
 
 ## 1. Goal
 
@@ -25,7 +25,7 @@ Run ARTEMIS perception and control loops against **local, lightweight vision-lan
 models** (3B–8B class, e.g. `qwen2.5vl:7b`) instead of — or alongside — cloud APIs,
 for low latency, offline operation, and privacy-first automation.
 
-This document describes the endpoint plumbing landed in issues #1/#2 and the design
+This document describes the endpoint plumbing proposed in issue #131 and the design
 for what comes next.
 
 ## 2. Deployment Shapes
@@ -39,7 +39,7 @@ runtimes directly. The only difference is where the server lives.
 | Host-side | llama.cpp `llama-server` | `http://localhost:8080/v1` | GGUF quantized VLMs; provider `custom` |
 | Host-side | vLLM | `http://localhost:8000/v1` | GPU hosts; provider `vllm` |
 | LAN edge box | Ollama/vLLM on a home server | `http://192.168.1.10:8000/v1` | Shares one GPU across workstations; set `api_key` if the box is shared |
-| On-SoC (future) | NPU/GPU runtime on the phone itself, fronted by a localhost shim app | `http://127.0.0.1:<port>/v1` | Requires the perception-pipeline work in issues #3–#5 (small context, tight latency budget) |
+| On-SoC (future) | NPU/GPU runtime on the phone itself, fronted by a localhost shim app | `http://127.0.0.1:<port>/v1` | Requires the perception-pipeline work in §4 (small context, tight latency budget) |
 
 ## 3. Endpoint Configuration Schema
 
@@ -86,7 +86,7 @@ for primary **and** fallback nodes alike:
       "model": "gemini-3.8-pro",
       "fallback": { "provider": "ollama", "model": "qwen2.5vl:7b" }
     },
-    // Coordinate grounding stays on a specialized ER model for now (see #3).
+    // Coordinate grounding stays on a specialized ER model for now (see §4).
     "object_detector": {
       "provider": "google",
       "model": "gemini-robotics-er-2-preview"
@@ -110,28 +110,28 @@ A ready-made starting point ships as the `local-ollama` preset in
 Endpoint plumbing alone does not make a 7B VLM a good UI agent. The following
 work items close the gap (tracked as separate issues):
 
-- **Screenshot resize policy (#3).** Local VLMs have small effective vision
+- **Screenshot resize policy.** Local VLMs have small effective vision
   resolutions and token budgets. Add a deterministic resize/tile stage before
   the screenshot enters the prompt: cap the long edge, keep the aspect ratio,
   and record the scale factor alongside the image so coordinates can be mapped
   back. Candidate home: the screenshot acquisition path in
   `artemis/drivers/` / the Explorer input builders in `artemis/agents/`.
-- **Coordinate adapter (#3).** Every `[x, y]` the model emits must be scaled
+- **Coordinate adapter.** Every `[x, y]` the model emits must be scaled
   back through the recorded factor before hitting the controller
   (`artemis/controllers/`). Until this lands, keep `object_detector` /
   `explorer` on cloud ER models (they are fine-tuned for sub-pixel grounding;
   see the `object_detector` note in `config/artemis.jsonc`).
-- **Small-context memory (#4).** The default transcript budget
+- **Small-context memory.** The default transcript budget
   (`agent.memory.transcript.context_budget_tokens`, currently tuned for
   1M-token cloud contexts) must scale down for 8k–32k local contexts: lower
   `start_ratio`/`soft_ratio`, lean harder on `image_scrub_depth` and the
   chunking/recall layers so history fits.
-- **Structured-output discipline (#5).** Small models degrade on long tool
+- **Structured-output discipline.** Small models degrade on long tool
   schemas. Prefer the structured-output path (`artemis/llm/structured.py`)
   with minimal schemas per node, and disable `include_thoughts`-style
   reasoning traces the endpoint cannot honor (`reasoning_effort` is already
   forwarded for vLLM/custom endpoints).
-- **Capability gating (#5).** Use `is_multimodal: false` to keep text-only
+- **Capability gating.** Use `is_multimodal: false` to keep text-only
   local models away from perception nodes; `_resolve_endpoint` already carries
   the flag to `ModelEndpoint`.