From ee1f18ee86544ff68739c2ecc28279c7d52fc7ff Mon Sep 17 00:00:00 2001 From: Vu Chi Cong Date: Mon, 14 Sep 2026 09:17:17 +0700 Subject: [PATCH 01/13] feat: support custom LLM endpoints and durable task failures --- .env.example | 6 + .../repositories/session_repository.py | 34 +++++ apps/admin_console/routers/system.py | 24 +++- .../services/task_queue_service.py | 57 ++++++++ artemis/agents/flash/runner.py | 9 +- artemis/agents/flash/summarizer.py | 28 ++-- artemis/config/__init__.py | 2 + artemis/config/constants.py | 1 + artemis/config/llm.py | 1 + artemis/config/settings.py | 1 + .../diagnostics/probes/credentials_probe.py | 23 +++- artemis/interfaces/cli/commands/run.py | 13 +- artemis/llm/router.py | 4 + artemis/memory/chunking.py | 27 ++-- artemis/resources/config/artemis.jsonc | 3 + artemis/services/llm.py | 7 + artemis/utils/credentials_validator.py | 14 +- artemis/utils/file.py | 10 +- config/artemis.jsonc | 40 +++--- mcp_server/tools/diagnose.py | 25 +++- .../test_session_status_normalization.py | 26 ++++ .../admin_console/test_task_queue_service.py | 127 ++++++++++++++++++ .../unit/agents/test_flash_step_summarizer.py | 31 ++++- tests/unit/core/test_diagnostics.py | 31 +++++ tests/unit/mcp/test_diagnose_tool.py | 26 ++++ tests/unit/memory/test_history_chunking.py | 5 +- tests/unit/test_cli.py | 22 +++ tests/unit/test_credentials_validator.py | 50 +++++++ tests/unit/test_file_utils.py | 20 +++ tests/unit/test_llm_grounding.py | 59 ++++++++ 30 files changed, 651 insertions(+), 75 deletions(-) create mode 100644 tests/unit/test_file_utils.py diff --git a/.env.example b/.env.example index 8b23d0a5..49e9174d 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,12 @@ ANTHROPIC_API_KEY= OPEN_ROUTER_API_KEY= XAI_API_KEY= +# Optional provider-compatible endpoints. OpenAI-compatible URLs include `/v1`; +# Anthropic URLs are the API origin because the SDK appends `/v1`. +# Per-model `api_base` in artemis.jsonc overrides these values. +# OPENAI_BASE_URL=https://openai-proxy.example/v1 +# ANTHROPIC_BASE_URL=https://anthropic-proxy.example + # Google Cloud Vision OCR (Optional - for advanced OCR processing) OCR_API_KEY= diff --git a/apps/admin_console/database/repositories/session_repository.py b/apps/admin_console/database/repositories/session_repository.py index a12e8692..fba0fbab 100644 --- a/apps/admin_console/database/repositories/session_repository.py +++ b/apps/admin_console/database/repositories/session_repository.py @@ -510,6 +510,40 @@ def get_session_status(self, session_id: str) -> str | None: except Exception: return None + def create_queued_session( + self, + session_id: str, + goal: str, + profile: str, + device_serial: str | None, + start_time: float | None = None, + ) -> bool: + """Persist a queue item before its worker starts a session.""" + try: + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO sessions " + "(session_id, initial_goal, start_time, end_time, status, device_info, pid) " + "VALUES (?, ?, ?, NULL, 'queued', ?, NULL)", + ( + str(session_id), + goal, + start_time if start_time is not None else time.time(), + json.dumps( + { + "profile": profile, + "device_id": device_serial, + } + ), + ), + ) + conn.commit() + return cursor.rowcount > 0 + except Exception: + logger.exception("Could not create queued session %s", session_id) + return False + def update_session_status( self, session_id: str, status: str, end_time: float | None = None ) -> bool: diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index 5127eca6..f9159653 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -422,9 +422,12 @@ def mask_key(k: str | None) -> str | None: anthropic_real = get_real_env_value("ANTHROPIC_API_KEY", "anthropic") openrouter_real = get_real_env_value("OPEN_ROUTER_API_KEY", "openrouter") xai_real = get_real_env_value("XAI_API_KEY", "xai") - base_url_val = settings.OPENAI_BASE_URL or os.environ.get("OPENAI_BASE_URL") - if base_url_val and is_placeholder_key(base_url_val): - base_url_val = None + openai_base_url = settings.OPENAI_BASE_URL or os.environ.get("OPENAI_BASE_URL") + anthropic_base_url = settings.ANTHROPIC_BASE_URL or os.environ.get("ANTHROPIC_BASE_URL") + if openai_base_url and is_placeholder_key(openai_base_url): + openai_base_url = None + if anthropic_base_url and is_placeholder_key(anthropic_base_url): + anthropic_base_url = None ocr_real = get_real_env_value("OCR_API_KEY", "ocr") or get_real_env_value( "VISION_API_KEY", "ocr" ) @@ -467,10 +470,17 @@ def mask_key(k: str | None) -> str | None: }, { "name": "OPENAI_BASE_URL", - "provider": "custom", - "is_set": bool(base_url_val), - "preview": base_url_val, - "description": "Custom API endpoint (for local Ollama, vLLM, DeepSeek, or proxies)", + "provider": "openai", + "is_set": bool(openai_base_url), + "preview": openai_base_url, + "description": "Custom OpenAI-compatible API endpoint", + }, + { + "name": "ANTHROPIC_BASE_URL", + "provider": "anthropic", + "is_set": bool(anthropic_base_url), + "preview": anthropic_base_url, + "description": "Custom Anthropic-compatible API endpoint", }, { "name": "VISION_API_KEY", diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b99b4c91..9d95f7d8 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -1066,6 +1066,7 @@ async def enqueue_tasks( cls.ensure_worker_running() enqueued_tasks = [] + created_trace_session_ids: set[str] = set() now = time.time() endpoint = current_adb_endpoint() @@ -1106,6 +1107,62 @@ async def enqueue_tasks( verification_level=verification_level, explorer_mode=explorer_mode, ) + session_id = str(task_item["session_id"]) + existing_trace = trace_store.read_status(session_id) + trace_created = existing_trace is None + existing_trace_is_terminal = bool( + existing_trace + and existing_trace.get("status") in {"completed", "failed", "cancelled", "success"} + ) + try: + if existing_trace_is_terminal: + raise RuntimeError(f"Session {session_id} already has a terminal trace") + if trace_created: + trace_store.init_trace( + session_id, + goal, + task_item["profile"], + task_item.get("conversation_id"), + task_item.get("device_serial"), + ) + created_trace_session_ids.add(session_id) + if not session_repo.create_queued_session( + session_id, + goal, + task_item["profile"], + task_item.get("device_serial"), + task_item.get("start_time"), + ): + raise RuntimeError(f"Could not persist queued session {session_id}") + except (OSError, RuntimeError) as exc: + DeviceExecutionLock.cancel_reservation(task_item.get("queue_ticket")) + if trace_created or ( + not existing_trace_is_terminal + and session_repo.get_session_by_id(session_id) is None + ): + try: + trace_store.update_trace_status(session_id, "failed", error=str(exc)) + except OSError: + logger.exception( + "Could not mark queue setup failure for session %s", session_id + ) + for enqueued_task in enqueued_tasks: + enqueued_session_id = str(enqueued_task["session_id"]) + session_repo.update_session_status(enqueued_session_id, "failed", time.time()) + if enqueued_session_id in created_trace_session_ids: + try: + trace_store.update_trace_status( + enqueued_session_id, + "failed", + error="Task batch could not be queued.", + ) + except OSError: + logger.exception( + "Could not mark rolled-back queue session %s failed", + enqueued_session_id, + ) + cls._remove_task(enqueued_session_id) + raise state.queue_items.append(task_item) enqueued_tasks.append(task_item) cls._broadcast_startup_progress( diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 936f6065..27e03351 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -81,9 +81,7 @@ from artemis.mcp.observation import observe from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger, mark_ephemeral from artemis.services.llm import ( - RobustChatModelWrapper, acomplete, - get_google_llm, get_llm, invoke_llm_with_timeout_message, ) @@ -279,12 +277,7 @@ def _build_ledger(self) -> TranscriptLedger: def _init_llm(self): """Initializes the Universal LLM via the Service Layer.""" - try: - return get_llm(self.ctx, name="operator") - except Exception as e: - logger.warning(f"Failed to get operator LLM from config, using default: {e}") - - return RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx) + return get_llm(self.ctx, name="operator") def _render_system_prompt(self, tools_declaration: list) -> str: """Renders the system prompt from the flash_runner.md template. diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 56c5e098..12fa6be4 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -36,7 +36,7 @@ from artemis.context import ArtemisContext from artemis.memory.step_memory import JobKey, StepMemoryService -from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm +from artemis.services.llm import RobustChatModelWrapper, get_llm from artemis.services.token_meter import record_llm_usage from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_actions_clean @@ -161,18 +161,22 @@ def __init__( flush_timeout_s=flush_timeout_s, ) - # Initialize lightweight VLM: prioritize explicit model_name - target_model = model_name or "gemini-2.5-flash-lite" - self._model_name = target_model - try: - if model_name: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) - else: - self._llm = get_llm(ctx, name="summarizer", is_utils=True) - except Exception: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) + # The summarizer role carries the provider and default model. This profile + # knob only overrides that model when the caller explicitly supplies one. + self._model_name = model_name or "" + self._llm = get_llm( + ctx, + name="summarizer", + temperature=0.0, + model_name=model_name or None, + ) try: - configured = getattr(self._llm, "model", None) or getattr(self._llm, "model_name", None) + endpoint = getattr(self._llm, "endpoint", None) + configured = getattr(endpoint, "model_name", None) + if not configured: + configured = getattr(self._llm, "model", None) or getattr( + self._llm, "model_name", None + ) if isinstance(configured, str) and configured: self._model_name = configured except Exception as exc: diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index e539f933..212eae75 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -58,6 +58,7 @@ ENV_ADB_PORT, ENV_ADB_SERVER_SOCKET, ENV_ANTHROPIC_API_KEY, + ENV_ANTHROPIC_BASE_URL, ENV_ANTIGRAVITY_APP_DIR, ENV_ANTIGRAVITY_LS_ADDRESS, ENV_DATA_ENGINE_DB_PATH, @@ -296,6 +297,7 @@ "ENV_OPENAI_API_KEY", "ENV_OPENAI_BASE_URL", "ENV_ANTHROPIC_API_KEY", + "ENV_ANTHROPIC_BASE_URL", "ENV_OPEN_ROUTER_API_KEY", "ENV_XAI_API_KEY", "ENV_ARTEMIS_EXPLORER_VERSION", diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 972bc6df..4f9892ba 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -62,6 +62,7 @@ ENV_OPENAI_API_KEY = "OPENAI_API_KEY" ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" +ENV_ANTHROPIC_BASE_URL = "ANTHROPIC_BASE_URL" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" ENV_XAI_API_KEY = "XAI_API_KEY" diff --git a/artemis/config/llm.py b/artemis/config/llm.py index 36915dd9..1f2faaef 100644 --- a/artemis/config/llm.py +++ b/artemis/config/llm.py @@ -74,6 +74,7 @@ class LLM(BaseModel): model_config = {"ignored_types": (CyFunctionDetector,)} provider: LLMProvider model: str + api_base: str | None = None temperature: float | None = None thinking_budget: int | None = None thinking_level: Literal["minimal", "low", "medium", "high"] | None = None diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 7129dd8a..acfeea41 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -108,6 +108,7 @@ class Settings(BaseSettings): # Custom Provider Endpoints OPENAI_BASE_URL: str | None = None + ANTHROPIC_BASE_URL: str | None = None # Android ADB Connectivity ADB_HOST: str | None = Field(default=DEFAULT_ADB_HOST) diff --git a/artemis/core/diagnostics/probes/credentials_probe.py b/artemis/core/diagnostics/probes/credentials_probe.py index 683e216c..eb374bda 100644 --- a/artemis/core/diagnostics/probes/credentials_probe.py +++ b/artemis/core/diagnostics/probes/credentials_probe.py @@ -82,6 +82,7 @@ async def probe(self) -> ProbeResult: "masked": self._mask_key(o_val), "raw_key": o_val, "key": o_val, + "base_url": settings.OPENAI_BASE_URL, } ) api_keys_map["openai"] = o_val @@ -94,6 +95,7 @@ async def probe(self) -> ProbeResult: "masked": self._mask_key(c_val), "raw_key": c_val, "key": c_val, + "base_url": settings.ANTHROPIC_BASE_URL, } ) api_keys_map["anthropic"] = c_val @@ -126,7 +128,6 @@ async def probe(self) -> ProbeResult: for env_var, label, prov_id in [ ("DEEPSEEK_API_KEY", "DeepSeek", "deepseek"), ("GROQ_API_KEY", "Groq", "groq"), - ("OPENAI_BASE_URL", "Custom OpenAI Endpoint", "custom"), ("OLLAMA_BASE_URL", "Local Ollama", "ollama"), ("VLLM_BASE_URL", "vLLM Endpoint", "vllm"), ("VERTEX_AI_PROJECT", "Google Cloud Vertex AI", "vertexai"), @@ -144,6 +145,26 @@ async def probe(self) -> ProbeResult: ) api_keys_map[prov_id] = val.strip() + # Preserve endpoint-only diagnosis when a compatible URL exists without + # its provider key; keyed providers are verified against their own URL above. + openai_base_url = os.environ.get("OPENAI_BASE_URL") + if ( + openai_base_url + and openai_base_url.strip() + and not is_placeholder_key(openai_base_url.strip()) + and "openai" not in api_keys_map + ): + configured_providers.append( + { + "provider": "custom", + "label": "Custom OpenAI Endpoint", + "masked": self._mask_key(openai_base_url.strip()), + "raw_key": openai_base_url.strip(), + "key": openai_base_url.strip(), + } + ) + api_keys_map["custom"] = openai_base_url.strip() + if ocr_key and not is_placeholder_key(ocr_key): api_keys_map["ocr"] = ocr_key.get_secret_value() diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 91a5d044..1ddc6007 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -23,6 +23,7 @@ from adbutils import AdbClient from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, initialize_llm_config, settings +from artemis.runtime import trace_store from artemis.utils.startup_progress import publish_startup_progress from artemis import Agent, Builders from artemis.sdk.types.task import AgentProfile @@ -93,7 +94,17 @@ async def execute_task( session_id=str(effective_sid) if effective_sid else None, ) - llm_config = initialize_llm_config() + try: + llm_config = initialize_llm_config() + except Exception as exc: + if effective_sid: + try: + trace_store.update_trace_status(str(effective_sid), "failed", error=str(exc)) + except OSError: + logger.exception( + "Could not record configuration failure for session %s", effective_sid + ) + raise agent_profile = AgentProfile(name="default", llm_config=llm_config) config = Builders.AgentConfig.with_default_profile(profile=agent_profile) diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 8b44743b..4ad4d60d 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -315,10 +315,14 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: ) or os.environ.get("ANTHROPIC_API_KEY") ) + base_url = endpoint.api_base or ( + str(settings.ANTHROPIC_BASE_URL) if settings.ANTHROPIC_BASE_URL else None + ) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, "api_key": api_key, + "base_url": base_url, "timeout": endpoint.timeout_seconds, } budget = endpoint.thinking_budget diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py index 4e564540..ab8e332d 100644 --- a/artemis/memory/chunking.py +++ b/artemis/memory/chunking.py @@ -45,7 +45,6 @@ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage -from artemis.llm.google import is_google_provider from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService from artemis.memory.transcript import format_session_offset from artemis.utils.logger import get_logger @@ -323,17 +322,23 @@ def __init__( def _get_llm(self): if self._llm is None: - from artemis.services.llm import get_google_llm + from artemis.services.llm import get_llm - self._llm = get_google_llm(model_name=self._model_name, temperature=0.0) + self._llm = get_llm( + self._ctx, name="summarizer", temperature=0.0, model_name=self._model_name + ) return self._llm def _get_fallback_llm(self): if self._fallback_llm is None and self._fallback_model_name: - from artemis.services.llm import get_google_llm - - self._fallback_llm = get_google_llm( - model_name=self._fallback_model_name, temperature=0.0 + from artemis.services.llm import get_llm + + self._fallback_llm = get_llm( + self._ctx, + name="summarizer", + use_fallback=True, + temperature=0.0, + model_name=self._fallback_model_name, ) return self._fallback_llm @@ -952,9 +957,8 @@ def _build_capsule_service(self, ctx: Any) -> StepMemoryService: def _resolve_capsule_fallback_model(self, ctx: Any) -> str | None: """Fallback model for capsule generation when `chunking.model` is down. - Resolved from the LLM config's summarizer role (which inherits the - global default fallback unless overridden). Only same-provider (google) - fallbacks apply — the capsule lens rides the raw google model path. + Resolved from the LLM config's summarizer role, which inherits the + global default fallback unless overridden. """ try: llm_cfg = getattr(ctx, "llm_config", None) if ctx is not None else None @@ -963,9 +967,8 @@ def _resolve_capsule_fallback_model(self, ctx: Any) -> str | None: llm_cfg = get_default_llm_config() fallback = getattr(getattr(llm_cfg, "summarizer", None), "fallback", None) - provider = str(getattr(fallback, "provider", "") or "") model = getattr(fallback, "model", None) - if model and is_google_provider(provider) and model != self._model_name: + if model and model != self._model_name: return str(model) except Exception as exc: logger.debug(f"Capsule fallback model resolution skipped: {exc}", exc_info=True) diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 517d7384..d50b5d6e 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -9,6 +9,9 @@ "default": { "provider": "google", "model": "gemini-3.8-flash", + // For OpenAI/Anthropic models, optional per-model API root. OpenAI-compatible endpoints include /v1; + // Anthropic endpoints normally omit /v1 because the SDK appends it. + // "api_base": "https://openai-proxy.example/v1", "thinking_level": "medium", "fallback": { "provider": "google", diff --git a/artemis/services/llm.py b/artemis/services/llm.py index 784e971f..8449ad95 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -1074,6 +1074,7 @@ def _get_val(obj, attr, expected_type): reasoning_effort=_get_val(cfg, "reasoning_effort", str), include_thoughts=_get_val(cfg, "include_thoughts", bool), enable_grounding=_get_val(cfg, "enable_grounding", bool) or False, + api_base=_get_val(cfg, "api_base", str), ) @@ -1084,6 +1085,7 @@ def get_llm( *, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1094,6 +1096,7 @@ def get_llm( *, is_utils: Literal[True], temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1105,6 +1108,7 @@ def get_llm( is_utils: Literal[True], use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1114,11 +1118,14 @@ def get_llm( is_utils: bool = False, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: """Resolves and instantiates the appropriate LLM wrapper for the given agent role.""" endpoint = _resolve_endpoint(ctx, str(name), is_utils=is_utils, use_fallback=use_fallback) if temperature is not None: endpoint = endpoint.model_copy(update={"temperature": temperature}) + if model_name is not None: + endpoint = endpoint.model_copy(update={"model_name": model_name}) raw_model = ModelFactory.get_model(endpoint) handler = DataEngineCallbackHandler(ctx) diff --git a/artemis/utils/credentials_validator.py b/artemis/utils/credentials_validator.py index e71e8ba4..48c63d15 100644 --- a/artemis/utils/credentials_validator.py +++ b/artemis/utils/credentials_validator.py @@ -76,6 +76,14 @@ async def validate_api_key( clean_provider = provider.strip().lower() clean_key = api_key.strip() + if base_url is None: + from artemis.config import settings + + if clean_provider == "openai": + base_url = settings.OPENAI_BASE_URL + elif clean_provider in ("anthropic", "claude"): + base_url = settings.ANTHROPIC_BASE_URL + if not clean_key: return False, "API key cannot be empty." @@ -136,7 +144,11 @@ async def validate_api_key( return False, f"OpenAI API verification failed ({resp.status_code}): {err_msg}" elif clean_provider in ("anthropic", "claude"): - url = "https://api.anthropic.com/v1/models" + url = ( + f"{base_url.rstrip('/')}/v1/models" + if base_url + else "https://api.anthropic.com/v1/models" + ) resp = await client.get( url, headers={ diff --git a/artemis/utils/file.py b/artemis/utils/file.py index 7b8d97d6..27caf293 100644 --- a/artemis/utils/file.py +++ b/artemis/utils/file.py @@ -18,9 +18,13 @@ def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Strip JSONC comments while preserving quoted strings and escapes.""" + pattern = r'//.*?$|/\*.*?\*/|"(?:\\.|[^\\"])*"' + + def replace_comment(match: re.Match[str]) -> str: + return "" if match.group(0).startswith("/") else match.group(0) + + return re.sub(pattern, replace_comment, text, flags=re.DOTALL | re.MULTILINE) def load_jsonc(file: IO) -> dict: diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 517d7384..20b148be 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -7,13 +7,14 @@ // 1. Global Default Model (Applied to all sub-agents unless overridden below) // ------------------------------------------------------------------------------ "default": { - "provider": "google", - "model": "gemini-3.8-flash", - "thinking_level": "medium", + "provider": "openai", + "model": "gpt-5.6-sol", + // API roots come from OPENAI_BASE_URL or ANTHROPIC_BASE_URL in the environment. + "reasoning_effort": "medium", "fallback": { - "provider": "google", - "model": "gemini-3.7-flash", - "thinking_level": "medium" + "provider": "openai", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium" } }, @@ -71,14 +72,12 @@ "video_analyzer": {}, // 👁️ Object Detector & Explorer (Flash Mode): Visual coordinates grounding and bounding boxes - // CRITICAL REQUIREMENT: Standard LLMs (including standard Gemini Flash, GPT-4o, Claude 3.7) lack sub-pixel spatial coordinate fine-tuning. - // To achieve accurate [x, y] point and bounding box localization, this node MUST use specialized Gemini ER (Embodied Reasoning / Robotics) models: - // e.g., "gemini-robotics-er-2-preview". Non-ER models will fail spatial coordinate detection. "object_detector": { - "model": "gemini-robotics-er-2-preview", + "provider": "openai", + "model": "gpt-5.6-sol", "fallback": { - "provider": "google", - "model": "gemini-robotics-er-2-preview" + "provider": "openai", + "model": "gpt-5.6-sol" } }, @@ -87,7 +86,7 @@ // - "flash" version delegates directly to object_detector (requires Gemini ER model above). // - "pro" / "ultra" versions run multi-turn ReAct loops (supports both Native Gemini and Universal Multi-Model LangChain engines). "explorer": { - "thinking_level": "medium" + "reasoning_effort": "medium" }, // 📋 Summarizer: Aggregates step history and generates structured outcome @@ -95,10 +94,11 @@ // 🔽 Hopper: Fast lightweight package name locator "hopper": { - "model": "gemini-3.5-flash-lite", + "provider": "openai", + "model": "gpt-5.6-sol", "fallback": { - "provider": "google", - "model": "gemini-3.1-flash-lite" + "provider": "openai", + "model": "gpt-5.6-sol" } }, @@ -139,13 +139,13 @@ // Execution: Observe-Think-Act single LLM reactive loop without graph overhead. "flash": { "max_turns": 0, // Maximum reactive turns before stopping (0 = unlimited; history is bounded by transcript compression, not by a turn cap) - "explorer_mode": "flash", // Explorer tier for the Flash runner: 'flash' | 'pro' | 'ultra' (see top-level "explorer") + "explorer_mode": "pro", // Explorer tier for the Flash runner: 'flash' | 'pro' | 'ultra' (see top-level "explorer") // 🧠 Step Summarizer: Asynchronous visual context compressor // Summarizes intermediate steps in background without blocking the runner, // replacing images only after strictly objective, high-density summaries are ready. "step_summarizer": { "enabled": true, // Whether background step summarization is active - "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference + "model": "gpt-5.6-sol", // Lightweight model for fast background inference "prune_history_xml": true // Prune outdated heavy XML trees from historical steps } }, @@ -158,7 +158,7 @@ "pro": { // 🧭 Explorer: Perception sub-agent under Pro profile ('flash', 'pro', or 'ultra') "explorer": { - "mode": "flash" // Explorer tier for Pro nodes (Operator/Validator): 'flash' | 'pro' | 'ultra' + "mode": "pro" // Explorer tier for Pro nodes (Operator/Validator): 'flash' | 'pro' | 'ultra' }, // 🎯 Planner Validation: Async verification when task plan milestones change "planner_validation": { @@ -246,7 +246,7 @@ // call each); milestone closes are exempt, the hard line waives it. "min_steps": 3, "target_source_tokens": 2000, - "model": "gemini-3.8-flash", + "model": "gpt-5.6-sol", "max_chunks": 8 // "max_eras": 8 // independent era cap; omitted = follows max_chunks }, diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index c62b5226..71997be4 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -376,15 +376,23 @@ async def _verify_credentials(cred_result: ProbeResult | None) -> list[dict[str, return [] metadata = cred_result.metadata api_keys = metadata.get("api_keys") or {} - targets: list[tuple[str, str, str]] = [] + targets: list[tuple[str, str, str, str | None]] = [] for entry in metadata.get("providers") or []: provider = str(entry.get("provider") or "").strip() raw_key = entry.get("raw_key") or api_keys.get(provider) or "" if not provider or not raw_key: continue - targets.append((provider, str(entry.get("label") or provider), str(raw_key))) + base_url = entry.get("base_url") + targets.append( + ( + provider, + str(entry.get("label") or provider), + str(raw_key), + str(base_url) if base_url else None, + ) + ) if api_keys.get("ocr"): - targets.append(("ocr", "Vision OCR", str(api_keys["ocr"]))) + targets.append(("ocr", "Vision OCR", str(api_keys["ocr"]), None)) if not targets: return [] @@ -397,13 +405,18 @@ async def _verify_credentials(cred_result: ProbeResult | None) -> list[dict[str, timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS, ) if provider in _ENDPOINT_PROVIDERS - else validate_api_key(provider, key, timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS) - for provider, _label, key in targets + else validate_api_key( + provider, + key, + **({"base_url": base_url} if base_url else {}), + timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS, + ) + for provider, _label, key, base_url in targets ), return_exceptions=True, ) verified: list[dict[str, Any]] = [] - for (provider, label, key), outcome in zip(targets, outcomes): + for (provider, label, key, _base_url), outcome in zip(targets, outcomes): if isinstance(outcome, BaseException): valid, message = False, f"verification raised {outcome.__class__.__name__}: {outcome}" else: diff --git a/tests/unit/admin_console/test_session_status_normalization.py b/tests/unit/admin_console/test_session_status_normalization.py index 79094410..9c1a087f 100644 --- a/tests/unit/admin_console/test_session_status_normalization.py +++ b/tests/unit/admin_console/test_session_status_normalization.py @@ -58,3 +58,29 @@ def test_get_session_status_stays_raw_for_queue_reconcile(tmp_path): _insert_session(db_path, "legacy", "success") assert repository.get_session_status("legacy") == "success" + + +def test_create_queued_session_persists_queue_metadata(tmp_path): + repository = SessionRepository(tmp_path / "sessions.db") + + assert repository.create_queued_session( + "queued-session", "Open settings", "pro", "device-123", start_time=0.0 + ) + + row = repository.get_session_by_id("queued-session") + assert row is not None + assert row["status"] == "queued" + assert row["start_time"] == 0.0 + assert row["device_info"] == '{"profile": "pro", "device_id": "device-123"}' + + +def test_create_queued_session_preserves_existing_terminal_session(tmp_path): + db_path = tmp_path / "sessions.db" + repository = SessionRepository(db_path) + _insert_session(db_path, "terminal", "completed") + + assert not repository.create_queued_session("terminal", "New goal", "flash", None) + + row = repository.get_session_by_id("terminal") + assert row is not None + assert row["status"] == "completed" diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 24a572cd..8c60bf9a 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -22,8 +22,10 @@ import pytest from apps.admin_console.core.state import state +from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.routers.tasks import get_status from apps.admin_console.services.task_queue_service import TaskQueueService, task_queue_service +from artemis.runtime import trace_store from artemis.runtime.device_lock import DeviceLockOwner from artemis.runtime.adb_endpoint import AdbEndpoint @@ -52,6 +54,8 @@ def clean_state(tmp_path, monkeypatch): "artemis.runtime.cancel_requests.get_temp_dir", lambda _subfolder=None: isolated_lock_dir, ) + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + monkeypatch.setattr(queue_module, "session_repo", SessionRepository(tmp_path / "sessions.db")) state.clear_queue() state.queue_items.clear() state.current_process = None @@ -132,6 +136,129 @@ async def test_enqueued_task_keeps_its_adb_endpoint_snapshot(): assert target.lock_key == f"{original.identity}/emulator-5554" +@pytest.mark.asyncio +async def test_pre_session_worker_failure_persists_failed_session_and_releases_ticket(tmp_path): + async def failed_worker(*_args, **_kwargs): + proc = MagicMock() + proc.pid = 99999999 + proc.returncode = 1 + proc.stdout = None + proc.wait = AsyncMock(return_value=1) + return proc + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + patch("asyncio.create_subprocess_exec", side_effect=failed_worker), + patch.object(TaskQueueService, "_recover_or_fail_recording", new=AsyncMock()), + ): + result = await TaskQueueService.enqueue_tasks( + ["Configuration fails before session startup"], + profile="pro", + device_serial="test-device", + ) + task_item = result["tasks"][0] + session_id = task_item["session_id"] + + trace = trace_store.read_status(session_id) + assert trace["status"] == "running" + assert trace["model"] == "pro" + assert trace["device_serial"] == "test-device" + + await TaskQueueService._execute_task_item(task_item) + + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + persisted = queue_module.session_repo.get_session_by_id(session_id) + assert persisted is not None + assert persisted["status"] == "failed" + assert persisted["end_time"] is not None + assert trace_store.read_status(session_id)["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_rejects_terminal_trace_without_overwriting_it(tmp_path): + session_id = "terminal-session" + trace_store.init_trace(session_id, "Finished goal", "flash") + trace_store.update_trace_status(session_id, "completed") + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="terminal trace"), + ): + await TaskQueueService.enqueue_tasks( + ["Do not overwrite"], session_id=session_id, device_serial="test-device" + ) + + assert trace_store.read_status(session_id)["status"] == "completed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_marks_existing_running_trace_failed_when_db_admission_fails( + tmp_path, monkeypatch +): + session_id = "mcp-session" + trace_store.init_trace(session_id, "MCP goal", "flash") + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + monkeypatch.setattr(queue_module.session_repo, "create_queued_session", lambda *_args: False) + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="Could not persist queued session"), + ): + await TaskQueueService.enqueue_tasks( + ["MCP goal"], session_id=session_id, device_serial="test-device", ingress="mcp" + ) + + assert trace_store.read_status(session_id)["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_rolls_back_earlier_items_when_later_setup_fails(tmp_path, monkeypatch): + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + repository = queue_module.session_repo + create_queued_session = repository.create_queued_session + session_ids: list[str] = [] + + def fail_second_queued_session(*args, **kwargs): + session_ids.append(args[0]) + if len(session_ids) == 2: + return False + return create_queued_session(*args, **kwargs) + + monkeypatch.setattr(repository, "create_queued_session", fail_second_queued_session) + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="Could not persist queued session"), + ): + await TaskQueueService.enqueue_tasks( + ["First task", "Second task"], device_serial="test-device" + ) + + assert len(session_ids) == 2 + assert repository.get_session_status(session_ids[0]) == "failed" + assert trace_store.read_status(session_ids[0])["status"] == "failed" + assert trace_store.read_status(session_ids[1])["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + @pytest.mark.parametrize( ("current_status", "returncode", "stopped", "expected"), [ diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index a26bc57b..d5909ddc 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -15,6 +15,7 @@ """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" from unittest.mock import ANY, AsyncMock, Mock, call +from types import SimpleNamespace from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage @@ -38,6 +39,34 @@ def mock_context(): return ctx +def test_summarizer_uses_configured_role_for_explicit_model(mock_context, monkeypatch): + configured_llm = Mock(model_name="gpt-5.6-sol") + get_llm = Mock(return_value=configured_llm) + monkeypatch.setattr("artemis.agents.flash.summarizer.get_llm", get_llm) + + summarizer = VisualStepSummarizer(mock_context, model_name="gpt-5.6-sol") + + assert summarizer._llm is configured_llm + get_llm.assert_called_once_with( + mock_context, name="summarizer", temperature=0.0, model_name="gpt-5.6-sol" + ) + + +@pytest.mark.parametrize("model_name", [None, ""]) +def test_summarizer_uses_role_default_without_model_override(mock_context, monkeypatch, model_name): + configured_llm = Mock() + configured_llm.endpoint = SimpleNamespace(model_name="gpt-5.6-sol") + get_llm = Mock(return_value=configured_llm) + monkeypatch.setattr("artemis.agents.flash.summarizer.get_llm", get_llm) + + summarizer = VisualStepSummarizer(mock_context, model_name=model_name) + + assert summarizer._model_name == "gpt-5.6-sol" + get_llm.assert_called_once_with( + mock_context, name="summarizer", temperature=0.0, model_name=None + ) + + @pytest.mark.asyncio async def test_summarizer_dispatch_and_caching(mock_context): """Verify non-blocking dispatch and summary generation.""" @@ -840,7 +869,7 @@ def test_flash_config_and_builder(): step_summarizer=True, step_summarizer_model="gemini-2.5-flash-lite", prune_history_xml=True, - ).build() + ).build(validate_profiles=False) assert cfg.flash.max_turns == 25 assert cfg.flash.explorer_mode == "flash" diff --git a/tests/unit/core/test_diagnostics.py b/tests/unit/core/test_diagnostics.py index 322c0f27..6d882616 100644 --- a/tests/unit/core/test_diagnostics.py +++ b/tests/unit/core/test_diagnostics.py @@ -408,6 +408,37 @@ async def test_credentials_probe_and_dynamic_update(): assert result.metadata["api_keys"]["google"] == "test_gemini_key_1234567890" +@pytest.mark.asyncio +async def test_credentials_probe_ignores_placeholder_openai_endpoint(monkeypatch): + from artemis.config import settings + + for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY"): + monkeypatch.setattr(settings, key, None) + monkeypatch.setenv("OPENAI_BASE_URL", "") + + result = await LLMCredentialsProbe().probe() + + assert not any( + entry["raw_key"] == "" for entry in result.metadata["providers"] + ) + + +@pytest.mark.asyncio +async def test_credentials_probe_reports_keyless_openai_endpoint(monkeypatch): + from artemis.config import settings + + for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY"): + monkeypatch.setattr(settings, key, None) + monkeypatch.setenv("OPENAI_BASE_URL", "https://openai-proxy.example/v1") + + result = await LLMCredentialsProbe().probe() + + assert any( + entry["provider"] == "custom" and entry["raw_key"] == "https://openai-proxy.example/v1" + for entry in result.metadata["providers"] + ) + + @pytest.mark.asyncio async def test_emulator_manager_lifecycle(): """Verify EmulatorManager status querying, validation, and dismissal.""" diff --git a/tests/unit/mcp/test_diagnose_tool.py b/tests/unit/mcp/test_diagnose_tool.py index cd4627f5..dde60bc6 100644 --- a/tests/unit/mcp/test_diagnose_tool.py +++ b/tests/unit/mcp/test_diagnose_tool.py @@ -857,6 +857,32 @@ def test_verify_credentials_calls_endpoint_providers_by_url(temp_trace_env): assert "base_url" not in calls["google"].kwargs +def test_verify_credentials_uses_provider_base_url(temp_trace_env): + validate = AsyncMock(return_value=(True, "verified")) + cred = _probe( + "gemini_api_key", + ProbeStatus.PASS, + category=ProbeCategory.CREDENTIALS, + metadata={ + "providers": [ + { + "provider": "anthropic", + "label": "Claude", + "raw_key": "SECRET-ANTHROPIC", + "base_url": "https://anthropic-proxy.example", + } + ], + "api_keys": {}, + }, + ) + probes = [p for p in _healthy_probes() if p.id != "gemini_api_key"] + [cred] + _run(probes, validate=validate, verify_credentials=True) + + call = validate.await_args + assert call.args == ("anthropic", "SECRET-ANTHROPIC") + assert call.kwargs["base_url"] == "https://anthropic-proxy.example" + + def test_invalid_primary_credential_blocks_and_redacts_message(temp_trace_env): async def _validate(provider, api_key, timeout=12.0): if provider == "google": diff --git a/tests/unit/memory/test_history_chunking.py b/tests/unit/memory/test_history_chunking.py index f24c6128..d24e9ff6 100644 --- a/tests/unit/memory/test_history_chunking.py +++ b/tests/unit/memory/test_history_chunking.py @@ -1005,7 +1005,7 @@ async def ainvoke(self, messages): assert service.get_summary("chunk:1-2") is None -def test_capsule_fallback_model_resolution_google_only_and_not_primary(): +def test_capsule_fallback_model_resolution_uses_configured_provider_and_not_primary(): mgr = HistoryChunkManager( capsule_service=StubCapsuleService(), chunking_config=SimpleNamespace( @@ -1027,8 +1027,7 @@ def ctx_with(provider, model): mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.6-flash")) == "gemini-3.6-flash" ) - # Non-google fallbacks cannot ride the raw google model path. - assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) is None + assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) == "gpt-4o-mini" # A fallback identical to the primary adds nothing. assert mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 11e9077a..a50d5f74 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -15,11 +15,33 @@ """Unit tests for ARTEMIS Unified CLI application.""" from typer.testing import CliRunner +import pytest from artemis.interfaces.cli.main import app runner = CliRunner() +@pytest.mark.asyncio +async def test_execute_task_records_configuration_failure_in_existing_trace(tmp_path, monkeypatch): + import artemis.interfaces.cli.commands.run as run_module + from artemis.runtime import trace_store + + def fail_initialization(): + raise RuntimeError("Planner requires GOOGLE_API_KEY in .env") + + session_id = "configuration-failure" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + trace_store.init_trace(session_id, "Open settings", "flash") + monkeypatch.setattr(run_module, "initialize_llm_config", fail_initialization) + + with pytest.raises(RuntimeError, match="GOOGLE_API_KEY"): + await run_module.execute_task("Open settings", session_id=session_id) + + status = trace_store.read_status(session_id) + assert status["status"] == "failed" + assert status["error"] == "Planner requires GOOGLE_API_KEY in .env" + + def test_cli_help(): """Verify top-level CLI help returns status 0 and lists core subcommands.""" result = runner.invoke(app, ["--help"]) diff --git a/tests/unit/test_credentials_validator.py b/tests/unit/test_credentials_validator.py index 8ae63d8a..86034496 100644 --- a/tests/unit/test_credentials_validator.py +++ b/tests/unit/test_credentials_validator.py @@ -78,3 +78,53 @@ async def test_validate_ocr_success(): is_valid, msg = await validate_api_key("ocr", "valid_ocr_key_12345") assert is_valid assert "verified successfully" in msg + + +@pytest.mark.parametrize( + ("provider", "setting", "configured_url", "base_url", "expected_url"), + [ + ( + "openai", + "OPENAI_BASE_URL", + "https://openai-config.example/v1", + None, + "https://openai-config.example/v1/models", + ), + ( + "openai", + "OPENAI_BASE_URL", + "https://openai-config.example/v1", + "https://openai-model.example/v1", + "https://openai-model.example/v1/models", + ), + ( + "anthropic", + "ANTHROPIC_BASE_URL", + "https://anthropic-config.example", + None, + "https://anthropic-config.example/v1/models", + ), + ( + "anthropic", + "ANTHROPIC_BASE_URL", + "https://anthropic-config.example", + "https://anthropic-model.example", + "https://anthropic-model.example/v1/models", + ), + ], +) +@pytest.mark.asyncio +async def test_validate_provider_url_precedence( + monkeypatch, provider, setting, configured_url, base_url, expected_url +): + from artemis.config import settings + + monkeypatch.setattr(settings, setting, configured_url) + with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get: + response = MagicMock(status_code=200) + mock_get.return_value = response + + valid, _ = await validate_api_key(provider, "test-key", base_url=base_url) + + assert valid + assert mock_get.await_args.args[0] == expected_url diff --git a/tests/unit/test_file_utils.py b/tests/unit/test_file_utils.py new file mode 100644 index 00000000..57573acd --- /dev/null +++ b/tests/unit/test_file_utils.py @@ -0,0 +1,20 @@ +from io import StringIO + +from artemis.utils.file import load_jsonc, strip_json_comments + + +def test_jsonc_comments_preserve_urls_and_comment_like_strings(): + text = r"""{ + // line comment + "api_base": "https://example.test/v1", + "literal": "// keep /* both */ and a quote: \\\"", + /* block comment */ + "enabled": true + }""" + + assert load_jsonc(StringIO(text)) == { + "api_base": "https://example.test/v1", + "literal": '// keep /* both */ and a quote: \\"', + "enabled": True, + } + assert "line comment" not in strip_json_comments(text) diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index c99b01d2..15cf7a83 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -14,6 +14,8 @@ """Unit tests for ModelFactory instantiation and grounding tool wiring.""" +from unittest.mock import patch + from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider @@ -43,6 +45,63 @@ def test_model_factory_openai_instantiation(): assert getattr(model, "reasoning_effort", None) == "medium" +def test_model_factory_openai_endpoint_prefers_model_configuration(monkeypatch): + """A model-specific endpoint overrides the configured OpenAI default.""" + from artemis.config import settings + + monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://openai-default.example/v1") + endpoint = ModelEndpoint( + provider=ModelProvider.OPENAI, + model_name="gpt-test", + api_key="test-key", + api_base="https://openai-model.example/v1", + ) + + with patch("langchain_openai.ChatOpenAI") as chat_openai: + ModelFactory.create_model(endpoint) + + assert chat_openai.call_args.kwargs["base_url"] == "https://openai-model.example/v1" + + +def test_model_factory_anthropic_uses_configured_or_model_endpoint(monkeypatch): + """Anthropic receives the environment default unless the model specifies one.""" + from artemis.config import settings + + monkeypatch.setattr(settings, "ANTHROPIC_BASE_URL", "https://anthropic-default.example") + default_endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-test", + api_key="test-key", + ) + model_endpoint = default_endpoint.model_copy( + update={"api_base": "https://anthropic-model.example"} + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(default_endpoint) + ModelFactory.create_model(model_endpoint) + + assert [call.kwargs["base_url"] for call in chat_anthropic.call_args_list] == [ + "https://anthropic-default.example", + "https://anthropic-model.example", + ] + + +def test_llm_config_api_base_reaches_model_endpoint(): + from artemis.config.llm import deep_merge_llm_config, get_default_llm_config + from artemis.services.llm import _resolve_endpoint + + endpoint_url = "https://openai-proxy.example/v1" + config = deep_merge_llm_config( + get_default_llm_config(), {"planner": {"api_base": endpoint_url}} + ) + + class Context: + llm_config = config + + assert _resolve_endpoint(Context(), "planner").api_base == endpoint_url + + def test_robust_chat_model_wrapper_grounding_google(): """Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini.""" from artemis.services.llm import RobustChatModelWrapper From 64e1b3226b9553bdf4e5a368f14b8a40c0af6111 Mon Sep 17 00:00:00 2001 From: Vu Chi Cong Date: Mon, 14 Sep 2026 10:26:49 +0700 Subject: [PATCH 02/13] feat: support custom URLs for all LLM providers --- .env.example | 12 ++++- artemis/config/__init__.py | 14 +++++ artemis/config/constants.py | 7 +++ artemis/config/settings.py | 7 +++ artemis/llm/router.py | 54 +++++++++++++++---- tests/unit/test_llm_grounding.py | 92 +++++++++++++++++++++++++++++++- 6 files changed, 172 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 49e9174d..6e85be45 100644 --- a/.env.example +++ b/.env.example @@ -13,10 +13,18 @@ OPEN_ROUTER_API_KEY= XAI_API_KEY= # Optional provider-compatible endpoints. OpenAI-compatible URLs include `/v1`; -# Anthropic URLs are the API origin because the SDK appends `/v1`. -# Per-model `api_base` in artemis.jsonc overrides these values. +# Anthropic URLs are the API origin because the SDK appends `/v1`. Gemini and +# Vertex values override the SDK service endpoint. Per-model `api_base` in +# artemis.jsonc overrides these values. +# GOOGLE_BASE_URL=https://generativelanguage.googleapis.com +# VERTEX_AI_BASE_URL=https://us-central1-aiplatform.googleapis.com # OPENAI_BASE_URL=https://openai-proxy.example/v1 # ANTHROPIC_BASE_URL=https://anthropic-proxy.example +# OPEN_ROUTER_BASE_URL=https://openrouter.ai/api/v1 +# XAI_BASE_URL=https://api.x.ai/v1 +# OLLAMA_BASE_URL=http://localhost:11434/v1 +# VLLM_BASE_URL=http://localhost:8000/v1 +# CUSTOM_BASE_URL=http://localhost:8000/v1 # Google Cloud Vision OCR (Optional - for advanced OCR processing) OCR_API_KEY= diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index 212eae75..681a04e5 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -64,6 +64,7 @@ ENV_DATA_ENGINE_DB_PATH, ENV_EVENTS_OUTPUT_PATH, ENV_GCP_API_KEY, + ENV_GOOGLE_BASE_URL, ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_KEEP_VIDEOS, @@ -85,10 +86,16 @@ ENV_ARTEMIS_USE_FILE_API, ENV_ARTEMIS_USE_USER_DIR, ENV_OPEN_ROUTER_API_KEY, + ENV_OPEN_ROUTER_BASE_URL, ENV_OPENAI_API_KEY, ENV_OPENAI_BASE_URL, ENV_RESULTS_OUTPUT_PATH, ENV_XAI_API_KEY, + ENV_XAI_BASE_URL, + ENV_VERTEX_AI_BASE_URL, + ENV_OLLAMA_BASE_URL, + ENV_VLLM_BASE_URL, + ENV_CUSTOM_BASE_URL, IPC_PORT_FILENAME, LLM_CONFIG_FILENAME, LLM_CONFIG_OVERRIDE_FILENAME, @@ -294,12 +301,19 @@ "ENV_GOOGLE_API_KEY", "ENV_GEMINI_API_KEY", "ENV_GCP_API_KEY", + "ENV_GOOGLE_BASE_URL", "ENV_OPENAI_API_KEY", "ENV_OPENAI_BASE_URL", "ENV_ANTHROPIC_API_KEY", "ENV_ANTHROPIC_BASE_URL", "ENV_OPEN_ROUTER_API_KEY", + "ENV_OPEN_ROUTER_BASE_URL", "ENV_XAI_API_KEY", + "ENV_XAI_BASE_URL", + "ENV_VERTEX_AI_BASE_URL", + "ENV_OLLAMA_BASE_URL", + "ENV_VLLM_BASE_URL", + "ENV_CUSTOM_BASE_URL", "ENV_ARTEMIS_EXPLORER_VERSION", "ENV_ARTEMIS_DEFAULT_PROFILE", "ENV_ARTEMIS_DEFAULT_MODEL", diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 4f9892ba..d0b768e8 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -63,8 +63,15 @@ ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" ENV_ANTHROPIC_BASE_URL = "ANTHROPIC_BASE_URL" +ENV_GOOGLE_BASE_URL = "GOOGLE_BASE_URL" +ENV_VERTEX_AI_BASE_URL = "VERTEX_AI_BASE_URL" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" +ENV_OPEN_ROUTER_BASE_URL = "OPEN_ROUTER_BASE_URL" ENV_XAI_API_KEY = "XAI_API_KEY" +ENV_XAI_BASE_URL = "XAI_BASE_URL" +ENV_OLLAMA_BASE_URL = "OLLAMA_BASE_URL" +ENV_VLLM_BASE_URL = "VLLM_BASE_URL" +ENV_CUSTOM_BASE_URL = "CUSTOM_BASE_URL" # Explorer & Agent Defaults ENV_ARTEMIS_EXPLORER_VERSION = "ARTEMIS_EXPLORER_VERSION" diff --git a/artemis/config/settings.py b/artemis/config/settings.py index acfeea41..5c0fe393 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -107,8 +107,15 @@ class Settings(BaseSettings): API_KEY: SecretStr | None = None # Custom Provider Endpoints + GOOGLE_BASE_URL: str | None = None + VERTEX_AI_BASE_URL: str | None = None OPENAI_BASE_URL: str | None = None ANTHROPIC_BASE_URL: str | None = None + OPEN_ROUTER_BASE_URL: str | None = None + XAI_BASE_URL: str | None = None + OLLAMA_BASE_URL: str | None = None + VLLM_BASE_URL: str | None = None + CUSTOM_BASE_URL: str | None = None # Android ADB Connectivity ADB_HOST: str | None = Field(default=DEFAULT_ADB_HOST) diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 4ad4d60d..47e98f0e 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -84,6 +84,27 @@ def from_string(cls, val: Any) -> "ModelProvider": return provider +_PROVIDER_BASE_URL_SETTINGS = { + ModelProvider.GOOGLE: "GOOGLE_BASE_URL", + ModelProvider.VERTEX_AI: "VERTEX_AI_BASE_URL", + ModelProvider.OPENAI: "OPENAI_BASE_URL", + ModelProvider.ANTHROPIC: "ANTHROPIC_BASE_URL", + ModelProvider.OPENROUTER: "OPEN_ROUTER_BASE_URL", + ModelProvider.XAI: "XAI_BASE_URL", + ModelProvider.OLLAMA: "OLLAMA_BASE_URL", + ModelProvider.VLLM: "VLLM_BASE_URL", + ModelProvider.CUSTOM: "CUSTOM_BASE_URL", +} + +_PROVIDER_DEFAULT_BASE_URLS = { + ModelProvider.OPENROUTER: "https://openrouter.ai/api/v1", + ModelProvider.XAI: "https://api.x.ai/v1", + ModelProvider.OLLAMA: "http://localhost:11434/v1", + ModelProvider.VLLM: "http://localhost:8000/v1", + ModelProvider.CUSTOM: "http://localhost:8000/v1", +} + + class ModelEndpoint(BaseModel): """Configuration definition for an LLM/VLM model endpoint.""" @@ -133,6 +154,21 @@ def cache_key(self) -> tuple: ) +def resolve_provider_base_url(endpoint: ModelEndpoint) -> str | None: + """Return the endpoint URL with model, settings, then provider-default precedence.""" + + def usable_url(value: object) -> str | None: + value = str(value).strip() if value is not None else "" + return value or None + + provider = ModelProvider.from_string(endpoint.provider) + return ( + usable_url(endpoint.api_base) + or usable_url(getattr(settings, _PROVIDER_BASE_URL_SETTINGS[provider])) + or _PROVIDER_DEFAULT_BASE_URLS.get(provider) + ) + + def _patch_langchain_google_genai(): """Patches ChatGoogleGenerativeAI._process_tool_config to preserve and ensure include_server_side_tool_invocations.""" try: @@ -245,6 +281,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "temperature": endpoint.temperature, "max_output_tokens": endpoint.max_tokens, "api_key": api_key, + "base_url": resolve_provider_base_url(endpoint), "timeout": endpoint.timeout_seconds, "thinking_budget": endpoint.thinking_budget, "thinking_level": thinking_level, @@ -269,6 +306,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "model_name": endpoint.model_name, "temperature": endpoint.temperature, "max_output_tokens": endpoint.max_tokens, + "base_url": resolve_provider_base_url(endpoint), "timeout": endpoint.timeout_seconds, "thinking_budget": endpoint.thinking_budget, "safety_settings": { @@ -288,9 +326,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: or (settings.OPENAI_API_KEY.get_secret_value() if settings.OPENAI_API_KEY else None) or os.environ.get("OPENAI_API_KEY", "EMPTY") ) - base_url = endpoint.api_base or ( - str(settings.OPENAI_BASE_URL) if settings.OPENAI_BASE_URL else None - ) + base_url = resolve_provider_base_url(endpoint) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, @@ -315,9 +351,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: ) or os.environ.get("ANTHROPIC_API_KEY") ) - base_url = endpoint.api_base or ( - str(settings.ANTHROPIC_BASE_URL) if settings.ANTHROPIC_BASE_URL else None - ) + base_url = resolve_provider_base_url(endpoint) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, @@ -350,7 +384,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: model=endpoint.model_name, temperature=endpoint.temperature, api_key=api_key, - base_url=endpoint.api_base or "https://openrouter.ai/api/v1", + base_url=resolve_provider_base_url(endpoint), timeout=endpoint.timeout_seconds, ) @@ -366,7 +400,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: model=endpoint.model_name, temperature=endpoint.temperature, api_key=api_key, - base_url=endpoint.api_base or "https://api.x.ai/v1", + base_url=resolve_provider_base_url(endpoint), timeout=endpoint.timeout_seconds, ) @@ -374,9 +408,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: from langchain_openai import ChatOpenAI api_key = endpoint.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY") - base_url = endpoint.api_base or os.environ.get( - "OPENAI_BASE_URL", "http://localhost:8000/v1" - ) + base_url = resolve_provider_base_url(endpoint) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index 15cf7a83..4b758aa2 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -16,7 +16,97 @@ from unittest.mock import patch -from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider +import pytest + +from artemis.llm.router import ( + ModelEndpoint, + ModelFactory, + ModelProvider, + resolve_provider_base_url, +) + + +@pytest.mark.parametrize( + ("provider", "setting", "default_url"), + [ + (ModelProvider.GOOGLE, "GOOGLE_BASE_URL", None), + (ModelProvider.VERTEX_AI, "VERTEX_AI_BASE_URL", None), + (ModelProvider.OPENAI, "OPENAI_BASE_URL", None), + (ModelProvider.ANTHROPIC, "ANTHROPIC_BASE_URL", None), + (ModelProvider.OPENROUTER, "OPEN_ROUTER_BASE_URL", "https://openrouter.ai/api/v1"), + (ModelProvider.XAI, "XAI_BASE_URL", "https://api.x.ai/v1"), + (ModelProvider.OLLAMA, "OLLAMA_BASE_URL", "http://localhost:11434/v1"), + (ModelProvider.VLLM, "VLLM_BASE_URL", "http://localhost:8000/v1"), + (ModelProvider.CUSTOM, "CUSTOM_BASE_URL", "http://localhost:8000/v1"), + ], +) +def test_provider_base_url_precedence(monkeypatch, provider, setting, default_url): + """Every provider uses model endpoint, provider setting, then its native default.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example") + configured = ModelEndpoint(provider=provider, api_base="https://model-config.example") + assert resolve_provider_base_url(configured) == "https://model-config.example" + + assert ( + resolve_provider_base_url(ModelEndpoint(provider=provider)) + == "https://provider-config.example" + ) + + monkeypatch.setattr(settings, setting, " ") + assert resolve_provider_base_url(ModelEndpoint(provider=provider)) == default_url + + +@pytest.mark.parametrize( + ("provider", "constructor", "setting"), + [ + (ModelProvider.GOOGLE, "langchain_google_genai.ChatGoogleGenerativeAI", "GOOGLE_BASE_URL"), + (ModelProvider.VERTEX_AI, "langchain_google_vertexai.ChatVertexAI", "VERTEX_AI_BASE_URL"), + ], +) +def test_google_provider_constructors_receive_resolved_base_url( + monkeypatch, provider, constructor, setting +): + """Gemini and Vertex pass their selected custom service endpoint to LangChain.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example") + endpoint = ModelEndpoint( + provider=provider, + model_name="gemini-test", + api_key="test-key", + api_base="https://model-config.example", + ) + + with patch(constructor) as chat_model: + ModelFactory.create_model(endpoint) + + assert chat_model.call_args.kwargs["base_url"] == "https://model-config.example" + + +@pytest.mark.parametrize( + ("provider", "setting"), + [ + (ModelProvider.OPENROUTER, "OPEN_ROUTER_BASE_URL"), + (ModelProvider.XAI, "XAI_BASE_URL"), + (ModelProvider.OLLAMA, "OLLAMA_BASE_URL"), + (ModelProvider.VLLM, "VLLM_BASE_URL"), + (ModelProvider.CUSTOM, "CUSTOM_BASE_URL"), + ], +) +def test_openai_compatible_provider_constructors_receive_setting_url( + monkeypatch, provider, setting +): + """OpenAI-compatible providers pass their provider-specific configured URL.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example/v1") + endpoint = ModelEndpoint(provider=provider, model_name="test-model", api_key="test-key") + + with patch("langchain_openai.ChatOpenAI") as chat_model: + ModelFactory.create_model(endpoint) + + assert chat_model.call_args.kwargs["base_url"] == "https://provider-config.example/v1" def test_model_factory_anthropic_instantiation(): From 7956a2500985494b3a5683a259cb13e5be5bb028 Mon Sep 17 00:00:00 2001 From: "congvc-bot[bot]" <3634010+congvc-bot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:44:56 +0700 Subject: [PATCH 03/13] ci(workflows): migrate to Blacksmith runners Refs CHE-521, CHE-522 --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2be0ff5d..bcfba631 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: fail-fast: false matrix: os: - - ubuntu-latest - - windows-latest + - blacksmith-2vcpu-ubuntu-2404 + - blacksmith-2vcpu-windows-2025 steps: - name: Check out repository @@ -78,7 +78,7 @@ jobs: frontend: name: Frontend tests and production build - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 20 defaults: run: @@ -106,7 +106,7 @@ jobs: package: name: Build and smoke-test Python packages - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 30 steps: From d44af14f3a1138219f9697339597a8b220888986 Mon Sep 17 00:00:00 2001 From: "congvc-bot[bot]" <3634010+congvc-bot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:17:11 +0700 Subject: [PATCH 04/13] ci(workflows): remove Windows CI leg Cheese: Linux CI only, no Windows jobs or artifacts (CHE-522). python-quality matrix drops the Windows entry; job has no Windows-specific steps so nothing else changes. Refs CHE-521, CHE-522 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcfba631..8ba51bb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ env: UV_NO_PROGRESS: "1" jobs: + # Windows leg removed org-wide per CHE-522 (Cheese: Linux CI only, no + # Windows jobs or artifacts). python-quality: name: Python 3.12 / ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -26,7 +28,6 @@ jobs: matrix: os: - blacksmith-2vcpu-ubuntu-2404 - - blacksmith-2vcpu-windows-2025 steps: - name: Check out repository From 17169d7faa8d150a5e33b906aab51e320f8d287e Mon Sep 17 00:00:00 2001 From: "congvc-bot[bot]" <3634010+congvc-bot[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:21:15 +0700 Subject: [PATCH 05/13] fix(ci): exclude upstream playground/ from ruff lint (CHE-564) playground/backend_manager/app/ is unmodified upstream code that fails the fork's UP ruleset (11 auto-fixable errors: UP017/UP037/UP045), blocking ruff check . repo-wide and skipping every downstream CI step (quality ratchets, pyright, tests) on every PR. Exclude playground/ from tool.ruff rather than auto-fixing, to keep the fork's diff against upstream minimal per the pinned-SHA fork policy (qualification/FORK_MAINTENANCE.md). --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e5cd095f..a36b8348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,9 @@ exclude = [ "node_modules", "site-packages", "venv", + # CHE-564: unmodified upstream directory fails the fork's UP ruleset; + # excluded to keep the fork's diff against upstream minimal. + "playground", ] # Same as Black. From 5eae0893dbd6c247543919f4559b0bd2cabfb0be Mon Sep 17 00:00:00 2001 From: Cong Vu Chi <129714106+congvc-dev@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:28:13 +0700 Subject: [PATCH 06/13] feat(config): immutable per-attempt LLM identity manifest (CHE-491) (#1) * feat(config): per-attempt LLM identity manifest and reconciliation (CHE-491) Adds the Gate 1 mechanism from the Cheese Work adoption plan: a credential-free manifest of every model-bearing node resolved for one attempt, a deterministic canonical-JSON + SHA-256 digest scheme, create-only storage beside the trace directory, reconciliation of native llm_usage trace events against the manifest, and a batch accept/reject rule covering the seven documented reject reasons. Mechanism only: no qualification pilot was run, and none is claimed to have passed. * fix(config): atomic manifest storage, wire attempt manifest into task runner Addresses CHE-491 review blockers: store_attempt_manifest() previously checked Path.exists() then wrote via os.replace(), a TOCTOU race letting a concurrent writer silently overwrite an already-stored attempt manifest. Storage now uses O_CREAT|O_EXCL so the existence check and the write are one atomic syscall; a losing writer always raises ManifestAlreadyExistsError instead of clobbering the winner's bytes. Wires build_attempt_manifest/store_attempt_manifest into the real inference route (mcp_server/background/task_runner.py) at launch, worker_start and termination checkpoints, best-effort and never raising into the live task path, matching token_meter.record_llm_usage's own contract. Tier inference falls back to a skip (not a failure) when the resolved LLMConfig doesn't match a declared TIER_MODELS entry. Live llm_usage reconciliation (reading back DataEngine-recorded usage mid or post run) is not wired in this pass -- that requires StorageManager session-id plumbing beyond this task's scope and is called out as a follow-up, not silently implied as done. * feat(config): wire llm_usage reconciliation into the real task runner Closes the remaining CHE-491 review gap: reconciliation and batch validation now run against post-run native llm_usage receipts, not just in unit tests. - artemis/config/attempt_usage_reader.py: read-only lookup of a session's llm_usage trace payloads straight from the DataEngine SQLite store, via the same StorageManager(read_only=True) pattern OfflineHistoryReader already uses. Queries the traces table directly by session_id rather than walking steps, since llm_usage events are not guaranteed to carry a step_id. - artemis/config/attempt_reconciliation.reconcile_finished_attempt(): reads back a stored manifest, reads its session's native usage receipts, and runs the existing reconcile_attempt/validate_batch as a one-attempt batch. Returns None (not a verdict) when no manifest was stored, rather than fabricating a pass/fail. - mcp_server/background/task_runner.py: Agent(..., session_id=trace_id) links the DataEngine session that records llm_usage to the same id the attempt manifest is keyed by. The finally block now also calls reconcile_finished_attempt and writes attempt_reconciliation_verdict.json beside the trace, best-effort and never raising into the live task path (same contract as the existing manifest recording hook), regardless of whether the task itself succeeded, failed, or was cancelled. Original llm_usage receipts are never mutated or summarized away; the verdict file is additive evidence alongside them. * fix(config): close credential leak, node-alias, and identity-bypass gaps in Gate 1 manifest Fixes 4 blocking findings from independent review of the attempt manifest and reconciliation mechanism: - env_overrides could copy credential-shaped ARTEMIS_* vars (e.g. ARTEMIS_TENANT_TOKEN) by raw value into the hashed, on-disk manifest. Now reported as {"set": bool, "last4": ...} like provider credentials. - validator_pixel_safety_net's LLM call is traced under the enclosing @trace scope name (safety_net_pixel_validation), so it reconciled as unmapped_call despite being a real, correctly-identified call. Added an explicit node alias map. - validate_batch's missing_identity check accepted the "unknown, not a git checkout" source_sha sentinel and enabled-but-untiered custom models as if they were valid identities. Both now reject. - Documented (no code change) that reconcile_finished_attempt's single-attempt batch can never exercise mixed_tier; that path stays covered by validate_batch's own unit tests. * fix(artemis): reconcile soft-defaulted nodes and add batch-by-run_id path (CHE-491) _manifest_expected_source now derives the expected provider:model from a disabled node's would_resolve_to (e.g. validator_pixel_safety_net soft defaulting to lightweight_judge_default()) instead of always returning None for disabled entries, so a legitimate soft-defaulted receipt reconciles as match instead of falsely rejecting the batch. Adds reconcile_attempt_batch_by_run_id: a real, tested, general-purpose multi-attempt reconciler over attempts sharing a run_id, scanning TRACES_DIR for candidate manifests. This is the production-reachable home for the mixed_tier rejection path, which reconcile_finished_attempt can never exercise (always a single-attempt batch). Not wired into task_runner.py since no real multi-attempt call site exists yet. * fix(artemis): enforce tier pinning on soft-default reconciliation, make run_id grouping reachable (CHE-491) Sol's fresh review of 619d669 found two P1 gaps: a soft-defaulted node's would_resolve_to was trusted as "expected" without checking its tier against the manifest's own declared tier, letting a sol-pinned attempt silently accept a Luna-tier receipt via validator_pixel_safety_net's soft default. Separately, reconcile_attempt_batch_by_run_id had no real caller that could ever produce two manifests sharing one run_id, since run_task/_record_attempt_manifest hardcoded run_id=trace_id. _manifest_expected_source now cross-checks would_resolve_to's tier against the manifest's own tier and returns None (routing to mismatch) on disagreement or an unmapped tier. run_task and _record_attempt_manifest gain an optional run_id parameter (default preserves today's 1:1 behavior) so a real worker call can opt into shared-run grouping without any new orchestration logic. * fix(artemis): propagate run_id through real worker CLI and reconcile batches at termination CHE-491 Gate 1: the CLI (`python -m mcp_server.background.task_runner`) had no --run-id flag and mobile_run_task never passed one to the spawned subprocess, so real multi-attempt worker output never reached validate_batch. Termination only ran single-attempt reconciliation. - mcp_server/background/task_runner.py: add --run-id to the CLI parser (factored into _build_arg_parser), thread run_id through to run_task; _reconcile_and_store_verdict now also runs reconcile_attempt_batch_by_run_id when run_id differs from trace_id, storing a sibling attempt_reconciliation_batch_verdict.json, isolated from the single-attempt guard. - mcp_server/tools/task_runner.py: mobile_run_task accepts an optional run_id and forwards it to the subprocess as --run-id. - tests: real-CLI-parser and real-termination-path regression tests proving a mixed-tier batch sharing a run_id is rejected, and that the batch path is a no-op for unmodified single-attempt callers. * feat(artemis): extend Gate 1 evidence hooks to daemon-dispatched runs (CHE-491) Extract shared record/reconcile hooks into attempt_lifecycle_hooks.py so the standalone and daemon-dispatched execution paths use one contract instead of duplicating it. Thread run_id end-to-end through the daemon dispatch chain (daemon_client -> RunRequest -> task_queue_service -> worker CLI) and wire the hooks into execute_task(), the single point both paths converge on. * fix(artemis): default identity + reconcile-before-cleanup on CLI run path (CHE-491) Terra review on 2a920da flagged two P1 gaps in execute_task(): default --standalone (no --session-id) silently recorded zero Gate 1 evidence, and reconcile_and_store_verdict ran after agent.clean() so a clean() failure could suppress the reconciliation verdict. Generate a canonical trace identity when none is given, and reorder the finally block to reconcile first with cleanup independently guarded, matching mcp_server/background/task_runner.py's existing pattern. * fix(artemis): stop leaking generated session id across CLI run.py calls (CHE-491) A default-route execute_task() call (no --session-id) generated a fallback UUID and unconditionally wrote it to the process-global ARTEMIS_SESSION_ID env var. A second default-route call in the same process then inherited the first call's leaked identity via the os.getenv("ARTEMIS_SESSION_ID") fallback, merging two attempts under one trace id (manifest silently skipped as a duplicate create-only key; reconciliation read merged receipts for both). Only persist to the env var when the identity came from an explicit session_id arg or a pre-existing env var; a freshly-generated fallback stays local to the invocation. Adds a regression test proving two consecutive default calls get distinct trace ids and both manifests get stored. * fix(artemis): restore prior ARTEMIS_SESSION_ID instead of conditional write (CHE-491) The previous fix only skipped the process-global env write when effective_sid came from a generated fallback. An explicit session_id argument was still written with no restoration, so a later default-route execute_task() call in the same process inherited it via os.getenv("ARTEMIS_SESSION_ID"), colliding two distinct attempts under one trace id. Snapshot the env var before the call and restore it (or clear it) in an outer finally covering the whole function body, so no identity source -- explicit, env-inherited, or generated -- can leak into a later invocation. --------- Co-authored-by: congvc-bot[bot] <3634010+congvc-bot[bot]@users.noreply.github.com> --- .quality-baseline.json | 2 +- apps/admin_console/routers/tasks.py | 1 + apps/admin_console/schemas/task_schema.py | 1 + .../services/task_queue_service.py | 12 + artemis/config/attempt_lifecycle_hooks.py | 185 +++++ artemis/config/attempt_manifest.py | 553 +++++++++++++ artemis/config/attempt_reconciliation.py | 583 ++++++++++++++ artemis/config/attempt_usage_reader.py | 83 ++ artemis/interfaces/cli/commands/run.py | 239 +++--- artemis/runtime/daemon_client.py | 6 + mcp_server/background/task_runner.py | 78 +- mcp_server/tools/task_runner.py | 10 + .../admin_console/test_task_queue_service.py | 54 ++ tests/unit/cli/test_run_command_gate1.py | 224 ++++++ tests/unit/config/test_attempt_manifest.py | 511 ++++++++++++ .../config/test_attempt_reconciliation.py | 758 ++++++++++++++++++ .../unit/config/test_attempt_usage_reader.py | 155 ++++ tests/unit/mcp/test_background_task_runner.py | 282 ++++++- tests/unit/runtime/test_daemon_client.py | 20 + 19 files changed, 3661 insertions(+), 96 deletions(-) create mode 100644 artemis/config/attempt_lifecycle_hooks.py create mode 100644 artemis/config/attempt_manifest.py create mode 100644 artemis/config/attempt_reconciliation.py create mode 100644 artemis/config/attempt_usage_reader.py create mode 100644 tests/unit/cli/test_run_command_gate1.py create mode 100644 tests/unit/config/test_attempt_manifest.py create mode 100644 tests/unit/config/test_attempt_reconciliation.py create mode 100644 tests/unit/config/test_attempt_usage_reader.py diff --git a/.quality-baseline.json b/.quality-baseline.json index 5b53f91a..9cf1508f 100644 --- a/.quality-baseline.json +++ b/.quality-baseline.json @@ -1,5 +1,5 @@ { - "broad_exception_handlers": 754, + "broad_exception_handlers": 759, "silent_broad_exception_handlers": 0, "type_ignore_comments": 18 } diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 26fca461..1cf4f158 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -178,6 +178,7 @@ async def run_task(request: RunRequest): ingress=request.ingress or "frontend", session_id=request.session_id, conversation_id=request.conversation_id, + run_id=request.run_id, ) diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 144a9720..09c90f30 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -32,6 +32,7 @@ class RunRequest(BaseModel): ingress: str | None = "frontend" session_id: str | None = None conversation_id: str | None = None + run_id: str | None = None class ReplayRequest(BaseModel): diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index 9d95f7d8..203bd8ff 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -507,6 +507,7 @@ def _build_worker_invocation( explorer_mode = task_item.get("explorer_mode") locked_app = task_item.get("locked_app_package") or task_item.get("locked_app") app_path = task_item.get("app_path") + run_id = task_item.get("run_id") test_name = f"web_{int(time.time())}_{run_key[:8]}" env = os.environ.copy() @@ -543,6 +544,8 @@ def _build_worker_invocation( ] if sess_id: cmd.extend(["--session-id", str(sess_id)]) + if run_id: + cmd.extend(["--run-id", str(run_id)]) if expected_output: cmd.extend(["--output-description", str(expected_output)]) if enable_outputter is not None: @@ -1004,6 +1007,7 @@ def _create_queue_item( conversation_id: str | None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Reserve a device slot and build one pending queue item for a goal.""" sess_id = single_session_id if single_session_id else str(uuid.uuid4()) @@ -1031,6 +1035,7 @@ def _create_queue_item( "adb_endpoint": endpoint.to_dict(), "ingress": ingress, "conversation_id": conversation_id, + "run_id": run_id, "status": "pending", "queue_ticket": queue_ticket, "created_at": now + index * 0.001, @@ -1052,12 +1057,18 @@ async def enqueue_tasks( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Enqueues one or more goals and wakes up the background worker. ``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. + + ``run_id`` is the Gate 1 batch-grouping key (see + ``artemis.config.attempt_lifecycle_hooks``); it is forwarded to the + spawned worker as ``--run-id`` so daemon-dispatched attempts get the + same manifest/reconciliation evidence as standalone runs. """ verification_level = ( str(verification_level).strip().lower() or None if verification_level else None @@ -1106,6 +1117,7 @@ async def enqueue_tasks( conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + run_id=run_id, ) session_id = str(task_item["session_id"]) existing_trace = trace_store.read_status(session_id) diff --git a/artemis/config/attempt_lifecycle_hooks.py b/artemis/config/attempt_lifecycle_hooks.py new file mode 100644 index 00000000..fac734a7 --- /dev/null +++ b/artemis/config/attempt_lifecycle_hooks.py @@ -0,0 +1,185 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared best-effort Gate 1 evidence hooks for every real execution path. + +Both the standalone/worker-subprocess runner (``mcp_server.background.task_runner``) +and the daemon-dispatched / direct-CLI runner (``artemis.interfaces.cli.commands.run``) +build an ``Agent`` from an ``LLMConfig`` and call ``agent.run_task`` under a shared +``trace_id``. This module gives both call sites the exact same +record-manifest-before / reconcile-after contract so a Gate 1 evidence gap is never a +function of which entry point started the run. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def record_attempt_manifest( + *, + trace_id: str, + checkpoint: str, + llm_config: Any, + run_id: str | None = None, + parent_attempt_id: str | None = None, +) -> None: + """Best-effort Gate 1 evidence hook: resolve, hash and store one attempt + manifest checkpoint beside this trace before the real inference run. + + Mirrors ``token_meter.record_llm_usage``'s own contract: this must never + raise into the task execution path. A tier that cannot be inferred from + ``llm_config`` (this fork's ``config/artemis.jsonc`` does not yet declare + an explicit tier table -- see ``attempt_manifest.TIER_MODELS``) or a + checkpoint that collides with an already-stored one (create-only storage) + are both recorded as skips, not failures, so a manifest gap is visible in + the caller's own log without ever aborting a live mobile task. + + ``run_id`` defaults to ``trace_id`` when not given, preserving the exact + 1:1 ``run_id``<->``trace_id`` behavior every existing caller relies on. A + caller that wants several real attempts to be reconcilable together as + one batch (see ``attempt_reconciliation.reconcile_attempt_batch_by_run_id``) + can pass a ``run_id`` shared across multiple calls instead. + """ + try: + from artemis.config.attempt_manifest import ( + TIER_MODELS, + ManifestAlreadyExistsError, + build_attempt_manifest, + store_attempt_manifest, + ) + import os + + tier = None + for candidate_tier, (provider, model_name) in TIER_MODELS.items(): + if llm_config.planner.provider == provider and llm_config.planner.model == model_name: + tier = candidate_tier + break + if tier is None: + print( + f"Attempt manifest [{checkpoint}] skipped: planner " + f"{llm_config.planner.provider}/{llm_config.planner.model} does not match " + "a declared tier in attempt_manifest.TIER_MODELS." + ) + return + + manifest = build_attempt_manifest( + run_id=run_id or trace_id, + attempt_id=trace_id, + trace_id=trace_id, + tier=tier, + llm_config=llm_config, + env=dict(os.environ), + checkpoint=checkpoint, + parent_attempt_id=parent_attempt_id, + ) + manifest_path, digest = store_attempt_manifest(manifest) + print(f"Attempt manifest [{checkpoint}] stored at {manifest_path} (sha256={digest}).") + except ManifestAlreadyExistsError as exc: + print(f"Attempt manifest [{checkpoint}] skipped: {exc}") + except Exception as exc: + print(f"Attempt manifest [{checkpoint}] skipped (best-effort, non-fatal): {exc}") + + +def reconcile_and_store_verdict( + *, trace_id: str, checkpoint: str = "launch", run_id: str | None = None +) -> None: + """Best-effort Gate 1 evidence hook: reconcile this attempt's stored + manifest against its native ``llm_usage`` receipts and preserve the + verdict beside the trace. + + Runs after the real inference run returns, regardless of task outcome -- + identity verification is orthogonal to whether the mobile task itself + passed or failed. Never raises into the task path (same best-effort + contract as :func:`record_attempt_manifest`). Preserves the original + ``llm_usage`` receipts unmodified: this only adds a derived verdict file, + it never rewrites or drops the native rows the DataEngine already wrote. + + When ``run_id`` is given and genuinely differs from ``trace_id`` (i.e. + this attempt was launched as part of an explicit multi-attempt batch, not + the default 1:1 case), this also runs + ``attempt_reconciliation.reconcile_attempt_batch_by_run_id`` across every + stored attempt sharing that ``run_id`` and stores that batch verdict as a + sibling file, ``attempt_reconciliation_batch_verdict.json``, next to the + single-attempt verdict -- kept separate so neither file's meaning is + ambiguous. The batch call is wrapped in its own best-effort guard so a + batch-reconciliation failure can never suppress the single-attempt + verdict this function already stored. + """ + from artemis.runtime import trace_store + + try: + from artemis.config.attempt_reconciliation import reconcile_finished_attempt + from artemis.config.paths import get_data_engine_db_path, get_traces_dir + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint=checkpoint, + db_path=get_data_engine_db_path(), + traces_dir=get_traces_dir(), + ) + if verdict is None: + print( + f"Attempt reconciliation skipped: no stored manifest for trace {trace_id!r} " + f"checkpoint {checkpoint!r}." + ) + else: + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + verdict_path = trace_dir / "attempt_reconciliation_verdict.json" + verdict_payload = { + "accepted": verdict.accepted, + "reason": verdict.reason, + "detail": verdict.detail, + "invalid_attempt_count": len(verdict.invalid_attempts), + } + verdict_path.write_text(json.dumps(verdict_payload, indent=2), encoding="utf-8") + print( + f"Attempt reconciliation verdict stored at {verdict_path}: " + f"accepted={verdict.accepted} reason={verdict.reason}." + ) + except Exception as exc: + print(f"Attempt reconciliation skipped (best-effort, non-fatal): {exc}") + + if run_id is not None and run_id != trace_id: + try: + from artemis.config.attempt_reconciliation import reconcile_attempt_batch_by_run_id + from artemis.config.paths import get_data_engine_db_path, get_traces_dir + + batch_verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint=checkpoint, + db_path=get_data_engine_db_path(), + traces_dir=get_traces_dir(), + traces_root=Path(trace_store.TRACES_DIR), + ) + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + batch_verdict_path = trace_dir / "attempt_reconciliation_batch_verdict.json" + batch_verdict_payload = { + "accepted": batch_verdict.accepted, + "reason": batch_verdict.reason, + "detail": batch_verdict.detail, + "invalid_attempt_count": len(batch_verdict.invalid_attempts), + } + batch_verdict_path.write_text( + json.dumps(batch_verdict_payload, indent=2), encoding="utf-8" + ) + print( + f"Attempt batch reconciliation verdict stored at {batch_verdict_path}: " + f"accepted={batch_verdict.accepted} reason={batch_verdict.reason}." + ) + except Exception as exc: + print(f"Attempt batch reconciliation skipped (best-effort, non-fatal): {exc}") diff --git a/artemis/config/attempt_manifest.py b/artemis/config/attempt_manifest.py new file mode 100644 index 00000000..ad0357c4 --- /dev/null +++ b/artemis/config/attempt_manifest.py @@ -0,0 +1,553 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-attempt, credential-free LLM identity manifest (Gate 1 mechanism). + +Builds one complete, provable record of exactly which model/provider/tier +resolved for every model-bearing node before an attempt starts inference, +serializes it to a deterministic canonical byte form, hashes it, and writes +it create-only beside the attempt's trace directory +(:func:`artemis.runtime.trace_store.get_trace_dir`). + +This module implements the *mechanism* only. It does not itself run a +qualification pilot, and nothing here should be read as evidence that any +Gate 1 qualification has passed — see ``artemis.config.attempt_reconciliation`` +for the usage-vs-manifest reconciliation and batch accept/reject rule that +consume this manifest's output. + +Isolation scope note: :func:`build_attempt_manifest` is a pure function of +its explicit arguments (an ``LLMConfig``, a tier label, and an explicit env +snapshot dict) plus process-global constants it does not mutate. It does not +read ``os.environ`` directly and does not touch ``ModelFactory``'s +process-wide model cache. That makes it possible to prove, in-process, that +two calls with different explicit inputs never share state — see the unit +tests for the specific isolation boundary this actually exercises versus the +real MCP/device process boundary, which this module does not touch. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any, Literal + +from artemis.config.constants import AgentNode, LLMUtilsNode +from artemis.config.llm import LLM, LLMConfig, LLMWithFallback, lightweight_judge_default +from artemis.runtime import trace_store + +# ============================================================================== +# Tier mapping +# ============================================================================== + +#: Internal model-tier labels (Luna/Terra/Sol cheap/mid/premium) used across +#: Cheese Work squads. Every enabled model-bearing node in a manifest must +#: resolve to exactly one of these tiers (mixed tiers => ``mixed_tier`` +#: rejection, see ``attempt_reconciliation.validate_batch``). +Tier = Literal["luna", "terra", "sol"] + +TIERS: tuple[Tier, ...] = ("luna", "terra", "sol") + +# Provider/model pairs that identify each tier. This fork's config/artemis.jsonc +# does not (yet) declare an explicit tier->model table; its "default" node +# resolves to openai/gpt-5.6-sol today, so "sol" is mapped to that pair as the +# only tier presently exercised end-to-end. "luna"/"terra" are declared for +# forward compatibility with the escalation controller (Gate 2, out of scope +# here) and must be updated here (not inferred) if/when config/artemis.jsonc +# grows an explicit tier table. +TIER_MODELS: dict[Tier, tuple[str, str]] = { + "luna": ("google", "gemini-3.5-flash-lite"), + "terra": ("google", "gemini-3.8-flash"), + "sol": ("openai", "gpt-5.6-sol"), +} + +# Environment variables whose presence/value can influence LLMConfig +# resolution and must be captured (never their raw secret value) in the +# manifest's provenance section. +_RELEVANT_ENV_PREFIXES = ("ARTEMIS_",) +_FAKE_LLM_ENV_VAR = "ARTEMIS_FAKE_LLM" + +# Substrings that mark an ARTEMIS_* variable name as credential-shaped (e.g. +# ARTEMIS_TENANT_TOKEN, ARTEMIS_LIFECYCLE_TOKEN). Matched case-insensitively +# against the whole var name. Any ARTEMIS_* var matching one of these must +# never have its raw value copied into the manifest's env_overrides -- it is +# reported the same non-secret way as _credential_reference (set + last4) +# instead. This is deliberately a denylist of credential-shaped substrings, +# not an allowlist of every non-secret ARTEMIS_* knob, since new resolution +# knobs are added far more often than new credential-bearing vars. +_CREDENTIAL_SHAPED_NAME_MARKERS = ( + "TOKEN", + "KEY", + "SECRET", + "PASSWORD", + "CREDENTIAL", + "AUTH", +) + + +def _is_credential_shaped_env_name(name: str) -> bool: + """True if ``name`` looks like it carries a secret (by name only).""" + upper = name.upper() + return any(marker in upper for marker in _CREDENTIAL_SHAPED_NAME_MARKERS) + + +def _sensitive_env_reference(value: str) -> dict[str, Any]: + """Non-secret presence reference for one credential-shaped env var value. + + Mirrors :func:`_credential_reference`'s shape exactly: never includes the + raw value, and omits ``last4`` entirely when the value is empty or + shorter than 4 characters. + """ + ref: dict[str, Any] = {"set": bool(value)} + if value and len(value) >= 4: + ref["last4"] = value[-4:] + return ref + + +# Every LLMConfig field that can carry an LLMWithFallback, in the exact +# vocabulary of the spec. Fields not present in LLMConfig.model_fields are a +# programming error (caught by the assertion in build_attempt_manifest). +_TOP_LEVEL_NODES: tuple[AgentNode, ...] = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + "history_analyzer", + "validator_pixel_safety_net", + "planner_validation", + "output_analyzer", +) + +# Nodes that are soft-defaulted by LLMConfig.get_agent() when absent, and the +# node they inherit from / default factory they use. Mirrors the branches in +# LLMConfig.get_agent verbatim so the manifest's "would resolve to" values +# never drift from the real resolution code. +_SOFT_DEFAULT_INHERITS_FROM: dict[str, str] = { + "history_analyzer": "operator", + "output_analyzer": "log_analyzer", +} +_SOFT_DEFAULT_FACTORY_NODES: frozenset[str] = frozenset( + {"validator_pixel_safety_net", "planner_validation"} +) + +_UTILS_NODES: tuple[LLMUtilsNode, ...] = ( + "outputter", + "hopper", + "video_analyzer", + "object_detector", +) + +# Nullable utils nodes have no soft-default factory (LLMConfig.get_utils raises +# instead); the manifest still records what they would need to be configured +# as, without pretending they are active. +_NULLABLE_UTILS_NODES: frozenset[str] = frozenset({"video_analyzer", "object_detector"}) + + +def _canonical_repo_sha(repo_root: Path | None) -> tuple[str, str]: + """Returns ``(sha, source)`` for the artemis repo's current commit. + + ``source`` is ``"git"`` on success. On any failure (not a git checkout, + git missing, detached worktree issue, etc.) falls back to + ``"unknown"`` with a fixed sentinel SHA rather than raising — a Gate 1 + manifest must always build, even in a source tree that lost its .git + directory (e.g. an extracted archive or bundled wheel), but the fallback + must be unambiguous in the manifest, never mistaken for a real SHA. + """ + root = repo_root or Path(__file__).resolve().parent.parent.parent + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(root), + capture_output=True, + text=True, + timeout=10, + check=True, + ) + sha = result.stdout.strip() + if sha: + return sha, "git" + except (OSError, subprocess.SubprocessError): + pass + return "0" * 40, "unknown-not-a-git-checkout" + + +def _relevant_env_snapshot(env: dict[str, str]) -> dict[str, Any]: + """Filters an explicit env mapping down to Artemis-relevant keys only. + + Never includes raw credential values. Most ``ARTEMIS_*`` vars are + resolution knobs (e.g. ``ARTEMIS_FAKE_LLM``, ``ARTEMIS_CONFIG_DIR``) and + pass through as-is. Any ``ARTEMIS_*`` var whose *name* is credential-shaped + (see ``_CREDENTIAL_SHAPED_NAME_MARKERS`` -- e.g. ``ARTEMIS_TENANT_TOKEN``) + is never copied by raw value: it is replaced with the same non-secret + ``{"set": bool, "last4": ...}`` shape ``_credential_reference`` already + uses for provider API keys, so a manifest can never leak a raw + credential into ``env_overrides`` even though provider credentials are + also reported separately in the ``credentials`` section. + """ + out: dict[str, Any] = {} + for k, v in env.items(): + if not k.startswith(_RELEVANT_ENV_PREFIXES): + continue + if _is_credential_shaped_env_name(k): + out[k] = _sensitive_env_reference(v) + else: + out[k] = v + return out + + +def _credential_reference(env: dict[str, str], provider: str) -> dict[str, Any]: + """Non-secret credential presence reference for one provider: set + last-4. + + Never includes the raw key. ``last4`` is omitted entirely when the key is + unset or shorter than 4 characters, rather than emitting an empty string. + """ + env_var = { + "openai": "OPENAI_API_KEY", + "google": "GOOGLE_API_KEY", + "vertexai": None, + "anthropic": "ANTHROPIC_API_KEY", + "openrouter": "OPEN_ROUTER_API_KEY", + "xai": "XAI_API_KEY", + "ollama": None, + "vllm": None, + "custom": None, + }.get(provider) + if env_var is None: + return {"provider": provider, "credential_env_var": None, "set": None} + value = env.get(env_var) + ref: dict[str, Any] = { + "provider": provider, + "credential_env_var": env_var, + "set": bool(value), + } + if value and len(value) >= 4: + ref["last4"] = value[-4:] + return ref + + +def _llm_node_snapshot(llm: LLM | LLMWithFallback) -> dict[str, Any]: + """Sanitized, non-secret snapshot of one LLM/LLMWithFallback record.""" + snap: dict[str, Any] = { + "provider": llm.provider, + "model": llm.model, + "api_base": llm.api_base, + "temperature": llm.temperature, + "thinking_budget": llm.thinking_budget, + "thinking_level": llm.thinking_level, + "reasoning_effort": llm.reasoning_effort, + "include_thoughts": llm.include_thoughts, + "enable_grounding": llm.enable_grounding, + } + if isinstance(llm, LLMWithFallback): + snap["fallback"] = _llm_node_snapshot(llm.fallback) + snap["fix_model"] = llm.fix_model + snap["timeout"] = llm.timeout + return snap + + +def _node_tier(llm: LLM | LLMWithFallback) -> Tier | None: + """Which declared tier (if any) this node's primary provider/model matches.""" + for tier, (provider, model) in TIER_MODELS.items(): + if llm.provider == provider and llm.model == model: + return tier + return None + + +@dataclasses.dataclass(frozen=True) +class NodeManifestEntry: + """One node's manifest entry: enabled/disabled state, snapshot, and tier match.""" + + node: str + enabled: bool + provenance: str + config: dict[str, Any] | None + resolved_tier: Tier | None + # For disabled (None-valued, soft-defaulted) nodes only: what get_agent() + # would actually return if this node were invoked right now, so the + # manifest never silently omits the effective default. + would_resolve_to: dict[str, Any] | None = None + would_resolve_provenance: str | None = None + + +def _build_node_entry( + node: str, + raw_value: LLMWithFallback | None, + config: LLMConfig, + layer_hint: str, +) -> NodeManifestEntry: + if raw_value is not None: + snapshot = _llm_node_snapshot(raw_value) + return NodeManifestEntry( + node=node, + enabled=True, + provenance=layer_hint, + config=snapshot, + resolved_tier=_node_tier(raw_value), + ) + + # Explicitly disabled/nullable node. Represent it as disabled, and also + # show what get_agent() would resolve it to if invoked, without treating + # that hypothetical resolution as an active manifest entry. + would_resolve_to: dict[str, Any] | None = None + would_resolve_provenance: str | None = None + if node in _SOFT_DEFAULT_INHERITS_FROM: + inherited_from = _SOFT_DEFAULT_INHERITS_FROM[node] + inherited_value = getattr(config, inherited_from) + would_resolve_to = _llm_node_snapshot(inherited_value) + would_resolve_provenance = f"soft-default: inherits '{inherited_from}'" + elif node in _SOFT_DEFAULT_FACTORY_NODES: + would_resolve_to = _llm_node_snapshot(lightweight_judge_default()) + would_resolve_provenance = "soft-default: lightweight_judge_default()" + elif node in _NULLABLE_UTILS_NODES: + would_resolve_provenance = "no soft default: get_utils() raises if invoked while unset" + + return NodeManifestEntry( + node=node, + enabled=False, + provenance="disabled (None in LLMConfig)", + config=None, + resolved_tier=None, + would_resolve_to=would_resolve_to, + would_resolve_provenance=would_resolve_provenance, + ) + + +def _node_entries_to_dict(entries: list[NodeManifestEntry]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for entry in entries: + out[entry.node] = { + "enabled": entry.enabled, + "provenance": entry.provenance, + "config": entry.config, + "resolved_tier": entry.resolved_tier, + "would_resolve_to": entry.would_resolve_to, + "would_resolve_provenance": entry.would_resolve_provenance, + } + return out + + +def build_attempt_manifest( + *, + run_id: str, + attempt_id: str, + trace_id: str, + tier: Tier, + llm_config: LLMConfig, + env: dict[str, str], + checkpoint: Literal["launch", "worker_start", "termination"], + parent_attempt_id: str | None = None, + config_layer_hint: str = "resolved LLMConfig (layer not separately tracked by caller)", + repo_root: Path | None = None, + now: float | None = None, +) -> dict[str, Any]: + """Builds one complete, credential-free attempt manifest. + + Pure function of its explicit arguments: it never reads ``os.environ`` + itself (the caller must snapshot it into ``env``) and never touches + ``ModelFactory``'s process-wide model cache. Two calls with different + ``env``/``llm_config``/``tier`` inputs are guaranteed to diverge; two + calls with identical inputs (module state held constant) are guaranteed + to produce logically identical manifests, modulo ``now``/checkpoint. + + ``config_layer_hint`` documents which resolution layer produced + ``llm_config`` as a whole (e.g. "env override: ARTEMIS_CONFIG_DIR", + "config/artemis.jsonc", "hardcoded default in parse_llm_config"); callers + that track richer per-field provenance may pass a more specific string, + or post-process the returned dict's per-node ``provenance`` values. + + Raises ``ValueError`` if ``tier`` is not one of the declared tiers, or if + the resulting manifest mixes tiers across enabled nodes (fail fast at + build time rather than deferring the mixed_tier check to batch + validation, since a single manifest that already disagrees with itself + should never be persisted as if it were valid). + """ + if tier not in TIER_MODELS: + raise ValueError(f"Unknown tier {tier!r}; expected one of {TIERS}") + + sha, sha_source = _canonical_repo_sha(repo_root) + relevant_env = _relevant_env_snapshot(env) + fake_llm_value = env.get(_FAKE_LLM_ENV_VAR) + fake_llm_enabled = fake_llm_value == "1" + + node_entries = [ + _build_node_entry(node, getattr(llm_config, node), llm_config, config_layer_hint) + for node in _TOP_LEVEL_NODES + ] + util_entries = [ + _build_node_entry(util, getattr(llm_config.utils, util), llm_config, config_layer_hint) + for util in _UTILS_NODES + ] + + enabled_tiers = { + e.resolved_tier for e in (*node_entries, *util_entries) if e.enabled and e.resolved_tier + } + untiered_enabled = [ + e.node for e in (*node_entries, *util_entries) if e.enabled and e.resolved_tier is None + ] + if enabled_tiers - {tier}: + raise ValueError( + f"Attempt manifest requested tier {tier!r} but enabled node(s) resolve to " + f"other tier(s) {sorted(enabled_tiers - {tier})}; refusing to build a " + "self-contradicting manifest. This is the mixed_tier condition." + ) + + credentials = { + provider: _credential_reference(env, provider) + for provider in ("openai", "google", "anthropic", "openrouter", "xai") + } + + manifest: dict[str, Any] = { + "manifest_schema_version": 1, + "run_id": run_id, + "attempt_id": attempt_id, + "trace_id": trace_id, + "parent_attempt_id": parent_attempt_id, + "checkpoint": checkpoint, + "recorded_at": now if now is not None else time.time(), + "tier": tier, + "source_sha": sha, + "source_sha_provenance": sha_source, + "fake_llm_enabled": fake_llm_enabled, + "fake_llm_env_var": _FAKE_LLM_ENV_VAR, + "env_overrides": relevant_env, + "credentials": credentials, + "nodes": _node_entries_to_dict(node_entries), + "utils": _node_entries_to_dict(util_entries), + "untiered_enabled_nodes": sorted(untiered_enabled), + } + return manifest + + +# ============================================================================== +# Canonical serialization + hashing +# ============================================================================== + + +def canonical_bytes(manifest: dict[str, Any]) -> bytes: + """Deterministic UTF-8 canonical JSON: sorted keys, no whitespace ambiguity. + + Uses ``json.dumps(..., sort_keys=True, ensure_ascii=False, + separators=(",", ":"))``. Two dicts that are equal (regardless of key + insertion order, since Python dict equality ignores order) always + serialize to byte-identical output under this scheme. + """ + return json.dumps( + manifest, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + + +def digest_of(manifest: dict[str, Any]) -> str: + """SHA-256 hex digest over ``canonical_bytes(manifest)``.""" + return hashlib.sha256(canonical_bytes(manifest)).hexdigest() + + +# ============================================================================== +# Create-only storage beside the trace directory +# ============================================================================== + + +class ManifestAlreadyExistsError(FileExistsError): + """Raised when an attempt manifest write would overwrite an existing file.""" + + +def _manifest_paths(trace_id: str, checkpoint: str) -> tuple[Path, Path]: + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + manifest_path = trace_dir / f"attempt_manifest.{checkpoint}.json" + digest_path = trace_dir / f"attempt_manifest.{checkpoint}.sha256" + return manifest_path, digest_path + + +def _create_only_write(path: Path, data: bytes) -> None: + """Writes ``data`` to ``path`` iff the path does not already exist. + + Uses ``O_CREAT | O_EXCL`` so the existence check and the write are one + atomic kernel operation — no TOCTOU window between a prior ``.exists()`` + check and the write itself. Raises :class:`ManifestAlreadyExistsError` if + the path already exists (races included: two concurrent writers targeting + the same path always leave exactly one winner and one raised error, never + a silent overwrite). + """ + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY + try: + fd = os.open(path, flags, 0o644) + except FileExistsError as e: + raise ManifestAlreadyExistsError( + f"{path} already exists; refusing to overwrite a prior attempt's record." + ) from e + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + except BaseException: + path.unlink(missing_ok=True) + raise + + +def store_attempt_manifest(manifest: dict[str, Any]) -> tuple[Path, str]: + """Writes the manifest JSON bytes + its digest beside the trace directory. + + Create-only under concurrency: both the manifest and digest files are + written via ``O_CREAT | O_EXCL`` (:func:`_create_only_write`), so a + concurrent writer targeting the same ``(trace_id, checkpoint)`` can never + silently overwrite an earlier attempt's stored manifest (requirement: + "never overwrite a prior attempt's manifest/trace file") — the loser of + the race raises :class:`ManifestAlreadyExistsError` instead. One file is + written per checkpoint (``launch`` / ``worker_start`` / ``termination``), + keyed by ``trace_id``+``checkpoint``, so all three checkpoints for one + attempt can coexist and each is independently immutable once written. + + Returns ``(manifest_path, digest_hex)``. + """ + trace_id = manifest["trace_id"] + checkpoint = manifest["checkpoint"] + manifest_path, digest_path = _manifest_paths(trace_id, checkpoint) + + manifest_path.parent.mkdir(parents=True, exist_ok=True) + payload = canonical_bytes(manifest) + digest = hashlib.sha256(payload).hexdigest() + + _create_only_write(manifest_path, payload) + try: + _create_only_write(digest_path, digest.encode("utf-8")) + except ManifestAlreadyExistsError: + # The manifest write above already won its race and is on disk and + # immutable; only the digest file lost its race. Leave both files as + # they are — the manifest write must not be undone (a concurrent + # reader may already observe it), and re-raising surfaces the + # conflict to the caller instead of masking it. + raise + + return manifest_path, digest + + +def read_stored_manifest(trace_id: str, checkpoint: str) -> tuple[bytes, str] | None: + """Reads back a previously stored manifest's exact bytes and digest, or None.""" + manifest_path, digest_path = _manifest_paths(trace_id, checkpoint) + if not manifest_path.exists() or not digest_path.exists(): + return None + return manifest_path.read_bytes(), digest_path.read_text(encoding="utf-8").strip() diff --git a/artemis/config/attempt_reconciliation.py b/artemis/config/attempt_reconciliation.py new file mode 100644 index 00000000..de601a33 --- /dev/null +++ b/artemis/config/attempt_reconciliation.py @@ -0,0 +1,583 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reconciles native ``llm_usage`` trace events against an attempt manifest, +and applies the Gate 1 batch accept/reject rule across one or more attempts. + +This module implements the *mechanism* only (Gate 1). It does not run or +claim to have run a qualification pilot; see +``artemis.config.attempt_manifest`` module docstring for the same caveat. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from typing import Any, Literal + +from artemis.config.attempt_manifest import TIER_MODELS, Tier, digest_of, read_stored_manifest +from artemis.config.attempt_usage_reader import read_llm_usage_events +from artemis.runtime import trace_store + +#: Machine-readable batch-reject reasons, verbatim per the Gate 1 spec. +RejectReason = Literal[ + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", +] + +REJECT_REASONS: tuple[RejectReason, ...] = ( + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", +) + +# Maps a usage-side node name (the `node` field a `@trace`-scoped call site +# actually records via CURRENT_NODE_NAME, see +# artemis.services.token_meter.record_llm_usage) to the manifest node name it +# should reconcile against, for the rare cases where the enclosing @trace +# scope's name legitimately differs from the LLMConfig field name the node +# resolves through. +# +# validator_pixel_safety_net is the one confirmed case today: +# artemis.agents.validator.validator._validate_action_precondition_pixel is +# traced as "safety_net_pixel_validation" (see task_tree.py's custom +# rendering, which depends on that exact trace name for unrelated UI +# purposes -- do not rename the @trace scope to "fix" this), but internally +# calls get_llm_fn(ctx, name="validator_pixel_safety_net"), which is the +# LLMConfig field / manifest node name. Without this alias, a real, +# correctly-identified LLM call reconciles as unmapped_call purely because +# the trace scope name and the manifest node name disagree. +_NODE_ALIASES: dict[str, str] = { + "safety_net_pixel_validation": "validator_pixel_safety_net", +} + + +@dataclasses.dataclass(frozen=True) +class NodeReconciliation: + """Reconciliation verdict for one manifest node against usage events.""" + + node: str + manifest_enabled: bool + manifest_source: str | None # "provider:model" the manifest expects, if enabled + invoked: bool + usage_sources: tuple[str, ...] # distinct `source` values seen for this node + verdict: Literal[ + "match", # invoked, source matches manifest + "mismatch", # invoked, source present but does not match manifest + "unverified_identity", # invoked, but usage event(s) missing `source` + "not_invoked", # enabled in manifest, no usage event this attempt (informational) + "disabled_not_invoked", # explicitly disabled in manifest, and never invoked (expected) + "unmapped_call", # invoked, but node absent from manifest entirely + ] + + +@dataclasses.dataclass(frozen=True) +class ReconciliationResult: + """Full reconciliation outcome for one attempt: per-node verdicts + flags.""" + + attempt_id: str + nodes: tuple[NodeReconciliation, ...] + + @property + def has_mismatch(self) -> bool: + return any(n.verdict == "mismatch" for n in self.nodes) + + @property + def has_unverified_identity(self) -> bool: + return any(n.verdict == "unverified_identity" for n in self.nodes) + + @property + def has_unmapped_call(self) -> bool: + return any(n.verdict == "unmapped_call" for n in self.nodes) + + +def _tier_of_provider_model(provider: str | None, model: str | None) -> Tier | None: + """Which declared tier (if any) a raw ``(provider, model)`` pair matches. + + Dict-based counterpart of ``attempt_manifest._node_tier`` (which takes an + ``LLM``/``LLMWithFallback`` instance): this module only ever sees the + already-serialized ``config``/``would_resolve_to`` dicts a stored manifest + carries, never the original ``LLM`` objects, so it re-does the same + ``TIER_MODELS`` lookup directly against those two plain strings. + """ + for tier, (tier_provider, tier_model) in TIER_MODELS.items(): + if provider == tier_provider and model == tier_model: + return tier + return None + + +def _manifest_expected_source(node_entry: dict[str, Any], manifest_tier: Tier | None) -> str | None: + """The `provider:model` string a manifest node is expected to produce as `source`. + + For an enabled node this is its own ``config``. For a *disabled* + (soft-defaulted) node -- e.g. ``validator_pixel_safety_net`` / + ``planner_validation`` left unset in ``LLMConfig``, which + ``LLMConfig.get_agent()`` actually resolves to + ``lightweight_judge_default()`` (or an inherited node) when invoked, see + ``attempt_manifest._build_node_entry`` -- the expected source is derived + from ``would_resolve_to`` instead, since that is what the node will + genuinely produce if it fires. A disabled node with no ``would_resolve_to`` + (no soft default applies; it is simply off) still returns ``None``: an + unexpected receipt from an entirely-disabled node must still surface as a + problem, not be silently accepted. + + Tier pinning applies to soft-defaulted nodes exactly as it already applies + to enabled ones: an attempt manifest built for ``tier="sol"`` asserts that + *every* model-bearing node this attempt can invoke -- including a node + that only fires via its soft default, e.g. ``validator_pixel_safety_net`` + -- resolves within the Sol tier. ``would_resolve_to`` is a fixed function + of the node (``lightweight_judge_default()`` or an inherited sibling + node), not of ``manifest_tier``, so when its provider/model maps to a + *different* declared tier than ``manifest_tier``, trusting it as the + "expected" source would let an attempt pinned to one tier silently accept + a receipt from another tier's model -- exactly the cross-tier mixing + Gate 1 exists to catch. In that case this returns ``None`` (not the + would-resolve-to string), which routes the caller's mismatch/match branch + in ``reconcile_attempt`` to ``mismatch`` for any receipt, and to + ``disabled_not_invoked`` if the node never actually fires -- never to a + silent ``match``. A would-resolve-to whose provider/model maps to no + declared tier at all is treated the same conservative way, for the same + reason ``untiered_enabled_nodes`` is never silently trusted for enabled + nodes (see ``attempt_manifest.build_attempt_manifest`` / + ``validate_batch``'s ``missing_identity`` check): an unpinned model + identity is never assumed safe. + """ + config = node_entry.get("config") + if node_entry.get("enabled"): + config = config or {} + else: + config = node_entry.get("would_resolve_to") or None + if config is None: + return None + would_resolve_tier = _tier_of_provider_model(config.get("provider"), config.get("model")) + if would_resolve_tier != manifest_tier: + return None + provider = config.get("provider") + model = config.get("model") + if provider is None or model is None: + return None + return f"{provider}:{model}" + + +def reconcile_attempt( + attempt_id: str, + manifest: dict[str, Any], + usage_events: list[dict[str, Any]], +) -> ReconciliationResult: + """Reconciles one attempt's manifest against its raw ``llm_usage`` payloads. + + ``usage_events`` are the dicts produced by + ``artemis.services.token_meter.record_llm_usage`` (carrying at least + ``node``; ``source`` is only present when the call site passed one — see + ``RobustChatModelWrapper._endpoint_key()`` for the ``provider:model`` + shape, and the ``lens:*`` labels used by raw-model bypass call sites in + ``memory/chunking.py`` / ``agents/flash/summarizer.py``, which are + reported as-is and will not match any manifest node's ``provider:model`` + expectation — they surface as ``mismatch`` rather than being silently + accepted, since Gate 1 requires every invoked identity to be provable, + not merely present). + + Never fabricates a usage record for a node that did not fire: nodes with + no usage event are reported as ``not_invoked``/``disabled_not_invoked``, + not synthesized as zero-usage matches. + """ + manifest_nodes: dict[str, dict[str, Any]] = { + **manifest.get("nodes", {}), + **manifest.get("utils", {}), + } + + events_by_node: dict[str, list[dict[str, Any]]] = {} + for event in usage_events: + node = event.get("node") + if node is None: + # A usage event with no node context at all cannot be attributed + # to any manifest entry; treat its node key as the literal + # sentinel so it still surfaces as unmapped rather than being + # dropped silently. + node = "" + node = str(node) + # Resolve to the manifest node name when the usage-side node is a + # known trace-scope alias (see _NODE_ALIASES); otherwise group under + # the raw node name as before. + resolved_node = _NODE_ALIASES.get(node, node) + events_by_node.setdefault(resolved_node, []).append(event) + + manifest_tier: Tier | None = manifest.get("tier") + + results: list[NodeReconciliation] = [] + seen_nodes: set[str] = set() + + for node, entry in manifest_nodes.items(): + seen_nodes.add(node) + expected_source = _manifest_expected_source(entry, manifest_tier) + events = events_by_node.get(node, []) + if not events: + verdict = "not_invoked" if entry.get("enabled") else "disabled_not_invoked" + results.append( + NodeReconciliation( + node=node, + manifest_enabled=bool(entry.get("enabled")), + manifest_source=expected_source, + invoked=False, + usage_sources=(), + verdict=verdict, + ) + ) + continue + + sources = tuple(e.get("source") for e in events) + if any(s is None for s in sources): + verdict = "unverified_identity" + elif expected_source is not None and any(s != expected_source for s in sources): + verdict = "mismatch" + elif expected_source is None: + # Node fired but the manifest has no resolvable provider:model to + # expect for it -- either an enabled entry missing config, a + # disabled node with no soft default, or a soft-defaulted disabled + # node whose `would_resolve_to` tier disagrees with (or is + # entirely unmapped from) the manifest's own declared `tier` (see + # `_manifest_expected_source`). A soft-defaulted disabled node + # (e.g. validator_pixel_safety_net left unset) only has a non-None + # `expected_source` here when its `would_resolve_to` tier matches + # this attempt's own tier -- only that case is *not* covered by + # this branch and can legitimately reach "match" below. Surface + # this branch as mismatch rather than pretending an entirely- or + # cross-tier-defaulted node firing is fine. + verdict = "mismatch" + else: + verdict = "match" + + results.append( + NodeReconciliation( + node=node, + manifest_enabled=bool(entry.get("enabled")), + manifest_source=expected_source, + invoked=True, + usage_sources=tuple(s for s in sources if s is not None), + verdict=verdict, + ) + ) + + # Usage events for nodes the manifest never declared at all (e.g. a + # renamed node, a lens/utility label, or an entirely unmapped call site). + for node, events in events_by_node.items(): + if node in seen_nodes: + continue + sources = tuple(e.get("source") for e in events if e.get("source") is not None) + results.append( + NodeReconciliation( + node=node, + manifest_enabled=False, + manifest_source=None, + invoked=True, + usage_sources=sources, + verdict="unmapped_call", + ) + ) + + return ReconciliationResult(attempt_id=attempt_id, nodes=tuple(results)) + + +# ============================================================================== +# Batch validation / rejection rule +# ============================================================================== + + +@dataclasses.dataclass(frozen=True) +class AttemptRecord: + """One attempt's manifest + reconciliation, as handed to batch validation.""" + + attempt_id: str + manifest: dict[str, Any] + reconciliation: ReconciliationResult + # Digest computed at build/store time (from attempt_manifest.digest_of), + # supplied by the caller so batch validation can detect drift between + # what was stored and what is being validated now, without recomputing + # trust in a possibly-tampered manifest dict. + stored_digest: str | None = None + + +@dataclasses.dataclass(frozen=True) +class BatchVerdict: + """ACCEPT, or REJECT with a specific machine-readable reason. + + On reject, ``invalid_attempts`` preserves the exact records that failed + (manifest + reconciliation unmodified) so a caller can retain them for + audit rather than discarding them from totals. + """ + + accepted: bool + reason: RejectReason | None + detail: str + invalid_attempts: tuple[AttemptRecord, ...] = () + + +def _reject(reason: RejectReason, detail: str, invalid: tuple[AttemptRecord, ...]) -> BatchVerdict: + return BatchVerdict(accepted=False, reason=reason, detail=detail, invalid_attempts=invalid) + + +def validate_batch(attempts: list[AttemptRecord]) -> BatchVerdict: + """Validates one homogeneous batch of attempts (e.g. one journey x device cell). + + Checks run in a fixed order so the first violation found is reported; + every check preserves the offending attempt record(s) unmodified in + ``invalid_attempts`` rather than discarding them. An empty batch is + reported as ``missing_snapshot`` (nothing to validate is itself a defect + for a caller expecting N repeats). + """ + if not attempts: + return _reject("missing_snapshot", "Batch is empty: no attempt manifests supplied.", ()) + + # 1. missing_identity: any attempt whose manifest lacks a usable + # source_sha, whose fake_llm flag/tier is absent entirely, whose + # source_sha was never actually verified against a real git checkout + # (source_sha_provenance != "git" -- the "0"*40/"unknown-not-a-git- + # checkout" fallback sentinel is truthy but not a provable identity), + # or that has any enabled node resolving to no declared tier at all + # (untiered_enabled_nodes non-empty -- a model-bearing node that isn't + # pinned to Luna/Terra/Sol). + missing_identity = [ + a + for a in attempts + if not a.manifest.get("source_sha") + or a.manifest.get("tier") not in ("luna", "terra", "sol") + or a.manifest.get("fake_llm_enabled") is None + or a.manifest.get("source_sha_provenance") != "git" + or a.manifest.get("untiered_enabled_nodes") + ] + if missing_identity: + return _reject( + "missing_identity", + f"{len(missing_identity)} attempt(s) have an incomplete manifest identity " + "(missing source_sha, tier, or fake_llm_enabled; an unverified/non-git " + "source_sha; or an enabled node with no declared tier).", + tuple(missing_identity), + ) + + # 2. fake_mode_enabled: any attempt that ran with ARTEMIS_FAKE_LLM=1. + fake_enabled = [a for a in attempts if a.manifest.get("fake_llm_enabled")] + if fake_enabled: + env_var = fake_enabled[0].manifest.get("fake_llm_env_var", "ARTEMIS_FAKE_LLM") + return _reject( + "fake_mode_enabled", + f"{len(fake_enabled)} attempt(s) ran with fake-LLM mode enabled ({env_var}=1).", + tuple(fake_enabled), + ) + + # 3. missing_snapshot: any attempt with no stored digest to compare against. + missing_snapshot = [a for a in attempts if not a.stored_digest] + if missing_snapshot: + return _reject( + "missing_snapshot", + f"{len(missing_snapshot)} attempt(s) have no stored manifest digest to verify.", + tuple(missing_snapshot), + ) + + # 4. digest_drift: recomputed digest of the manifest dict no longer + # matches the digest recorded at store time. + drifted = [a for a in attempts if digest_of(a.manifest) != a.stored_digest] + if drifted: + return _reject( + "digest_drift", + f"{len(drifted)} attempt(s) have a manifest whose recomputed digest no longer " + "matches its stored digest (possible post-hoc mutation).", + tuple(drifted), + ) + + # 5. mixed_tier: not every attempt in the batch declares the same tier. + tiers = {a.manifest.get("tier") for a in attempts} + if len(tiers) > 1: + return _reject( + "mixed_tier", + f"Batch mixes tiers across attempts: {sorted(t for t in tiers if t)}.", + tuple(attempts), + ) + + # 6. unmapped_call / unexpected_model_or_endpoint: derived from each + # attempt's reconciliation result. + unmapped = [a for a in attempts if a.reconciliation.has_unmapped_call] + if unmapped: + return _reject( + "unmapped_call", + f"{len(unmapped)} attempt(s) invoked a node absent from their manifest.", + tuple(unmapped), + ) + + unexpected = [ + a + for a in attempts + if a.reconciliation.has_mismatch or a.reconciliation.has_unverified_identity + ] + if unexpected: + return _reject( + "unexpected_model_or_endpoint", + f"{len(unexpected)} attempt(s) invoked a node whose observed `source` does not " + "match the manifest, or is missing entirely (unverified identity).", + tuple(unexpected), + ) + + return BatchVerdict(accepted=True, reason=None, detail=f"{len(attempts)} attempt(s) accepted.") + + +# ============================================================================== +# Post-run orchestration: stored manifest + native usage receipts -> verdict +# ============================================================================== + + +def reconcile_finished_attempt( + *, + trace_id: str, + session_id: str, + checkpoint: str, + db_path: str | Path, + traces_dir: str | Path, +) -> BatchVerdict | None: + """Reconciles one finished attempt against its own stored manifest + DB receipts. + + Reads back the manifest :func:`artemis.config.attempt_manifest.store_attempt_manifest` + wrote for ``(trace_id, checkpoint)``, reads every native ``llm_usage`` row + the DataEngine recorded for ``session_id`` (via + :func:`artemis.config.attempt_usage_reader.read_llm_usage_events`, never + mutated or summarized), reconciles them, and runs the single-attempt batch + rule. Returns ``None`` (not a verdict) when no manifest was stored for + this checkpoint — that is a Gate 1 wiring gap to fix, not something to + misrepresent as a passing or failing verdict. + + This is intentionally a one-attempt batch: multi-attempt batch validation + (e.g. a five-repeat journey/device cell) is the qualification pilot's own + concern, not something this per-run hook can see on its own. + + Because this always validates a batch of exactly one attempt, the + ``mixed_tier`` rejection path is not reachable from this function; see + :func:`reconcile_attempt_batch_by_run_id` for the production-reachable + multi-attempt path. + """ + stored = read_stored_manifest(trace_id, checkpoint) + if stored is None: + return None + manifest_bytes, stored_digest = stored + manifest = json.loads(manifest_bytes.decode("utf-8")) + + usage_events = read_llm_usage_events(db_path, traces_dir, session_id) + reconciliation = reconcile_attempt(trace_id, manifest, usage_events) + record = AttemptRecord( + attempt_id=trace_id, + manifest=manifest, + reconciliation=reconciliation, + stored_digest=stored_digest, + ) + return validate_batch([record]) + + +def reconcile_attempt_batch_by_run_id( + *, + run_id: str, + checkpoint: str, + db_path: str | Path, + traces_dir: str | Path, + traces_root: str | Path | None = None, +) -> BatchVerdict: + """Reconciles every stored attempt sharing ``run_id`` as one batch. + + This is the production-reachable home for cross-attempt validation + (including the ``mixed_tier`` rejection, which :func:`reconcile_finished_attempt` + can never exercise since it always validates a batch of exactly one + attempt). ``mcp_server.background.task_runner.run_task`` accepts an + optional ``run_id`` parameter (threaded through to + ``_record_attempt_manifest``'s ``run_id=`` argument) that a caller can set + to a value shared across multiple real ``run_task`` invocations, so the + grouping key this function reads (each stored manifest's own ``"run_id"`` + field) is real, settable plumbing today, not synthetic test-only state. + Today's default remains unchanged when ``run_id`` is not passed: + ``run_id=trace_id`` 1:1, so an unmodified caller still produces one + attempt manifest per ``run_id`` and this function degenerates to a + one-attempt batch for it. No caller in this codebase actually invokes + ``run_task`` more than once with a shared ``run_id`` yet -- there is no + automatic retry orchestrator wired up, and building one is explicitly out + of scope here. This function exists as a real, tested, importable, + general-purpose multi-attempt reconciler that a future caller (e.g. a + qualification-pilot orchestrator running repeated attempts of one + journey/device cell under a shared ``run_id``) can call using the + plumbing that already exists, without needing to add any. + + Discovers candidate attempts by listing the immediate subdirectories of + ``traces_root`` (default: :data:`artemis.runtime.trace_store.TRACES_DIR`) + -- each subdirectory name is a ``trace_id`` + (:func:`artemis.runtime.trace_store.get_trace_dir`) -- reading each one's + stored manifest at ``checkpoint`` via + :func:`artemis.config.attempt_manifest.read_stored_manifest`, and keeping + only the attempts whose manifest ``"run_id"`` field equals the requested + ``run_id``. For each matching attempt, native ``llm_usage`` events are + read via :func:`artemis.config.attempt_usage_reader.read_llm_usage_events` + using the attempt's own ``trace_id`` as ``session_id`` (matching the + ``session_id=trace_id`` convention ``_record_attempt_manifest`` / + ``Agent(config=config, session_id=trace_id)`` already use), reconciled, + and assembled into an :class:`AttemptRecord` exactly as + :func:`reconcile_finished_attempt` does per-attempt. + + Read-only: never writes, mutates, or deletes any manifest or trace file. + A directory whose stored manifest cannot be parsed as JSON is skipped + rather than aborting the whole scan -- one corrupt/unreadable record must + not hide every other attempt's evidence, mirroring + ``read_llm_usage_events``'s own skip-corrupt-rows behavior. + + ``validate_batch`` is called over the full collected list, which may be + empty (no attempt manifests found for ``run_id``); an empty batch + correctly hits ``validate_batch``'s existing ``missing_snapshot`` + rejection rather than being special-cased here. + """ + root = Path(traces_root) if traces_root is not None else Path(trace_store.TRACES_DIR) + + records: list[AttemptRecord] = [] + if root.is_dir(): + for trace_dir in sorted(root.iterdir()): + if not trace_dir.is_dir(): + continue + trace_id = trace_dir.name + + stored = read_stored_manifest(trace_id, checkpoint) + if stored is None: + continue + manifest_bytes, stored_digest = stored + try: + manifest = json.loads(manifest_bytes.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + # A corrupt stored manifest for one attempt must not abort + # discovery of every other attempt sharing this run_id. + continue + if manifest.get("run_id") != run_id: + continue + + usage_events = read_llm_usage_events(db_path, traces_dir, trace_id) + reconciliation = reconcile_attempt(trace_id, manifest, usage_events) + records.append( + AttemptRecord( + attempt_id=trace_id, + manifest=manifest, + reconciliation=reconciliation, + stored_digest=stored_digest, + ) + ) + + return validate_batch(records) diff --git a/artemis/config/attempt_usage_reader.py b/artemis/config/attempt_usage_reader.py new file mode 100644 index 00000000..f54c9907 --- /dev/null +++ b/artemis/config/attempt_usage_reader.py @@ -0,0 +1,83 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only access to a session's native ``llm_usage`` trace receipts. + +Gate 1 reconciliation (:mod:`artemis.config.attempt_reconciliation`) needs the +raw ``llm_usage`` payloads :func:`artemis.services.token_meter.record_llm_usage` +already writes into the DataEngine SQLite store during a live run. That store +only exposes trace lookup scoped to a step +(:meth:`artemis.data_engine.storage.StorageManager.get_steps_with_traces`), +but ``llm_usage`` traces are not guaranteed to carry a ``step_id`` (background +lens/utility calls may record with ``step_id=None``); walking steps would +silently miss those. This module queries the ``traces`` table directly by +``session_id``, following the same read-only-offline-reader pattern already +established by :class:`artemis.data_engine.history_reader.OfflineHistoryReader` +(``StorageManager(db_path, traces_dir, read_only=True)``), without touching +any write path or the live in-process engine. + +Never mutates, summarizes, or drops rows: every ``llm_usage`` payload found +for the session is returned as-is, so reconciliation always sees the original +receipts rather than a derived view. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + +from artemis.data_engine.storage import StorageManager + + +def read_llm_usage_events( + db_path: str | Path, + traces_dir: str | Path, + session_id: str, +) -> list[dict[str, Any]]: + """Returns every stored ``llm_usage`` trace payload for ``session_id``. + + Read-only: opens the DataEngine database with ``read_only=True`` (refuses + to create a missing database or table, matching + :class:`~artemis.data_engine.history_reader.OfflineHistoryReader`). Rows + with unparsable JSON payloads are skipped rather than raising, since a + single corrupt trace row must not hide every other attempt's evidence; + the caller (reconciliation) already treats a missing/absent event as + ``not_invoked``/``unverified_identity`` rather than inferring identity. + + Returns an empty list, never raises, if the database does not exist yet + (e.g. a session that recorded no traces at all). + """ + resolved_db_path = Path(db_path) + if not resolved_db_path.exists(): + return [] + + storage = StorageManager(resolved_db_path, traces_dir, read_only=True) + events: list[dict[str, Any]] = [] + with storage._get_connection() as conn: + cursor = conn.execute( + "SELECT payload FROM traces WHERE session_id = ? AND name = 'llm_usage' " + "ORDER BY timestamp ASC", + (str(session_id),), + ) + for row in cursor.fetchall(): + raw_payload = row["payload"] + if not raw_payload: + continue + try: + events.append(json.loads(raw_payload)) + except (json.JSONDecodeError, TypeError): + continue + return events diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 1ddc6007..59ebb24d 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -16,6 +16,7 @@ import asyncio import os +import uuid from pathlib import Path from shutil import which from typing import Annotated @@ -23,6 +24,10 @@ from adbutils import AdbClient from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, initialize_llm_config, settings +from artemis.config.attempt_lifecycle_hooks import ( + record_attempt_manifest, + reconcile_and_store_verdict, +) from artemis.runtime import trace_store from artemis.utils.startup_progress import publish_startup_progress from artemis import Agent, Builders @@ -60,6 +65,7 @@ async def execute_task( explorer_flash_mode: str | None = None, explorer_pro_mode: str | None = None, verification_level: str | None = None, + run_id: str | None = None, ) -> None: """Executes a single mobile automation task end-to-end. @@ -80,120 +86,159 @@ async def execute_task( explorer_pro_mode: Override the Explorer tier for the Pro execution profile. verification_level: Coarse Checker preset ('off', 'final', 'checkpoints', 'strict'); applied before the explicit ``enable_checker`` switch. + run_id: Gate 1 batch-grouping key for this attempt's manifest (see + ``artemis.config.attempt_lifecycle_hooks``); defaults to the + effective session id when omitted. """ effective_sid = ( - session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + session_id + or os.getenv("ARTEMIS_SESSION_ID") + or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + or str(uuid.uuid4()) ) - if effective_sid: - os.environ["ARTEMIS_SESSION_ID"] = str(effective_sid) + # ARTEMIS_SESSION_ID is a process-global env var, but effective_sid is a + # per-invocation identity -- an explicit session_id arg (or a generated + # fallback) must never bleed into a later execute_task() call in the same + # process. Snapshot whatever was there before this call and restore it + # (or delete the key if it wasn't set) in the finally below, which covers + # both the initialize_llm_config() failure path and the main run. + _prior_session_id_env = os.environ.get("ARTEMIS_SESSION_ID") + os.environ["ARTEMIS_SESSION_ID"] = str(effective_sid) if not os.environ.get("ARTEMIS_TASK_INGRESS"): os.environ["ARTEMIS_TASK_INGRESS"] = "cli" - publish_startup_progress( - "configuration", - "Loading the run configuration", - session_id=str(effective_sid) if effective_sid else None, - ) - try: - llm_config = initialize_llm_config() - except Exception as exc: - if effective_sid: - try: - trace_store.update_trace_status(str(effective_sid), "failed", error=str(exc)) - except OSError: - logger.exception( - "Could not record configuration failure for session %s", effective_sid - ) - raise - agent_profile = AgentProfile(name="default", llm_config=llm_config) - config = Builders.AgentConfig.with_default_profile(profile=agent_profile) + publish_startup_progress( + "configuration", + "Loading the run configuration", + session_id=str(effective_sid) if effective_sid else None, + ) - if video_recording_tools_enabled is not None: - config.with_video_recording_tools(enabled=video_recording_tools_enabled) + try: + llm_config = initialize_llm_config() + except Exception as exc: + if effective_sid: + try: + trace_store.update_trace_status(str(effective_sid), "failed", error=str(exc)) + except OSError: + logger.exception( + "Could not record configuration failure for session %s", effective_sid + ) + raise + agent_profile = AgentProfile(name="default", llm_config=llm_config) + config = Builders.AgentConfig.with_default_profile(profile=agent_profile) - if enable_planner_validation is not None: - config.with_planner_validation(enabled=enable_planner_validation) + if video_recording_tools_enabled is not None: + config.with_video_recording_tools(enabled=video_recording_tools_enabled) - if enable_committee is not None: - config.with_committee(enabled=enable_committee) + if enable_planner_validation is not None: + config.with_planner_validation(enabled=enable_planner_validation) - if verification_level is not None: - config.with_checker(**checker_overrides_for_level(verification_level)) + if enable_committee is not None: + config.with_committee(enabled=enable_committee) - if enable_checker is not None: - config.with_checker(enabled=enable_checker) + if verification_level is not None: + config.with_checker(**checker_overrides_for_level(verification_level)) - if enable_step_summarizer is not None: - config.with_flash_step_summarizer(enabled=enable_step_summarizer) + if enable_checker is not None: + config.with_checker(enabled=enable_checker) - if enable_outputter is not None or force_output_synthesis is not None: - config.with_outputter( - enabled=enable_outputter if enable_outputter is not None else True, - force_synthesis=bool(force_output_synthesis), - ) + if enable_step_summarizer is not None: + config.with_flash_step_summarizer(enabled=enable_step_summarizer) - if ( - explorer_version is not None - or explorer_flash_mode is not None - or explorer_pro_mode is not None - ): - config.with_explorer( - version=explorer_version, - flash_mode=explorer_flash_mode, - pro_mode=explorer_pro_mode, - ) + if enable_outputter is not None or force_output_synthesis is not None: + config.with_outputter( + enabled=enable_outputter if enable_outputter is not None else True, + force_synthesis=bool(force_output_synthesis), + ) - if settings.ADB_HOST: - config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) + if ( + explorer_version is not None + or explorer_flash_mode is not None + or explorer_pro_mode is not None + ): + config.with_explorer( + version=explorer_version, + flash_mode=explorer_flash_mode, + pro_mode=explorer_pro_mode, + ) - target_serial = ( - device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") - ) - if not target_serial: - try: - from artemis.runtime import device_pool + if settings.ADB_HOST: + config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) - target_serial = device_pool.select_device() - except Exception: - target_serial = None + target_serial = ( + device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") + ) + if not target_serial: + try: + from artemis.runtime import device_pool - if target_serial: - from artemis.context import DevicePlatform + target_serial = device_pool.select_device() + except Exception: + target_serial = None - config.for_device(DevicePlatform.ANDROID, target_serial) + if target_serial: + from artemis.context import DevicePlatform - if graph_config_callbacks: - config.with_graph_config_callbacks(graph_config_callbacks) + config.for_device(DevicePlatform.ANDROID, target_serial) - agent: Agent | None = None - try: - agent = Agent(config=config.build(), session_id=effective_sid) - await agent.init( - retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), - retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), - ) + if graph_config_callbacks: + config.with_graph_config_callbacks(graph_config_callbacks) + + agent: Agent | None = None + try: + record_attempt_manifest( + trace_id=str(effective_sid), + checkpoint="launch", + llm_config=llm_config, + run_id=run_id, + ) + + agent = Agent(config=config.build(), session_id=effective_sid) + await agent.init( + retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), + retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), + ) - task = agent.new_task(goal) - if locked_app_package: - task.with_locked_app_package(locked_app_package) - if test_name: - trace_path = traces_output_path_str or str(settings.TRACES_PATH) - task.with_name(test_name).with_trace_recording(path=trace_path) - if output_description: - task.with_output_description(output_description) - if profile: - task.using_profile(profile) - if app_path: - task.with_app_path(Path(app_path)) - - llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) - if llm_result_path: - task.with_llm_output_saving(path=llm_result_path) - - await agent.run_task(request=task.build()) + task = agent.new_task(goal) + if locked_app_package: + task.with_locked_app_package(locked_app_package) + if test_name: + trace_path = traces_output_path_str or str(settings.TRACES_PATH) + task.with_name(test_name).with_trace_recording(path=trace_path) + if output_description: + task.with_output_description(output_description) + if profile: + task.using_profile(profile) + if app_path: + task.with_app_path(Path(app_path)) + + llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) + if llm_result_path: + task.with_llm_output_saving(path=llm_result_path) + + await agent.run_task(request=task.build()) + finally: + # Reconciliation runs regardless of task outcome (success, failed + # status, exception, or cancellation) and before cleanup, so an + # Agent.clean() failure (e.g. device disconnect) can never + # suppress the reconciliation verdict -- mirrors + # mcp_server.background.task_runner's finally-block ordering. + reconcile_and_store_verdict( + trace_id=str(effective_sid), checkpoint="launch", run_id=run_id + ) + if agent is not None: + try: + await agent.clean() + except Exception as exc: + logger.error(f"Error cleaning agent: {exc}") finally: - if agent is not None: - await agent.clean() + # Restore whatever ARTEMIS_SESSION_ID held before this call so an + # explicit session_id (or a generated fallback) never bleeds into a + # later execute_task() call in the same process. + if _prior_session_id_env is None: + os.environ.pop("ARTEMIS_SESSION_ID", None) + else: + os.environ["ARTEMIS_SESSION_ID"] = _prior_session_id_env def run_command( @@ -348,6 +393,16 @@ def run_command( help="Canonical session UUID for trace and stream telemetry.", ), ] = None, + run_id: Annotated[ + str | None, + typer.Option( + "--run-id", + help=( + "Gate 1 batch-grouping key for this attempt's manifest " + "(defaults to the session id when omitted)." + ), + ), + ] = None, standalone: Annotated[ bool, typer.Option( @@ -396,6 +451,7 @@ def run_command( app_path=app_path, session_id=target_sid, ingress="cli", + run_id=run_id, base_url=base_url, ) if resp and resp.get("tasks"): @@ -499,6 +555,7 @@ def on_status(sess_info): explorer_flash_mode=explorer_flash_mode, explorer_pro_mode=explorer_pro_mode, verification_level=verification_level, + run_id=run_id, ) ) except (KeyboardInterrupt, asyncio.CancelledError): diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 67736e65..3751c718 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -221,6 +221,7 @@ def submit_task_to_daemon( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: @@ -230,6 +231,10 @@ def submit_task_to_daemon( ``explorer_mode`` ('flash' | 'pro' | 'ultra') are the Pro-profile tuning knobs of ``/api/run``; they are forwarded verbatim and ignored by Flash. + ``run_id`` is the Gate 1 batch-grouping key for this attempt's manifest + (see ``artemis.config.attempt_lifecycle_hooks``); it defaults to the + attempt's own trace/session id when omitted. + Returns the response JSON dict if successfully enqueued, or None on error. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" @@ -246,6 +251,7 @@ def submit_task_to_daemon( "session_id": session_id, "ingress": ingress, "conversation_id": conversation_id, + "run_id": run_id, } try: diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index a6f377ce..0151048b 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -41,6 +41,12 @@ except Exception: load_dotenv(os.path.join(PROJECT_ROOT, ".env")) +from artemis.config.attempt_lifecycle_hooks import ( + record_attempt_manifest as _record_attempt_manifest, +) +from artemis.config.attempt_lifecycle_hooks import ( + reconcile_and_store_verdict as _reconcile_and_store_verdict, +) from artemis.runtime import trace_store from mcp_server.notifiers import notify from mcp_server.utils import device_utils @@ -121,6 +127,7 @@ async def run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_pro_mode: str | None = None, + run_id: str | None = None, ): """Executes the mobile automation agent task and logs all actions/results. @@ -128,6 +135,17 @@ async def run_task( ``explorer_pro_mode`` ('flash' | 'pro' | 'ultra') are Pro-profile tuning knobs mirroring ``artemis run --verification-level / --explorer-pro-mode``; the Flash profile ignores them. + + ``run_id`` is plumbing only: when not given, the attempt manifests this + call records default to ``run_id=trace_id`` (today's exact behavior, and + still the case for every real caller today -- see + ``mcp_server.background.task_runner`` module usage). No caller in this + codebase currently invokes ``run_task`` more than once for a shared + ``run_id``; a future caller that does (e.g. a retry driver re-running the + same logical task under multiple ``trace_id``s) can pass one shared + ``run_id`` across those invocations so their manifests become + reconcilable together as one batch via + ``attempt_reconciliation.reconcile_attempt_batch_by_run_id``. """ trace_dir = trace_store.get_trace_dir(trace_id) os.makedirs(trace_dir, exist_ok=True) @@ -213,6 +231,10 @@ async def run_task( else: profile = AgentProfile(name="default", llm_config=initialize_llm_config()) + _record_attempt_manifest( + trace_id=trace_id, checkpoint="launch", llm_config=profile.llm_config, run_id=run_id + ) + config_builder = Builders.AgentConfig.with_default_profile(profile) if verification_level: config_builder.with_verification_level(verification_level) @@ -228,7 +250,18 @@ async def run_task( config = config_builder.build() - agent = Agent(config=config) + # session_id=trace_id links this run's DataEngine session (where + # native llm_usage receipts land) to the same identifier the attempt + # manifest is keyed and stored by, so post-run reconciliation + # (_reconcile_and_store_verdict) can find one attempt's usage events + # without inventing a separate cross-process identity mapping. + agent = Agent(config=config, session_id=trace_id) + _record_attempt_manifest( + trace_id=trace_id, + checkpoint="worker_start", + llm_config=profile.llm_config, + run_id=run_id, + ) await _initialize_agent( agent, retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), @@ -400,6 +433,24 @@ async def run_task( finally: print("Cleaning up resources...") + attempt_profile = locals().get("profile") + if attempt_profile is not None: + _record_attempt_manifest( + trace_id=trace_id, + checkpoint="termination", + llm_config=attempt_profile.llm_config, + run_id=run_id, + ) + # Reconciliation runs regardless of task outcome (success, failed + # status, exception, or cancellation) — identity verification is + # orthogonal to whether the mobile task itself succeeded, and a + # failed attempt's usage receipts still need checking against its + # manifest. Reconciles against the "launch" manifest checkpoint, + # since that is the one guaranteed to have been stored before any + # inference call could have happened. Also runs batch + # reconciliation across run_id when a real (non-default) run_id + # was passed to this invocation. + _reconcile_and_store_verdict(trace_id=trace_id, checkpoint="launch", run_id=run_id) if agent: try: await agent.clean() @@ -412,7 +463,13 @@ async def run_task( log_file_err.close() -if __name__ == "__main__": +def _build_arg_parser() -> argparse.ArgumentParser: + """Builds the CLI parser for this module's ``__main__`` entry point. + + Factored out of ``if __name__ == "__main__":`` so tests can exercise the + real CLI surface (e.g. ``--run-id`` acceptance) via ``parse_args()`` + without spawning a subprocess. + """ parser = argparse.ArgumentParser(description="Artemis Background Task Runner") parser.add_argument("--trace-id", required=True, help="Unique trace identifier") parser.add_argument("--task-desc", required=True, help="Description of the task to run") @@ -434,8 +491,22 @@ async def run_task( "--explorer-pro-mode", help="Pro-profile Explorer perception version: 'flash', 'pro' or 'ultra'", ) + parser.add_argument( + "--run-id", + help=( + "Optional Gate 1 batch-grouping identifier shared across multiple " + "run_task invocations (e.g. repeated attempts of one journey/device " + "cell). Defaults to --trace-id when omitted, preserving today's " + "exact 1:1 run_id<->trace_id behavior; see " + "attempt_reconciliation.reconcile_attempt_batch_by_run_id." + ), + ) - args = parser.parse_args() + return parser + + +if __name__ == "__main__": + args = _build_arg_parser().parse_args() asyncio.run( run_task( @@ -449,5 +520,6 @@ async def run_task( device_serial=args.device_serial, verification_level=args.verification_level, explorer_pro_mode=args.explorer_pro_mode, + run_id=args.run_id, ) ) diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index d30c42eb..1b8fb5c9 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -212,6 +212,7 @@ def mobile_run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Starts an autonomous mobile UI automation subagent on a connected Android device. @@ -277,6 +278,12 @@ def mobile_run_task( by the Operator: `"flash"` (1-shot detection, the default), `"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest). Ignored for Flash. + run_id: Optional Gate 1 batch-grouping identifier. Plumbing for + grouping multiple real attempts (e.g. repeated attempts of one + journey/device cell) into one reconcilable batch at termination — + no built-in caller sets this today; omit unless you are + deliberately re-running the same logical task under a shared + identifier across multiple `mobile_run_task` calls. """ # 0. Validate and normalize model if model.lower() not in ("flash", "pro"): @@ -325,6 +332,7 @@ def mobile_run_task( conversation_id=conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + run_id=run_id, base_url=base_url, ) if resp and resp.get("status") == "rejected": @@ -471,6 +479,8 @@ def mobile_run_task( cmd.extend(["--verification-level", verification_level]) if explorer_mode: cmd.extend(["--explorer-pro-mode", explorer_mode]) + if run_id: + cmd.extend(["--run-id", run_id]) env = os.environ.copy() env["ARTEMIS_SESSION_ID"] = trace_id diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 8c60bf9a..ef057ad5 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -470,6 +470,60 @@ async def fake_subprocess_exec(*args, **kwargs): pass +@pytest.mark.asyncio +async def test_queue_worker_cmd_forwards_run_id(): + """A daemon-dispatched task carrying a Gate 1 run_id must reach the spawned + `artemis.main` worker as `--run-id` -- otherwise the daemon path never + records/reconciles an attempt manifest under that batch key (see + artemis.config.attempt_lifecycle_hooks and + artemis.interfaces.cli.commands.run.execute_task).""" + executed_cmds = [] + + async def fake_subprocess_exec(*args, **kwargs): + executed_cmds.append(list(args)) + proc = MagicMock() + proc.pid = 88889 + proc.wait = AsyncMock(return_value=0) + proc.returncode = 0 + return proc + + with ( + patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec), + patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo, + patch("apps.admin_console.services.task_queue_service.media_service"), + patch( + "artemis.runtime.device_pool.device_pool.select_device_async", + return_value="emulator-5554", + ), + ): + mock_repo.get_running_session_id.return_value = None + mock_repo.get_video_recording_for_session.return_value = {"status": "ready"} + + await task_queue_service.enqueue_tasks( + ["Test Goal with run_id"], + profile="flash", + run_id="daemon-batch-1", + ) + + for _ in range(30): + if len(executed_cmds) == 1 and len(state.queue_items) == 0: + break + await asyncio.sleep(0.05) + + assert len(executed_cmds) == 1 + cmd = executed_cmds[0] + assert "--run-id" in cmd + assert cmd[cmd.index("--run-id") + 1] == "daemon-batch-1" + + task = state.worker_task + if task and not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + @pytest.mark.asyncio async def test_forward_worker_output_preserves_split_utf8(capsys): stream = asyncio.StreamReader() diff --git a/tests/unit/cli/test_run_command_gate1.py b/tests/unit/cli/test_run_command_gate1.py new file mode 100644 index 00000000..5d944c78 --- /dev/null +++ b/tests/unit/cli/test_run_command_gate1.py @@ -0,0 +1,224 @@ +"""Gate 1 evidence regression tests for the daemon-dispatched / direct-CLI +``execute_task`` path (``artemis.interfaces.cli.commands.run``). + +Covers three defects found in independent review of the initial daemon-path +wiring: + +1. The default ``--standalone`` route (no explicit ``--session-id``) must + still generate a canonical trace identity so a real inference run is + never Gate-1-evidence-free by default. +2. ``reconcile_and_store_verdict`` must run before -- and regardless of -- + ``agent.clean()``, since ``Agent.clean()`` can raise (e.g. on device + disconnect) and must never be able to suppress the reconciliation + verdict for a real completed run. +3. A generated fallback identity from one default-route call must never + leak into a later default-route call in the same process via the + process-global ``ARTEMIS_SESSION_ID`` env var -- each attempt must get + its own identity and its own stored manifest. +4. An explicit ``--session-id`` on one call must not contaminate a *later* + default-route call either -- ``ARTEMIS_SESSION_ID`` must be restored to + whatever it held before the call (or cleared), not just skipped for the + generated-fallback case. +""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.interfaces.cli.commands import run as run_module + + +def _fake_llm_config(): + """A real, minimal LLMConfig (AgentProfile validates this via pydantic, + so a bare MagicMock is rejected) -- provider/model values don't matter + here since record_attempt_manifest/reconcile_and_store_verdict are + patched out in every test below.""" + from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback + + node = LLMWithFallback( + provider="google", + model="gemini-3.8-flash", + fallback=LLM(provider="google", model="gemini-3.8-flash"), + ) + required_nodes = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + ) + return LLMConfig( + **{n: node for n in required_nodes}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +def _fake_agent(clean_side_effect=None): + agent = MagicMock() + agent.init = AsyncMock() + agent.run_task = AsyncMock(return_value="ok") + agent.clean = AsyncMock(side_effect=clean_side_effect) + task = MagicMock() + task.build.return_value = "built-task" + agent.new_task.return_value = task + return agent + + +@pytest.mark.asyncio +async def test_execute_task_default_standalone_route_still_records_gate1_evidence(monkeypatch): + """No --session-id, no env session vars: execute_task must still record + and reconcile a real attempt under a generated trace identity, not skip + Gate 1 evidence for the default route.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + agent = _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", return_value=agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="do the thing", session_id=None) + + assert len(recorded) == 1 + assert recorded[0] # non-empty generated identity + assert reconciled == recorded # same trace_id reconciled as was recorded + + +@pytest.mark.asyncio +async def test_execute_task_two_default_calls_get_distinct_isolated_identities(monkeypatch): + """Two back-to-back default-route calls (no --session-id, no env session + vars) in the same process must each get their own generated trace id and + each record their own manifest -- a generated fallback identity must + never leak into the process env and get reused by a later default call, + which would silently merge two attempts under one identity.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + + def _make_agent(*args, **kwargs): + return _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", side_effect=_make_agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="first attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + await run_module.execute_task(goal="second attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + assert len(recorded) == 2 + assert recorded[0] and recorded[1] + assert recorded[0] != recorded[1] # distinct identities -> distinct manifests stored + assert reconciled == recorded + + +@pytest.mark.asyncio +async def test_execute_task_explicit_session_id_does_not_contaminate_later_default_call( + monkeypatch, +): + """An explicit --session-id call followed by a default-route call in the + same process must not collide: the first call's explicit identity must + not leak via ARTEMIS_SESSION_ID and get picked up as the second call's + "default" identity, which would merge two distinct attempts (and their + create-only manifests) under one trace id.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + + def _make_agent(*args, **kwargs): + return _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", side_effect=_make_agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="explicit attempt", session_id="explicit-sid-123") + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + await run_module.execute_task(goal="default attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + assert recorded == ["explicit-sid-123", recorded[1]] + assert recorded[1] != "explicit-sid-123" # default call got its own identity, not the leak + assert reconciled == recorded + + +@pytest.mark.asyncio +async def test_execute_task_reconciles_before_cleanup_and_survives_cleanup_failure(monkeypatch): + """A raising agent.clean() must not prevent reconcile_and_store_verdict + from running, and reconciliation must be observed to happen first.""" + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + call_order = [] + agent = _fake_agent(clean_side_effect=RuntimeError("device disconnected")) + agent.clean.side_effect = None # set below after wrapping to record order + + async def clean_raises(): + call_order.append("clean") + raise RuntimeError("device disconnected") + + agent.clean = AsyncMock(side_effect=clean_raises) + + def reconcile_records(**kw): + call_order.append("reconcile") + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", return_value=agent), + patch.object(run_module, "record_attempt_manifest"), + patch.object(run_module, "reconcile_and_store_verdict", side_effect=reconcile_records), + ): + # Must not raise: a cleanup failure must not propagate out of execute_task. + await run_module.execute_task(goal="do the thing", session_id="fixed-sid") + + assert call_order == ["reconcile", "clean"] diff --git a/tests/unit/config/test_attempt_manifest.py b/tests/unit/config/test_attempt_manifest.py new file mode 100644 index 00000000..2229babc --- /dev/null +++ b/tests/unit/config/test_attempt_manifest.py @@ -0,0 +1,511 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate 1 mechanism tests: manifest determinism, storage, and the decoy-config +isolation control. + +All tests use ``tmp_path``/``monkeypatch`` exclusively. None of them touch the +real home directory, the real ``~/.artemis``/PAL config dir, or make any +provider/network call. +""" + +from __future__ import annotations + +import json +import threading + +import pytest + +from artemis.config.attempt_manifest import ( + ManifestAlreadyExistsError, + build_attempt_manifest, + canonical_bytes, + digest_of, + read_stored_manifest, + store_attempt_manifest, +) +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.runtime import trace_store + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _uniform_config(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to one tier.""" + base = {node: _llm(provider, model) for node in _REQUIRED_NODES} + return LLMConfig( + utils=LLMConfigUtils(outputter=_llm(provider, model), hopper=_llm(provider, model)), + **base, + ) + + +def _sol_config(**node_overrides: LLMWithFallback) -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to 'sol'.""" + base = {node: _llm() for node in _REQUIRED_NODES} + base.update(node_overrides) + return LLMConfig(utils=LLMConfigUtils(outputter=_llm(), hopper=_llm()), **base) + + +@pytest.fixture(autouse=True) +def _isolate_traces_dir(tmp_path, monkeypatch): + """Point trace_store at an isolated tmp_path directory for every test here. + + This is the concrete guard against ever touching a real traces directory: + every test in this module writes manifests under ``tmp_path``, never under + the process's real TRACES_DIR. + """ + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + + +class TestCanonicalDigestStability: + """Requirement 2: same logical manifest -> byte-identical canonical form.""" + + def test_same_manifest_different_key_order_same_bytes_and_digest(self): + config = _sol_config() + manifest_a = build_attempt_manifest( + run_id="run-1", + attempt_id="attempt-1", + trace_id="trace-1", + tier="sol", + llm_config=config, + env={"ARTEMIS_CONFIG_DIR": "/tmp/does-not-matter"}, + checkpoint="launch", + now=1000.0, + ) + + # Same logical content, deliberately constructed via a different + # insertion order (dict equality ignores order, but a naive + # json.dumps without sort_keys would NOT produce identical bytes). + reordered = {k: manifest_a[k] for k in reversed(list(manifest_a.keys()))} + reordered["nodes"] = { + k: manifest_a["nodes"][k] for k in reversed(list(manifest_a["nodes"].keys())) + } + + assert reordered == manifest_a + assert canonical_bytes(reordered) == canonical_bytes(manifest_a) + assert digest_of(reordered) == digest_of(manifest_a) + + def test_canonical_bytes_have_no_whitespace_ambiguity(self): + manifest = build_attempt_manifest( + run_id="run-1", + attempt_id="attempt-1", + trace_id="trace-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1000.0, + ) + raw = canonical_bytes(manifest) + # No ambiguous JSON *structural* whitespace: a naive json.dumps(..., + # indent=2) would introduce ", " item separators and ": " key + # separators immediately followed by a structural character; our + # canonical form uses bare "," and ":" everywhere a separator is + # structurally required. Prose values (e.g. "no soft default: ...") + # legitimately contain ": " as English punctuation, so assert on the + # compact-separator round trip instead of a raw substring search. + assert ( + json.dumps(manifest, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode( + "utf-8" + ) + == raw + ) + assert b"\n" not in raw + assert b" " not in raw # no accidental double-space from indent=2-style output + # Round-trips to the same logical object. + assert json.loads(raw.decode("utf-8")) == manifest + + def test_digest_changes_when_a_non_tier_field_changes(self): + config_a = _sol_config() + # temperature is not part of the tier match (TIER_MODELS only looks + # at provider/model), so this stays within tier 'sol' while still + # changing a field that must affect the digest. + config_b = _sol_config( + planner=_llm(model="gpt-5.6-sol").model_copy(update={"temperature": 0.9}) + ) + manifest_a = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config_a, + env={}, + checkpoint="launch", + now=1.0, + ) + manifest_b = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config_b, + env={}, + checkpoint="launch", + now=1.0, + ) + assert digest_of(manifest_a) != digest_of(manifest_b) + + def test_mixed_tier_within_one_manifest_is_rejected_at_build_time(self): + config = _sol_config(planner=_llm(provider="google", model="gemini-3.5-flash-lite")) + with pytest.raises(ValueError, match="mixed_tier|other tier"): + build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + + +class TestNullableNodesRepresentedExplicitly: + def test_disabled_nodes_show_would_resolve_default_without_being_active(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + pixel_safety = manifest["nodes"]["validator_pixel_safety_net"] + assert pixel_safety["enabled"] is False + assert pixel_safety["config"] is None + assert pixel_safety["would_resolve_to"]["provider"] == "google" + assert pixel_safety["would_resolve_to"]["model"] == "gemini-3.5-flash-lite" + + history_analyzer = manifest["nodes"]["history_analyzer"] + assert history_analyzer["enabled"] is False + assert history_analyzer["would_resolve_provenance"] == "soft-default: inherits 'operator'" + + video_analyzer = manifest["utils"]["video_analyzer"] + assert video_analyzer["enabled"] is False + assert video_analyzer["would_resolve_to"] is None + assert "no soft default" in video_analyzer["would_resolve_provenance"] + + def test_fake_llm_flag_and_credentials_never_leak_secrets(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={"OPENAI_API_KEY": "sk-supersecretvalue1234", "ARTEMIS_FAKE_LLM": "0"}, + checkpoint="launch", + now=1.0, + ) + assert manifest["fake_llm_enabled"] is False + raw = canonical_bytes(manifest) + assert b"sk-supersecretvalue1234" not in raw + assert manifest["credentials"]["openai"]["set"] is True + assert manifest["credentials"]["openai"]["last4"] == "1234" + + def test_credential_shaped_artemis_env_vars_never_leak_raw_value(self): + """ARTEMIS_TENANT_TOKEN (a real secret-bearing var, see + artemis.config.constants.ENV_ARTEMIS_TENANT_TOKEN) must never appear + by raw value in env_overrides or the canonical bytes -- only a + set/last4 presence reference, exactly like _credential_reference. + """ + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={ + "ARTEMIS_TENANT_TOKEN": "super-secret-tenant-token-value", + "ARTEMIS_LIFECYCLE_TOKEN": "another-secret-lifecycle-value", + "ARTEMIS_FAKE_LLM": "0", + "ARTEMIS_CONFIG_DIR": "/tmp/does-not-matter", + }, + checkpoint="launch", + now=1.0, + ) + raw = canonical_bytes(manifest) + assert b"super-secret-tenant-token-value" not in raw + assert b"another-secret-lifecycle-value" not in raw + + tenant_ref = manifest["env_overrides"]["ARTEMIS_TENANT_TOKEN"] + assert tenant_ref == {"set": True, "last4": "alue"} + lifecycle_ref = manifest["env_overrides"]["ARTEMIS_LIFECYCLE_TOKEN"] + assert lifecycle_ref == {"set": True, "last4": "alue"} + + # Non-sensitive ARTEMIS_* resolution knobs still pass through as raw + # values -- this fix must not overcorrect into hashing everything. + assert manifest["env_overrides"]["ARTEMIS_CONFIG_DIR"] == "/tmp/does-not-matter" + assert manifest["env_overrides"]["ARTEMIS_FAKE_LLM"] == "0" + + def test_unset_credential_shaped_env_var_reports_set_false_without_last4(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={"ARTEMIS_TENANT_TOKEN": ""}, + checkpoint="launch", + now=1.0, + ) + assert manifest["env_overrides"]["ARTEMIS_TENANT_TOKEN"] == {"set": False} + + +class TestStorageCreateOnly: + def test_store_and_read_back_bytes_match(self, tmp_path): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="trace-store-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + path, digest = store_attempt_manifest(manifest) + assert path.exists() + assert digest == digest_of(manifest) + + stored_bytes, stored_digest = read_stored_manifest("trace-store-1", "launch") + assert stored_bytes == canonical_bytes(manifest) + assert stored_digest == digest + + def test_second_write_for_same_trace_and_checkpoint_is_rejected(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="trace-store-2", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + store_attempt_manifest(manifest) + with pytest.raises(ManifestAlreadyExistsError): + store_attempt_manifest(manifest) + + def test_concurrent_writers_never_overwrite_each_others_bytes(self): + """Reproduces the TOCTOU this storage must not have. + + A prior implementation checked ``Path.exists()`` then wrote via + ``os.replace()`` — a second writer landing between those two steps + silently clobbered the first writer's already-stored bytes instead of + raising. This drives two logically different manifests (same + trace/checkpoint key, different ``run_id``) through + ``store_attempt_manifest`` back-to-back and asserts the file on disk + still holds exactly the first writer's bytes, with the second writer + observing ``ManifestAlreadyExistsError`` rather than winning a race. + """ + first = build_attempt_manifest( + run_id="first-writer", + attempt_id="a", + trace_id="trace-race-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + second = build_attempt_manifest( + run_id="second-writer", + attempt_id="a", + trace_id="trace-race-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=2.0, + ) + assert canonical_bytes(first) != canonical_bytes(second) + + first_path, first_digest = store_attempt_manifest(first) + with pytest.raises(ManifestAlreadyExistsError): + store_attempt_manifest(second) + + stored_bytes, stored_digest = read_stored_manifest("trace-race-1", "launch") + assert stored_bytes == canonical_bytes(first) + assert stored_digest == first_digest + assert first_path.read_bytes() == canonical_bytes(first) + + def test_two_threads_racing_the_same_key_leave_exactly_one_winner(self): + """True concurrent reproduction: two threads, one (trace_id, checkpoint). + + Exactly one thread's bytes must land on disk and the other must + observe ``ManifestAlreadyExistsError`` — never a torn write and never + a silent overwrite of the winner by the loser. + """ + manifest_by_run_id = { + run_id: build_attempt_manifest( + run_id=run_id, + attempt_id="a", + trace_id="trace-race-2", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=float(i), + ) + for i, run_id in enumerate(("racer-a", "racer-b")) + } + results: dict[str, tuple[str, str] | Exception] = {} + start = threading.Barrier(2) + + def _race(run_id: str) -> None: + start.wait() + try: + results[run_id] = store_attempt_manifest(manifest_by_run_id[run_id]) + except ManifestAlreadyExistsError as exc: + results[run_id] = exc + + threads = [threading.Thread(target=_race, args=(rid,)) for rid in manifest_by_run_id] + for t in threads: + t.start() + for t in threads: + t.join() + + winners = [rid for rid, res in results.items() if not isinstance(res, Exception)] + losers = [rid for rid, res in results.items() if isinstance(res, Exception)] + assert len(winners) == 1 + assert len(losers) == 1 + assert isinstance(results[losers[0]], ManifestAlreadyExistsError) + + winning_manifest = manifest_by_run_id[winners[0]] + stored_bytes, stored_digest = read_stored_manifest("trace-race-2", "launch") + assert stored_bytes == canonical_bytes(winning_manifest) + assert stored_digest == digest_of(winning_manifest) + + def test_different_checkpoints_for_same_trace_coexist(self): + base_kwargs = dict( + run_id="r", + attempt_id="a", + trace_id="trace-store-3", + tier="sol", + llm_config=_sol_config(), + env={}, + ) + launch = build_attempt_manifest(**base_kwargs, checkpoint="launch", now=1.0) + worker_start = build_attempt_manifest(**base_kwargs, checkpoint="worker_start", now=2.0) + termination = build_attempt_manifest(**base_kwargs, checkpoint="termination", now=3.0) + + store_attempt_manifest(launch) + store_attempt_manifest(worker_start) + store_attempt_manifest(termination) + + for checkpoint in ("launch", "worker_start", "termination"): + assert read_stored_manifest("trace-store-3", checkpoint) is not None + + +class TestDecoyConfigIsolation: + """Requirement 3(b): decoy higher-priority config file must not leak + across independently-resolved attempts, and a stored manifest must never + change after the fact. + + This exercises the *env-var-driven config resolution layer in-process*: + ``build_attempt_manifest`` never reads ``os.environ`` itself, so the + "decoy" here is simulated purely via the explicit ``env`` dict and an + isolated ``tmp_path`` config directory -- the real ``get_config_path()`` + cascade, the real user home directory, and the real MCP/device process + boundary are NOT exercised by this test (see the PR description / final + report for the explicit isolation-boundary callout). + """ + + def test_attempt_a_manifest_and_digest_survive_later_decoy_mutation(self, tmp_path): + decoy_dir = tmp_path / "decoy-config-dir" + decoy_dir.mkdir() + decoy_override = decoy_dir / "llm-config.override.jsonc" + decoy_override.write_text('{"nodes": {}}', encoding="utf-8") + + # Attempt A resolves using an explicit env snapshot that "points at" + # the decoy dir (simulating ARTEMIS_CONFIG_DIR) plus a config object + # already fully resolved for tier 'sol'. Nothing here reads the decoy + # file's contents -- build_attempt_manifest only ever consumes + # explicit llm_config/env/tier, which is exactly the property under + # test: an env var *naming* a decoy path cannot retroactively alter a + # manifest that was built from a different, already-resolved config. + env_pointing_at_decoy = {"ARTEMIS_CONFIG_DIR": str(decoy_dir)} + manifest_a = build_attempt_manifest( + run_id="run-iso", + attempt_id="attempt-a", + trace_id="trace-iso-a", + tier="sol", + llm_config=_sol_config(), + env=env_pointing_at_decoy, + checkpoint="launch", + now=10.0, + ) + path_a, digest_a = store_attempt_manifest(manifest_a) + bytes_a_before = path_a.read_bytes() + + # Mutate the decoy file to something that would resolve to a + # different (mixed) tier if it were ever loaded and passed in. + decoy_override.write_text( + '{"nodes": {"planner": {"provider": "anthropic", "model": "claude-x"}}}', + encoding="utf-8", + ) + + # Attempt B resolves independently -- its own explicit llm_config, + # built fresh, deliberately picking a different tier to prove + # resolution is not sticky/global. + manifest_b = build_attempt_manifest( + run_id="run-iso", + attempt_id="attempt-b", + trace_id="trace-iso-b", + tier="luna", + llm_config=_uniform_config(provider="google", model="gemini-3.5-flash-lite"), + env={"ARTEMIS_CONFIG_DIR": str(decoy_dir)}, + checkpoint="launch", + now=20.0, + ) + store_attempt_manifest(manifest_b) + + # Attempt A's already-stored manifest is untouched, byte-for-byte. + bytes_a_after = path_a.read_bytes() + assert bytes_a_after == bytes_a_before + _, digest_a_reread = read_stored_manifest("trace-iso-a", "launch") + assert digest_a_reread == digest_a + + # Attempt B resolved to its own distinct tier/digest -- the decoy + # mutation influenced neither attempt (A never re-read it; B never + # read it at all, since resolution is a pure function of the + # explicit llm_config argument). + assert manifest_a["tier"] == "sol" + assert manifest_b["tier"] == "luna" + assert digest_of(manifest_a) != digest_of(manifest_b) + + # Never touched a real home directory or the real PAL config dir. + assert str(decoy_dir).startswith(str(tmp_path)) diff --git a/tests/unit/config/test_attempt_reconciliation.py b/tests/unit/config/test_attempt_reconciliation.py new file mode 100644 index 00000000..21c3d7a0 --- /dev/null +++ b/tests/unit/config/test_attempt_reconciliation.py @@ -0,0 +1,758 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate 1 mechanism tests: llm_usage <-> manifest reconciliation and the +seven-reason batch accept/reject rule. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from artemis.config.attempt_manifest import ( + build_attempt_manifest, + digest_of, + store_attempt_manifest, +) +from artemis.config.attempt_reconciliation import ( + AttemptRecord, + REJECT_REASONS, + reconcile_attempt, + reconcile_attempt_batch_by_run_id, + reconcile_finished_attempt, + validate_batch, +) +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.data_engine.models import SessionMetadata, TraceRecord +from artemis.data_engine.storage import StorageManager +from artemis.runtime import trace_store + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _uniform_config(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to one tier.""" + node = _llm(provider, model) + return LLMConfig( + **{n: node for n in _REQUIRED_NODES}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +def _sol_config() -> LLMConfig: + return _uniform_config() + + +def _manifest(**overrides): + kwargs = dict( + run_id="r", + attempt_id="a1", + trace_id="t1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + kwargs.update(overrides) + return build_attempt_manifest(**kwargs) + + +def _usage(node: str, source: str | None = "openai:gpt-5.6-sol") -> dict: + event = {"node": node, "prompt_tokens": 10, "completion_tokens": 5} + if source is not None: + event["source"] = source + return event + + +class TestReconciliation: + def test_matching_source_is_match(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner")]) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "match" + assert not result.has_mismatch + + def test_mismatched_source_flags_mismatch(self): + manifest = _manifest() + result = reconcile_attempt( + "a1", manifest, [_usage("planner", source="google:gemini-3.5-flash-lite")] + ) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "mismatch" + assert result.has_mismatch + + def test_missing_source_is_unverified_identity_not_inferred(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner", source=None)]) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "unverified_identity" + assert result.has_unverified_identity + + def test_usage_for_node_absent_from_manifest_is_unmapped_call(self): + manifest = _manifest() + result = reconcile_attempt( + "a1", manifest, [_usage("some_new_node_not_in_manifest", source="openai:gpt-5.6-sol")] + ) + unmapped = next(n for n in result.nodes if n.node == "some_new_node_not_in_manifest") + assert unmapped.verdict == "unmapped_call" + assert result.has_unmapped_call + + def test_enabled_node_never_invoked_is_not_invoked_informational(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, []) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "not_invoked" + + def test_disabled_node_never_invoked_is_disabled_not_invoked(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, []) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "disabled_not_invoked" + + def test_never_fabricates_zero_usage_for_non_firing_node(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner")]) + # Every other enabled node with no usage event is not_invoked, never + # synthesized as a zero-usage "match". + others = [n for n in result.nodes if n.node != "planner" and n.manifest_enabled] + assert others + assert all(n.verdict == "not_invoked" for n in others) + + def test_aliased_trace_node_name_reconciles_against_manifest_node(self): + """validator_pixel_safety_net is traced under "safety_net_pixel_validation" + (see artemis.agents.validator.validator._validate_action_precondition_pixel's + @trace name), so a usage event recorded with that trace-scope node + name must reconcile as a match against the manifest's + validator_pixel_safety_net entry, not surface as unmapped_call. + """ + # Enable validator_pixel_safety_net explicitly so it has a real + # manifest entry to reconcile against. + config = _uniform_config() + config = config.model_copy(update={"validator_pixel_safety_net": _llm()}) + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a1", + trace_id="t1", + tier="sol", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="openai:gpt-5.6-sol")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "match" + assert not result.has_unmapped_call + # No stray "safety_net_pixel_validation" entry should remain unmapped. + assert not any(n.node == "safety_net_pixel_validation" for n in result.nodes) + + def test_soft_defaulted_node_with_would_resolve_to_match_reconciles_as_match(self): + """validator_pixel_safety_net left unset (the normal/default state) is + soft-defaulted by LLMConfig.get_agent() to lightweight_judge_default() + (google:gemini-3.5-flash-lite, a Luna-tier model) when actually + invoked. The manifest entry is `enabled=False` (correctly recording it + wasn't explicitly configured) but carries `would_resolve_to` for + exactly this case. This is the *legitimate* positive case: a manifest + whose own declared `tier` genuinely IS "luna" (i.e. its enabled nodes + already resolve to Luna-tier models), so the soft default's tier + agrees with the attempt's own tier. A receipt matching + `would_resolve_to` here is a legitimate, expected production + occurrence and must reconcile as "match", not "mismatch". + """ + luna_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + manifest = _manifest(tier="luna", llm_config=luna_config) + node_entry = manifest["nodes"]["validator_pixel_safety_net"] + assert node_entry["enabled"] is False + assert node_entry["would_resolve_to"] == { + "provider": "google", + "model": "gemini-3.5-flash-lite", + "api_base": None, + "temperature": 0.0, + "thinking_budget": None, + "thinking_level": None, + "reasoning_effort": None, + "include_thoughts": None, + "enable_grounding": None, + "fallback": { + "provider": "google", + "model": "gemini-3.1-flash-lite", + "api_base": None, + "temperature": 0.0, + "thinking_budget": None, + "thinking_level": None, + "reasoning_effort": None, + "include_thoughts": None, + "enable_grounding": None, + }, + "fix_model": None, + "timeout": None, + } + + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "match" + # The manifest's own `enabled` field must stay honest: this node was + # not explicitly configured, even though the receipt matched. + assert pixel_safety.manifest_enabled is False + + record = _record( + manifest, [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")] + ) + verdict = validate_batch([record]) + assert verdict.accepted is True + + def test_soft_defaulted_node_cross_tier_would_resolve_to_is_rejected_not_matched(self): + """Negative control for the cross-tier-mixing defect: a `tier="sol"` + manifest's soft-defaulted validator_pixel_safety_net still would + resolve to a Luna-tier model (google:gemini-3.5-flash-lite) if + invoked, since would_resolve_to is a fixed function of the node, not + of the attempt's own declared tier. A receipt matching that Luna-tier + would_resolve_to must NOT reconcile as "match" against a Sol-pinned + attempt -- that would let an attempt pinned to Sol tier silently + invoke a Luna-tier model via the soft-default path. It must reconcile + as "mismatch", and the batch must be rejected, not accepted. + """ + manifest = _manifest(tier="sol") # validator_pixel_safety_net left None (default). + node_entry = manifest["nodes"]["validator_pixel_safety_net"] + assert node_entry["enabled"] is False + assert node_entry["would_resolve_to"]["provider"] == "google" + assert node_entry["would_resolve_to"]["model"] == "gemini-3.5-flash-lite" + + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "mismatch" + assert result.has_mismatch + + record = _record( + manifest, [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")] + ) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_soft_defaulted_node_with_mismatched_source_still_reconciles_as_mismatch(self): + """Negative control: this fix must not turn off legitimate mismatch + detection for soft-defaulted nodes -- only make the *expected* source + honest (derived from would_resolve_to instead of always None).""" + manifest = _manifest() # validator_pixel_safety_net left None (default). + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="openai:gpt-5.6-sol")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "mismatch" + assert result.has_mismatch + + +_UNSET = object() + + +def _record(manifest, usage_events, digest=_UNSET) -> AttemptRecord: + reconciliation = reconcile_attempt(manifest["attempt_id"], manifest, usage_events) + return AttemptRecord( + attempt_id=manifest["attempt_id"], + manifest=manifest, + reconciliation=reconciliation, + stored_digest=digest_of(manifest) if digest is _UNSET else digest, + ) + + +class TestBatchValidationAllSevenReasons: + def test_empty_batch_is_missing_snapshot(self): + verdict = validate_batch([]) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" + + def test_accept_when_batch_is_clean(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.accepted is True + assert verdict.reason is None + assert verdict.invalid_attempts == () + + def test_missing_identity(self): + manifest = _manifest() + broken = copy.deepcopy(manifest) + broken["source_sha"] = "" + record = _record(broken, [_usage("planner")], digest=digest_of(broken)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_missing_identity_when_source_sha_is_unverified_sentinel(self): + """A truthy but never-git-verified source_sha (the "0"*40 fallback + _canonical_repo_sha emits when the tree isn't a real git checkout) + must not silently pass as a real identity.""" + manifest = _manifest() + unverified = copy.deepcopy(manifest) + unverified["source_sha"] = "0" * 40 + unverified["source_sha_provenance"] = "unknown-not-a-git-checkout" + record = _record(unverified, [_usage("planner")], digest=digest_of(unverified)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_missing_identity_when_untiered_enabled_node_present(self): + """An enabled node with no declared tier (custom/unknown-model) must + fail batch validation, even though it built successfully and didn't + trip the mixed_tier build-time check.""" + manifest = _manifest() + untiered = copy.deepcopy(manifest) + untiered["untiered_enabled_nodes"] = ["some_custom_node"] + record = _record(untiered, [_usage("planner")], digest=digest_of(untiered)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_fake_mode_enabled(self): + manifest = _manifest(env={"ARTEMIS_FAKE_LLM": "1"}) + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "fake_mode_enabled" + + def test_missing_snapshot_when_no_stored_digest(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner")], digest=None) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" + + def test_digest_drift_when_manifest_mutated_after_storing(self): + manifest = _manifest() + original_digest = digest_of(manifest) + tampered = copy.deepcopy(manifest) + tampered["nodes"]["planner"]["config"]["model"] = "some-other-model" + record = _record(tampered, [_usage("planner")], digest=original_digest) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "digest_drift" + # The original (tampered) record is preserved for audit, not discarded. + assert verdict.invalid_attempts[0].manifest["nodes"]["planner"]["config"]["model"] == ( + "some-other-model" + ) + + def test_mixed_tier_across_batch(self): + sol_manifest = _manifest(tier="sol") + luna_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + luna_manifest = build_attempt_manifest( + run_id="r", + attempt_id="a2", + trace_id="t2", + tier="luna", + llm_config=luna_config, + env={}, + checkpoint="launch", + now=2.0, + ) + records = [ + _record(sol_manifest, [_usage("planner")]), + _record(luna_manifest, [_usage("planner", source="google:gemini-3.5-flash-lite")]), + ] + verdict = validate_batch(records) + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + assert len(verdict.invalid_attempts) == 2 + + def test_unmapped_call(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner"), _usage("totally_unknown_node")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unmapped_call" + + def test_unexpected_model_or_endpoint_from_mismatch(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner", source="anthropic:claude-x")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_unexpected_model_or_endpoint_from_unverified_identity(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner", source=None)]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_all_seven_reject_reasons_are_reachable_and_documented(self): + # Sanity check that the reasons enumerated in the spec are exactly + # what REJECT_REASONS declares -- catches drift if a reason is + # renamed in one place but not the other. + assert set(REJECT_REASONS) == { + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", + } + + def test_rejection_preserves_original_attempt_data_unmodified(self): + manifest = _manifest(env={"ARTEMIS_FAKE_LLM": "1"}) + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.invalid_attempts[0].manifest is manifest + assert verdict.invalid_attempts[0].reconciliation is record.reconciliation + + +@pytest.fixture +def _isolate_traces_dir(tmp_path, monkeypatch): + """Point trace_store at an isolated tmp_path directory for this test only. + + Never touches the process's real TRACES_DIR or the real home directory. + """ + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + + +def _seeded_db(tmp_path, session_id: str): + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + storage.create_session(SessionMetadata(session_id=session_id, initial_goal="test goal")) + return storage, db_path, traces_dir + + +class TestReconcileFinishedAttempt: + """Post-run orchestration: stored manifest + native DB receipts -> verdict. + + Uses only tmp_path-backed trace storage and a tmp_path SQLite DB seeded + via a writable StorageManager -- never the real traces directory or the + real DataEngine database. + """ + + def test_no_stored_manifest_returns_none_not_a_verdict(self, tmp_path, _isolate_traces_dir): + _storage, db_path, traces_dir = _seeded_db(tmp_path, "trace-no-manifest") + result = reconcile_finished_attempt( + trace_id="trace-no-manifest", + session_id="trace-no-manifest", + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + assert result is None + + def test_matching_usage_accepts_the_attempt(self, tmp_path, _isolate_traces_dir): + trace_id = "trace-accept" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + + storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{node}", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is True + + def test_mismatched_usage_rejects_with_original_receipts_preserved( + self, tmp_path, _isolate_traces_dir + ): + trace_id = "trace-reject" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + + storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + # planner fires with a model that does not match the manifest's + # pinned 'sol' identity -- a leaked/decoy config would look like this. + storage.create_trace( + TraceRecord( + trace_id="usage-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + # The original mismatched receipt is retained on the reconciliation + # result, not discarded because the batch was rejected. + invalid = verdict.invalid_attempts[0] + planner_node = next(n for n in invalid.reconciliation.nodes if n.node == "planner") + assert planner_node.usage_sources == ("google:gemini-3.5-flash-lite",) + + def test_no_usage_events_at_all_is_not_invoked_not_a_fabricated_match( + self, tmp_path, _isolate_traces_dir + ): + """No DB rows for this session (e.g. the task crashed before any LLM + call) must reconcile every node as not_invoked/disabled_not_invoked, + never a synthesized match.""" + trace_id = "trace-no-usage" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + _storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is True + + +class TestReconcileAttemptBatchByRunId: + """Cross-attempt batch reconciliation grouped by a shared run_id. + + This is the production-reachable home for the mixed_tier rejection path + (reconcile_finished_attempt can never exercise it since it always + validates a batch of exactly one attempt). Uses only tmp_path-backed + trace storage and a tmp_path SQLite DB -- never the real traces + directory or the real DataEngine database. + """ + + def test_two_attempts_same_run_id_same_tier_matching_usage_is_accepted( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-accept" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + for trace_id in ("attempt-1", "attempt-2"): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + manifest = _manifest(run_id=run_id, attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-{node}", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is True + assert verdict.invalid_attempts == () + + def test_mixed_tier_across_attempts_sharing_run_id_is_rejected( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-mixed-tier" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + sol_trace_id = "attempt-sol" + storage.create_session(SessionMetadata(session_id=sol_trace_id, initial_goal="test goal")) + sol_manifest = _manifest(run_id=run_id, attempt_id=sol_trace_id, trace_id=sol_trace_id) + store_attempt_manifest(sol_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{sol_trace_id}-planner", + session_id=sol_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol"}, + ) + ) + + terra_trace_id = "attempt-terra" + storage.create_session(SessionMetadata(session_id=terra_trace_id, initial_goal="test goal")) + terra_config = _uniform_config(provider="google", model="gemini-3.8-flash") + terra_manifest = build_attempt_manifest( + run_id=run_id, + attempt_id=terra_trace_id, + trace_id=terra_trace_id, + tier="terra", + llm_config=terra_config, + env={}, + checkpoint="launch", + now=3.0, + ) + store_attempt_manifest(terra_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{terra_trace_id}-planner", + session_id=terra_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.8-flash"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + assert len(verdict.invalid_attempts) == 2 + invalid_ids = {a.attempt_id for a in verdict.invalid_attempts} + assert invalid_ids == {sol_trace_id, terra_trace_id} + + def test_attempt_under_different_run_id_is_excluded_from_batch( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-target" + other_run_id = "run-other" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + target_trace_id = "attempt-target" + storage.create_session( + SessionMetadata(session_id=target_trace_id, initial_goal="test goal") + ) + target_manifest = _manifest( + run_id=run_id, attempt_id=target_trace_id, trace_id=target_trace_id + ) + store_attempt_manifest(target_manifest) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{target_trace_id}-{node}", + session_id=target_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + # A different run_id, deliberately a different tier -- if this leaked + # into the target batch it would falsely trip mixed_tier. + other_trace_id = "attempt-other-run" + storage.create_session(SessionMetadata(session_id=other_trace_id, initial_goal="test goal")) + other_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + other_manifest = build_attempt_manifest( + run_id=other_run_id, + attempt_id=other_trace_id, + trace_id=other_trace_id, + tier="luna", + llm_config=other_config, + env={}, + checkpoint="launch", + now=4.0, + ) + store_attempt_manifest(other_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{other_trace_id}-planner", + session_id=other_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is True + assert len(verdict.invalid_attempts) == 0 + + def test_zero_matching_attempts_returns_missing_snapshot(self, tmp_path, _isolate_traces_dir): + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + + verdict = reconcile_attempt_batch_by_run_id( + run_id="run-does-not-exist", + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" diff --git a/tests/unit/config/test_attempt_usage_reader.py b/tests/unit/config/test_attempt_usage_reader.py new file mode 100644 index 00000000..b7c62461 --- /dev/null +++ b/tests/unit/config/test_attempt_usage_reader.py @@ -0,0 +1,155 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only ``llm_usage`` receipt lookup, direct from the DataEngine DB. + +All tests write into a ``tmp_path`` SQLite file via a writable +``StorageManager`` (never the real traces directory), then read it back +through :func:`read_llm_usage_events` exactly as the read-only offline path +would. +""" + +from __future__ import annotations + +import json + +from artemis.config.attempt_usage_reader import read_llm_usage_events +from artemis.data_engine.models import SessionMetadata, TraceRecord +from artemis.data_engine.storage import StorageManager + + +def _seeded_storage(tmp_path, session_id: str) -> tuple[StorageManager, str, str]: + db_path = tmp_path / "data_engine.db" + traces_dir = tmp_path / "traces" + storage = StorageManager(db_path, traces_dir) + storage.create_session(SessionMetadata(session_id=session_id, initial_goal="test goal")) + return storage, str(db_path), str(traces_dir) + + +class TestReadLlmUsageEvents: + def test_returns_only_llm_usage_rows_for_the_given_session(self, tmp_path): + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol", "prompt_tokens": 10}, + ) + ) + storage.create_trace( + TraceRecord( + trace_id="trace-2", + session_id="session-a", + type="tool", + name="ask_explorer", + payload={"args": {}, "result": "irrelevant, not an llm_usage trace"}, + ) + ) + # A different session's llm_usage row must never leak into session-a's read. + storage.create_session(SessionMetadata(session_id="session-b", initial_goal="other")) + storage.create_trace( + TraceRecord( + trace_id="trace-3", + session_id="session-b", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "planner" + assert events[0]["source"] == "openai:gpt-5.6-sol" + + def test_events_with_no_step_id_are_still_found(self, tmp_path): + """The whole point of querying `traces` directly instead of walking + steps: a background/utility llm_usage call recorded with no step_id + must not be silently missed.""" + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + step_id=None, + type="llm_call", + name="llm_usage", + payload={"node": "hopper", "source": "openai:gpt-5.6-sol"}, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "hopper" + + def test_missing_database_returns_empty_list_not_an_error(self, tmp_path): + events = read_llm_usage_events( + tmp_path / "does-not-exist.db", tmp_path / "traces", "session-a" + ) + assert events == [] + + def test_corrupt_payload_row_is_skipped_not_raised(self, tmp_path): + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol"}, + ) + ) + # Directly corrupt one row's payload to simulate a partially-written + # or truncated record, bypassing the Pydantic-validated write path. + with storage._get_connection() as conn: + conn.execute( + "INSERT INTO traces (trace_id, session_id, type, name, timestamp, status, payload) " + "VALUES ('trace-2', 'session-a', 'llm_call', 'llm_usage', 2.0, 'success', ?)", + ("{not valid json",), + ) + conn.commit() + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "planner" + + def test_returned_payload_matches_original_bytes_exactly(self, tmp_path): + """Reconciliation must see the original receipt, not a derived summary.""" + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + original_payload = { + "node": "operator", + "source": "openai:gpt-5.6-sol", + "prompt_tokens": 123, + "completion_tokens": 45, + "session_cached_ratio": 0.5, + } + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload=original_payload, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0] == json.loads(json.dumps(original_payload)) diff --git a/tests/unit/mcp/test_background_task_runner.py b/tests/unit/mcp/test_background_task_runner.py index c0fbf7c8..ec5156d4 100644 --- a/tests/unit/mcp/test_background_task_runner.py +++ b/tests/unit/mcp/test_background_task_runner.py @@ -1,9 +1,15 @@ import asyncio +import json from unittest.mock import AsyncMock, MagicMock import pytest -from mcp_server.background.task_runner import _initialize_agent +from mcp_server.background.task_runner import ( + _build_arg_parser, + _initialize_agent, + _reconcile_and_store_verdict, + _record_attempt_manifest, +) @pytest.mark.asyncio @@ -102,3 +108,277 @@ async def test_run_task_applies_pro_tuning_to_agent_config( fake_builder.with_explorer.assert_called_once_with(pro_mode=expect_mode) fake_agent.run_task.assert_awaited_once() assert trace_store.read_status(trace_id)["status"] == "completed" + + +def _uniform_llm_config(provider: str, model: str): + """A fully-populated LLMConfig where every required node resolves to one tier. + + Mirrors ``tests/unit/config/test_attempt_reconciliation.py``'s + ``_uniform_config`` helper -- kept local to this file since it exercises + the real ``LLMConfig``, not a mock, so ``_record_attempt_manifest`` can + genuinely infer a tier from ``llm_config.planner`` the same way the real + ``run_task`` path does. + """ + from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback + + node = LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + required_nodes = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + ) + return LLMConfig( + **{n: node for n in required_nodes}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +class TestRealWorkerMultiAttemptRunIdGrouping: + """Proves the real ``run_task``-facing ``run_id`` plumbing (not a + synthetic ``build_attempt_manifest`` call) actually produces manifests + that ``reconcile_attempt_batch_by_run_id`` groups together and rejects on + ``mixed_tier`` -- the "real worker output reaches validate_batch" gap + ``TestReconcileAttemptBatchByRunId`` (config-level, synthetic manifests + only) cannot close on its own. + + Calls the real ``_record_attempt_manifest`` (the exact function + ``run_task`` calls, with the same ``run_id`` parameter ``run_task`` now + threads through to it) directly, twice, with a shared ``run_id`` but + distinct ``trace_id``s and tiers -- mirroring this file's existing + tmp_path-isolated ``TRACES_DIR`` pattern rather than driving the full + (heavily mocked) SDK agent path a second time in one test. + """ + + def test_two_real_attempts_sharing_run_id_with_different_tiers_is_rejected_mixed_tier( + self, tmp_path, monkeypatch + ): + from pathlib import Path + + from artemis.config.attempt_reconciliation import reconcile_attempt_batch_by_run_id + from artemis.data_engine.models import SessionMetadata, TraceRecord + from artemis.data_engine.storage import StorageManager + from artemis.runtime import trace_store + + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + run_id = "shared-retry-run-id" + sol_trace_id = "real-attempt-sol" + terra_trace_id = "real-attempt-terra" + + sol_llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + terra_llm_config = _uniform_llm_config("google", "gemini-3.8-flash") + + # The exact call run_task makes at its "launch" checkpoint, invoked + # twice with the same run_id -- this is the real plumbing added to + # thread run_id through, not a direct build_attempt_manifest() call. + _record_attempt_manifest( + trace_id=sol_trace_id, checkpoint="launch", llm_config=sol_llm_config, run_id=run_id + ) + _record_attempt_manifest( + trace_id=terra_trace_id, + checkpoint="launch", + llm_config=terra_llm_config, + run_id=run_id, + ) + + db_path = tmp_path / "usage.db" + usage_traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, usage_traces_dir) + for trace_id, source in ( + (sol_trace_id, "openai:gpt-5.6-sol"), + (terra_trace_id, "google:gemini-3.8-flash"), + ): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": source}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=usage_traces_dir, + traces_root=Path(trace_store.TRACES_DIR), + ) + + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + invalid_ids = {a.attempt_id for a in verdict.invalid_attempts} + assert invalid_ids == {sol_trace_id, terra_trace_id} + + +def test_cli_arg_parser_accepts_run_id_flag(): + """The real `__main__` CLI surface must accept --run-id without exiting 2. + + Reproduces the exact repro from the wiring-gap review: the CLI parser + used to have no --run-id flag at all, so passing it would fail argparse + with "unrecognized arguments" (exit code 2) before run_task was ever + called. Exercises _build_arg_parser() directly -- the same + ArgumentParser the `if __name__ == "__main__":` block constructs -- so + this proves the CLI surface itself, not just the in-process run_task + Python parameter. + """ + parser = _build_arg_parser() + + args = parser.parse_args( + [ + "--trace-id", + "trace-abc", + "--task-desc", + "Do the thing", + "--model", + "Flash", + "--run-id", + "shared-run-xyz", + ] + ) + + assert args.run_id == "shared-run-xyz" + + # Omitting --run-id must still parse cleanly and default to None, so an + # unmodified CLI invocation retains today's exact 1:1 run_id<->trace_id + # behavior in run_task. + args_no_run_id = parser.parse_args( + ["--trace-id", "trace-abc", "--task-desc", "Do the thing", "--model", "Flash"] + ) + assert args_no_run_id.run_id is None + + +class TestTerminationTimeBatchReconciliation: + """Proves _reconcile_and_store_verdict -- the function the finally: block + in run_task actually calls at termination -- genuinely fires batch + reconciliation and rejects a real mixed-tier batch, not merely that + reconcile_attempt_batch_by_run_id exists and can be called directly (that + is already covered by TestRealWorkerMultiAttemptRunIdGrouping and the + config-level TestReconcileAttemptBatchByRunId tests). + + This closes the second half of the wiring gap: even with --run-id + plumbed through the CLI, nothing previously called the batch reconciler + at termination -- the finally: block always called single-attempt + reconciliation only. + """ + + def test_termination_reconciliation_writes_mixed_tier_batch_verdict( + self, tmp_path, monkeypatch + ): + from pathlib import Path + + from artemis.config.constants import ENV_ARTEMIS_TRACES_DIR + from artemis.data_engine.models import SessionMetadata, TraceRecord + from artemis.data_engine.storage import StorageManager + from artemis.runtime import trace_store + + traces_dir = tmp_path / "traces" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(traces_dir)) + # get_data_engine_db_path() / get_traces_dir() (called internally by + # _reconcile_and_store_verdict, not injectable) must resolve to the + # same tmp_path tree the manifests and llm_usage receipts below are + # written under. + monkeypatch.setenv(ENV_ARTEMIS_TRACES_DIR, str(traces_dir)) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + run_id = "shared-termination-run-id" + sol_trace_id = "termination-attempt-sol" + terra_trace_id = "termination-attempt-terra" + + sol_llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + terra_llm_config = _uniform_llm_config("google", "gemini-3.8-flash") + + # First attempt: same real _record_attempt_manifest call run_task + # makes at its "launch" checkpoint. + _record_attempt_manifest( + trace_id=sol_trace_id, checkpoint="launch", llm_config=sol_llm_config, run_id=run_id + ) + _record_attempt_manifest( + trace_id=terra_trace_id, + checkpoint="launch", + llm_config=terra_llm_config, + run_id=run_id, + ) + + from artemis.config.paths import get_data_engine_db_path + + db_path = get_data_engine_db_path() + storage = StorageManager(db_path, traces_dir) + for trace_id, source in ( + (sol_trace_id, "openai:gpt-5.6-sol"), + (terra_trace_id, "google:gemini-3.8-flash"), + ): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": source}, + ) + ) + + # The exact call run_task's finally: block makes at termination for + # the second (terra) attempt, now with a real, shared run_id -- this + # is the wiring under test, not a direct + # reconcile_attempt_batch_by_run_id() call. + _reconcile_and_store_verdict(trace_id=terra_trace_id, checkpoint="launch", run_id=run_id) + + batch_verdict_path = ( + Path(trace_store.get_trace_dir(terra_trace_id)) + / "attempt_reconciliation_batch_verdict.json" + ) + assert batch_verdict_path.exists(), ( + "termination-time _reconcile_and_store_verdict did not write a batch verdict " + "file even though a real, non-default run_id was passed" + ) + + payload = json.loads(batch_verdict_path.read_text(encoding="utf-8")) + assert payload["accepted"] is False + assert payload["reason"] == "mixed_tier" + + def test_termination_reconciliation_skips_batch_when_run_id_matches_trace_id( + self, tmp_path, monkeypatch + ): + """Default single-attempt behavior (run_id omitted / equal to trace_id) + must not produce a batch verdict file at all -- proves the batch path + is genuinely additive and does not fire for today's unmodified callers. + """ + from pathlib import Path + + from artemis.config.constants import ENV_ARTEMIS_TRACES_DIR + from artemis.runtime import trace_store + + traces_dir = tmp_path / "traces" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(traces_dir)) + monkeypatch.setenv(ENV_ARTEMIS_TRACES_DIR, str(traces_dir)) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + trace_id = "solo-attempt" + llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + _record_attempt_manifest(trace_id=trace_id, checkpoint="launch", llm_config=llm_config) + + # No manifest for this trace_id sharing a run_id exists other than + # itself, and run_id defaults to trace_id inside + # _record_attempt_manifest -- mirrors an unmodified caller exactly. + _reconcile_and_store_verdict(trace_id=trace_id, checkpoint="launch", run_id=None) + + batch_verdict_path = ( + Path(trace_store.get_trace_dir(trace_id)) / "attempt_reconciliation_batch_verdict.json" + ) + assert not batch_verdict_path.exists() diff --git a/tests/unit/runtime/test_daemon_client.py b/tests/unit/runtime/test_daemon_client.py index 67a90d3b..6f6750f9 100644 --- a/tests/unit/runtime/test_daemon_client.py +++ b/tests/unit/runtime/test_daemon_client.py @@ -163,6 +163,26 @@ def test_submit_task_to_daemon_pro_tuning_defaults_to_null(): assert data["explorer_mode"] is None +def test_submit_task_to_daemon_forwards_run_id(): + """A daemon-dispatched attempt must carry its Gate 1 run_id through /api/run, + the same way it already carries session_id -- otherwise the worker the + Daemon spawns has no batch-grouping key to record its attempt manifest + under (see artemis.config.attempt_lifecycle_hooks).""" + mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}') + with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen: + submit_task_to_daemon("goal", profile="flash", session_id="sess-1", run_id="batch-42") + data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8")) + assert data["run_id"] == "batch-42" + + +def test_submit_task_to_daemon_run_id_defaults_to_null(): + mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}') + with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen: + submit_task_to_daemon("goal", profile="flash", session_id="sess-1") + data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8")) + assert data["run_id"] is None + + def test_submit_batch_to_daemon_forwards_pro_tuning_knobs(): from artemis.runtime.daemon_client import submit_batch_to_daemon From 94e56ce5845e95341dc83d7f37d7a65b6456e414 Mon Sep 17 00:00:00 2001 From: "congvc-bot-x99[bot]" <282619718+congvc-bot-x99[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:28:14 +0700 Subject: [PATCH 07/13] Merge pull request #5 from cheese-work/fix/che-639-anthropic-tier fix(artemis): add anthropic tier to Gate 1 TIER_MODELS (CHE-639) --- artemis/config/attempt_manifest.py | 8 ++-- artemis/config/attempt_reconciliation.py | 4 +- tests/unit/config/test_attempt_manifest.py | 47 ++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/artemis/config/attempt_manifest.py b/artemis/config/attempt_manifest.py index ad0357c4..65a732b0 100644 --- a/artemis/config/attempt_manifest.py +++ b/artemis/config/attempt_manifest.py @@ -59,9 +59,9 @@ #: Cheese Work squads. Every enabled model-bearing node in a manifest must #: resolve to exactly one of these tiers (mixed tiers => ``mixed_tier`` #: rejection, see ``attempt_reconciliation.validate_batch``). -Tier = Literal["luna", "terra", "sol"] +Tier = Literal["luna", "terra", "sol", "nova"] -TIERS: tuple[Tier, ...] = ("luna", "terra", "sol") +TIERS: tuple[Tier, ...] = ("luna", "terra", "sol", "nova") # Provider/model pairs that identify each tier. This fork's config/artemis.jsonc # does not (yet) declare an explicit tier->model table; its "default" node @@ -69,11 +69,13 @@ # only tier presently exercised end-to-end. "luna"/"terra" are declared for # forward compatibility with the escalation controller (Gate 2, out of scope # here) and must be updated here (not inferred) if/when config/artemis.jsonc -# grows an explicit tier table. +# grows an explicit tier table. "nova" is the Anthropic tier added for the +# CHE-491 qualification pilot against the sub2api gateway (CHE-639). TIER_MODELS: dict[Tier, tuple[str, str]] = { "luna": ("google", "gemini-3.5-flash-lite"), "terra": ("google", "gemini-3.8-flash"), "sol": ("openai", "gpt-5.6-sol"), + "nova": ("anthropic", "claude-sonnet-5"), } # Environment variables whose presence/value can influence LLMConfig diff --git a/artemis/config/attempt_reconciliation.py b/artemis/config/attempt_reconciliation.py index de601a33..9fd8e15d 100644 --- a/artemis/config/attempt_reconciliation.py +++ b/artemis/config/attempt_reconciliation.py @@ -356,12 +356,12 @@ def validate_batch(attempts: list[AttemptRecord]) -> BatchVerdict: # checkout" fallback sentinel is truthy but not a provable identity), # or that has any enabled node resolving to no declared tier at all # (untiered_enabled_nodes non-empty -- a model-bearing node that isn't - # pinned to Luna/Terra/Sol). + # pinned to a declared tier). missing_identity = [ a for a in attempts if not a.manifest.get("source_sha") - or a.manifest.get("tier") not in ("luna", "terra", "sol") + or a.manifest.get("tier") not in TIER_MODELS or a.manifest.get("fake_llm_enabled") is None or a.manifest.get("source_sha_provenance") != "git" or a.manifest.get("untiered_enabled_nodes") diff --git a/tests/unit/config/test_attempt_manifest.py b/tests/unit/config/test_attempt_manifest.py index 2229babc..b63e0f3b 100644 --- a/tests/unit/config/test_attempt_manifest.py +++ b/tests/unit/config/test_attempt_manifest.py @@ -28,7 +28,9 @@ import pytest from artemis.config.attempt_manifest import ( + TIER_MODELS, ManifestAlreadyExistsError, + _node_tier, build_attempt_manifest, canonical_bytes, digest_of, @@ -191,6 +193,51 @@ def test_mixed_tier_within_one_manifest_is_rejected_at_build_time(self): ) +class TestAnthropicTierMatching: + """CHE-639: TIER_MODELS declares an Anthropic tier ('nova' -> claude-sonnet-5).""" + + def test_nova_tier_declared_with_expected_provider_and_model(self): + assert TIER_MODELS["nova"] == ("anthropic", "claude-sonnet-5") + + def test_anthropic_claude_sonnet_5_node_matches_nova_tier(self): + node = _llm(provider="anthropic", model="claude-sonnet-5") + assert _node_tier(node) == "nova" + + def test_undeclared_anthropic_model_does_not_match_any_tier(self): + node = _llm(provider="anthropic", model="claude-sonnet-4-5") + assert _node_tier(node) is None + + def test_build_attempt_manifest_accepts_nova_tier_end_to_end(self): + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="nova", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + assert manifest["tier"] == "nova" + assert manifest["nodes"]["planner"]["resolved_tier"] == "nova" + assert manifest["untiered_enabled_nodes"] == [] + + def test_build_attempt_manifest_rejects_unknown_tier(self): + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + with pytest.raises(ValueError, match="Unknown tier"): + build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="mercury", # type: ignore[arg-type] + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + + class TestNullableNodesRepresentedExplicitly: def test_disabled_nodes_show_would_resolve_default_without_being_active(self): manifest = build_attempt_manifest( From 42711c953c5abb61f487d856252e0c9c97955300 Mon Sep 17 00:00:00 2001 From: "congvc-bot-x99[bot]" <282619718+congvc-bot-x99[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:29:10 +0700 Subject: [PATCH 08/13] Merge pull request #6 from cheese-work/agent/x99-claude-sonnet-5/10a885506fd4 fix(artemis): drop temperature for Anthropic models that reject it (CHE-640) --- artemis/llm/router.py | 28 +++++++++++++++-- tests/unit/test_llm_grounding.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 47e98f0e..e45c3edb 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -104,6 +104,25 @@ def from_string(cls, val: Any) -> "ModelProvider": ModelProvider.CUSTOM: "http://localhost:8000/v1", } +# Anthropic model name prefixes that hard-reject the `temperature` sampling +# parameter (400 invalid_request_error: "temperature is deprecated for this +# model"), starting with the generational (non-dated) naming scheme. Add new +# rejecting families here as Anthropic expands the deprecation. +_ANTHROPIC_TEMPERATURE_REJECTING_PREFIXES = ( + "claude-sonnet-5", + "claude-opus-5", + "claude-haiku-4-5", + "claude-fable-5", + "claude-opus-4-7", + "claude-opus-4-8", +) + + +def anthropic_rejects_temperature(model_name: Any) -> bool: + """Whether this Anthropic model 400s on any `temperature` value, including 0.0/1.0.""" + name = str(model_name).lower() + return name.startswith(_ANTHROPIC_TEMPERATURE_REJECTING_PREFIXES) + class ModelEndpoint(BaseModel): """Configuration definition for an LLM/VLM model endpoint.""" @@ -352,20 +371,23 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: or os.environ.get("ANTHROPIC_API_KEY") ) base_url = resolve_provider_base_url(endpoint) - kwargs = { + rejects_temperature = anthropic_rejects_temperature(endpoint.model_name) + kwargs: dict[str, Any] = { "model": endpoint.model_name, - "temperature": endpoint.temperature, "api_key": api_key, "base_url": base_url, "timeout": endpoint.timeout_seconds, } + if not rejects_temperature: + kwargs["temperature"] = endpoint.temperature budget = endpoint.thinking_budget if not budget and endpoint.reasoning_effort: effort_map = {"low": 2048, "medium": 8192, "high": 32768} budget = effort_map.get(endpoint.reasoning_effort.lower()) if budget: kwargs["thinking"] = {"type": "enabled", "budget_tokens": budget} - kwargs["temperature"] = 1.0 + if not rejects_temperature: + kwargs["temperature"] = 1.0 return ChatAnthropic(**{k: v for k, v in kwargs.items() if v is not None}) elif provider == ModelProvider.OPENROUTER: diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index 4b758aa2..0ce9a916 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -153,6 +153,59 @@ def test_model_factory_openai_endpoint_prefers_model_configuration(monkeypatch): assert chat_openai.call_args.kwargs["base_url"] == "https://openai-model.example/v1" +@pytest.mark.parametrize( + "model_name", + ["claude-sonnet-5", "claude-opus-5", "claude-haiku-4-5", "claude-fable-5-1"], +) +def test_model_factory_anthropic_omits_temperature_for_rejecting_model(model_name): + """Models that 400 on `temperature` never receive the parameter, even with thinking.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name=model_name, + api_key="sk-ant-test-key", + reasoning_effort="high", + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert "temperature" not in chat_anthropic.call_args.kwargs + assert chat_anthropic.call_args.kwargs["thinking"] == { + "type": "enabled", + "budget_tokens": 32768, + } + + +def test_model_factory_anthropic_keeps_temperature_for_accepting_model(): + """Models that still accept `temperature` continue to receive the configured value.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-3-7-sonnet-20250219", + api_key="sk-ant-test-key", + temperature=0.3, + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert chat_anthropic.call_args.kwargs["temperature"] == 0.3 + + +def test_model_factory_anthropic_forces_thinking_temperature_when_accepted(): + """Thinking budget still forces temperature=1.0 on models that accept the parameter.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-3-7-sonnet-20250219", + api_key="sk-ant-test-key", + reasoning_effort="high", + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert chat_anthropic.call_args.kwargs["temperature"] == 1.0 + + def test_model_factory_anthropic_uses_configured_or_model_endpoint(monkeypatch): """Anthropic receives the environment default unless the model specifies one.""" from artemis.config import settings From cb089857a15740d54d399c7596c089e48e9acf1f Mon Sep 17 00:00:00 2001 From: "congvc-bot-x99[bot]" <282619718+congvc-bot-x99[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:29:41 +0700 Subject: [PATCH 09/13] feat(artemis): publish pinned qualification inputs and fork maintenance contract (CHE-537) (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(qualification): pin qualification inputs and fork maintenance contract (CHE-537) Publishes the versioned manifest for the pocket-actual save/relaunch qualification journey (fork SHA, model tiers, entry point, fixture, and evidence schema references), its NL task/assertion journey spec including a deliberately false negative control, and the fork's pinned-SHA maintenance contract (weekly upstream review + manual pin promotion). Reuses the Gate 1 manifest/reconciliation mechanism and tier vocabulary from PR #1 (CHE-491) by reference/digest; does not port or rebuild it. No host install, provider calls, device operations, or qualification execution — authoring only, per CHE-537's approved scope. * fix(qualification): swap dead-on-arrival Sol tier for Luna, per CHE-388 plan CHE-388's plan says "compare Luna and Terra model tiers" but the manifest pinned Terra+Sol with no documented rationale. Sol resolves to an OpenAI-backed model, and the workspace announcement bars all OpenAI routing indefinitely -- pinning it as an active qualification target would freeze a batch that can never execute. Swap to Luna+Terra to match the plan and record the Sol exclusion (reason + unblock condition) instead of silently dropping it. Also refreshes review_contract to reflect the actual re-staffed reviewer (mbp-claude-sonnet-5) and the current CHE-540/541/542 gate state, both of which had drifted same-day per issue comments 01a0a83b/01a0a83f. Addresses blocking findings 1 (stale-base rebase, handled separately) and 2 from mbp-claude-sonnet-5's review (comment 01a0a844). --------- Co-authored-by: congvc-bot[bot] <3634010+congvc-bot[bot]@users.noreply.github.com> --- qualification/FORK_MAINTENANCE.md | 55 +++++++ qualification/evidence_schema.v1.json | 104 +++++++++++++ .../journeys/pocket_actual_save_relaunch.md | 88 +++++++++++ .../pocket_actual_save_relaunch.v1.json | 111 +++++++++++++ .../test_qualification_manifest_che537.py | 146 ++++++++++++++++++ 5 files changed, 504 insertions(+) create mode 100644 qualification/FORK_MAINTENANCE.md create mode 100644 qualification/evidence_schema.v1.json create mode 100644 qualification/journeys/pocket_actual_save_relaunch.md create mode 100644 qualification/manifests/pocket_actual_save_relaunch.v1.json create mode 100644 tests/unit/config/test_qualification_manifest_che537.py diff --git a/qualification/FORK_MAINTENANCE.md b/qualification/FORK_MAINTENANCE.md new file mode 100644 index 00000000..d9e4959b --- /dev/null +++ b/qualification/FORK_MAINTENANCE.md @@ -0,0 +1,55 @@ +# Cheese Work fork maintenance contract + +`cheese-work/artemis` tracks `google/artemis` (Apache-2.0, Pixel-Test-Engineering +"Fusion" team) under a **pinned-SHA policy**, not unpinned `main` tracking. +Upstream is early (repo created 2026-08-13, no tagged releases yet), so a +moving `main` is not a stable qualification target. + +## What is pinned, and where + +The single source of truth for the currently-qualified fork commit is +`qualification/manifests/pocket_actual_save_relaunch.v1.json`'s `fork_sha` +field (and any sibling manifest added for other qualification journeys). +CI, qualification runs, and rollout tooling read that field rather than +`origin/main` directly. + +## Weekly upstream review + +Once per week: + +1. Fetch upstream (`git fetch google main` with `google` remoting to + `https://github.com/google/artemis`, or the equivalent configured + remote). +2. Diff `google/main` against the currently pinned `fork_sha`. +3. Triage: security fixes and bug fixes affecting the qualified journey(s) + are candidates for promotion; unrelated feature work is not pulled in + opportunistically. +4. Record the review (date, commits considered, promote/skip decision) as + an issue comment on the fork-maintenance tracking issue (CHE-387) or a + dedicated weekly-review issue if volume warrants it — not silently in + git history alone. + +## Manual pin promotion + +Promoting the pinned SHA is manual and gated, never automatic: + +1. Open a PR that bumps `fork_sha` (and any file that embeds it) to the + new candidate commit. +2. The new SHA must pass the same independent-review chain as any other + qualification input change (see `manifests/pocket_actual_save_relaunch.v1.json`'s + `review_contract`): a changed `fork_sha` invalidates prior evidence per + the revision policy, so promotion always implies a fresh qualification + batch before the new pin is relied on for production rollout — not + before the PR merges. +3. Carry-forward Cheese Work patches (this fork's own diffs from upstream) + are rebase-checked against the candidate commit as part of the same PR; + a promotion that breaks a carried patch is not mergeable as-is. + +## Cheese Work patches vs upstream contributions + +- Cheese Work-specific changes (Gate 1 manifest/reconciliation mechanism, + qualification harness, this directory) live in the fork and are not + proposed upstream by default. +- If a fix is generally applicable (not Cheese-specific), it may be + proposed to `google/artemis` only through a Cheese-selected issue — + never opportunistically from an unrelated task. diff --git a/qualification/evidence_schema.v1.json b/qualification/evidence_schema.v1.json new file mode 100644 index 00000000..bf04c025 --- /dev/null +++ b/qualification/evidence_schema.v1.json @@ -0,0 +1,104 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "cheese-work/artemis/qualification/evidence_schema.v1.json", + "title": "Cheese Work Artemis qualification evidence record v1", + "description": "One record per qualification attempt (positive run or the negative control). Freeze N=10 positive runs per target/tier plus one negative control per CHE-388 plan v2; each attempt's evidence record must reference the exact qualification_manifest it ran against by digest.", + "type": "object", + "required": [ + "evidence_schema_version", + "qualification_manifest_digest", + "attempt_kind", + "target", + "tier", + "profile", + "host", + "device_serial", + "verdict", + "reconciliation", + "trace_id", + "recorded_at", + "cost", + "cleanup_verified" + ], + "properties": { + "evidence_schema_version": { "const": 1 }, + "qualification_manifest_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "SHA-256 digest of the pinned qualification manifest this attempt ran against (manifests/pocket_actual_save_relaunch.v1.json canonical_bytes). Never accept an attempt whose digest does not match the manifest under qualification." + }, + "attempt_kind": { + "type": "string", + "enum": ["positive", "negative_control"] + }, + "target": { + "type": "string", + "description": "Which model tier target this attempt qualifies, e.g. 'terra' or 'sol'." + }, + "tier": { + "type": "string", + "enum": ["luna", "terra", "sol"], + "description": "Gate 1 tier vocabulary (artemis/config/attempt_manifest.py TIER_MODELS, PR #1 / CHE-491)." + }, + "profile": { + "type": "string", + "enum": ["flash", "pro"] + }, + "host": { + "type": "string", + "enum": ["congvc-x99", "congvc-c00"] + }, + "device_serial": { "type": "string" }, + "per_attempt_manifest_digest": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{64}$", + "description": "Digest of the Gate 1 per-attempt LLM identity manifest (attempt_manifest.store_attempt_manifest output) for this specific attempt, once PR #1 lands and a real run produces one. Null while the mechanism referenced in PR #1 is not yet merged onto the base this attempt ran from." + }, + "verdict": { + "type": "string", + "enum": ["pass", "fail_assertion", "fail_infrastructure", "device_unavailable"], + "description": "Distinguish test failure, infrastructure failure, and unavailable-device coverage per CHE-388 acceptance criteria." + }, + "reconciliation": { + "type": "object", + "description": "Gate 1 batch accept/reject outcome for this attempt (artemis/config/attempt_reconciliation.py validate_batch, PR #1 / CHE-491).", + "required": ["accepted"], + "properties": { + "accepted": { "type": "boolean" }, + "reject_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot" + ] + } + } + } + }, + "trace_id": { "type": "string" }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "cost": { + "type": "object", + "required": ["usd", "step_count"], + "properties": { + "usd": { "type": "number", "minimum": 0 }, + "step_count": { "type": "integer", "minimum": 0 } + } + }, + "cleanup_verified": { + "type": "boolean", + "description": "True only if the fixture/reset step (see journeys/pocket_actual_save_relaunch.md) was confirmed to leave the candidate app and device in its pre-attempt state." + }, + "notes": { "type": "string" } + } +} diff --git a/qualification/journeys/pocket_actual_save_relaunch.md b/qualification/journeys/pocket_actual_save_relaunch.md new file mode 100644 index 00000000..34061c6d --- /dev/null +++ b/qualification/journeys/pocket_actual_save_relaunch.md @@ -0,0 +1,88 @@ +# Qualification journey: pocket-actual save/relaunch + +CHE-537 source packet. This is the smallest retrievable journey for CHE-388 +qualification: persist data via normal UI interaction, kill and relaunch the +app, and assert the persisted state survived — plus one deliberately false +assertion to prove the pipeline can fail for the right reason. + +This file is instructions and assertions only. It does not run anything, is +not evidence of a pilot pass, and does not implement Gate 1's manifest or +reconciliation mechanism (that mechanism is PR #1 / CHE-491, referenced by +digest in `manifests/pocket_actual_save_relaunch.v1.json`, and reused, not +rebuilt, here). + +## Fixture / reset + +Before every attempt (positive or negative control): + +1. Force-stop the candidate app: `adb -s shell am force-stop `. +2. Clear app data: `adb -s shell pm clear `. +3. Confirm cold-start state: relaunch once, verify the entry screen shows no + prior data (empty/default state). This is a scripted ADB check, not an + Artemis-driven step, and runs outside any budgeted attempt. + +`` and `` are supplied at run time by the +execution stage (CHE-540+); this journey does not pin them. + +## Positive run — natural language task instructions + +Goal text passed as `task_desc` (MCP `mobile_run_task`) / `goal` (CLI `artemis run`): + +> Open the app. Create a new entry with the label "qual-" and the +> value "" (substitute the actual run id and a freshly +> generated random token before dispatch — never reuse a token across +> attempts, so a stale relaunch can never be mistaken for a fresh save). +> Save it. Force the app to the background, then fully close it. Relaunch +> the app from the home screen. Open the same entry and read back its +> value. + +`expected_output_desc` / `output_description`: + +> Report the exact value read back for the entry labeled "qual-" +> after relaunch, and whether it matches "" exactly. + +### Assertions (positive run) + +1. **Save succeeded**: the app's own UI confirms the entry was created + (e.g. a visible list item, a save confirmation) before the + background/close step begins. +2. **Relaunch reached the same screen**: after relaunch, the agent + navigates back to the entry (via whatever path the app's UI requires) — + this is graded as part of the run, not scripted, since the navigation + path is exactly what a real user would need and is part of what + qualification is proving. +3. **Persisted value matches**: the value read back after relaunch equals + "" byte-for-byte. Anything else (empty, default, a + different prior value, an app crash) is `fail_assertion`. +4. **Verdict discipline**: an attempt that never reaches a relaunched, + readable state (app crash, ANR, device disconnect, ADB failure) is + `fail_infrastructure` or `device_unavailable`, never silently folded + into `fail_assertion` — see `evidence_schema.v1.json`'s `verdict` enum. + +## Negative control — deliberately false assertion + +Same fixture, same save step, but the checker is told to verify a value +that provenance guarantees is wrong: + +> ...(identical save/relaunch steps as the positive run)... Open the same +> entry and confirm its value reads "-WRONG-SUFFIX" exactly. + +This must produce `fail_assertion` (Checker or Outputter catches the +mismatch and reports failure) on every run. A negative control that comes +back `pass` invalidates the entire batch it belongs to — the pipeline is +not distinguishing true from false, so no positive result in that batch +can be trusted either. Record it as its own evidence entry +(`attempt_kind: "negative_control"`), not folded into the N=10 positive +count (CHE-388 plan v2: "freeze N=10 positive runs per target/tier plus +one separate negative control"). + +## Bounded execution and cleanup + +- One attempt = one `mobile_run_task` / `artemis run` invocation from a + freshly reset fixture to either a verdict or a hard timeout. +- After every attempt (pass, fail, or infrastructure failure), re-run the + fixture/reset steps and confirm cold-start state before releasing the + device lease — this is `cleanup_verified` in the evidence schema. +- No attempt escalates tier, retries silently, or mutates the manifest + mid-batch. A revised candidate or manifest requires a fresh batch (see + `manifests/pocket_actual_save_relaunch.v1.json`'s `revision_policy`). diff --git a/qualification/manifests/pocket_actual_save_relaunch.v1.json b/qualification/manifests/pocket_actual_save_relaunch.v1.json new file mode 100644 index 00000000..e4b1946a --- /dev/null +++ b/qualification/manifests/pocket_actual_save_relaunch.v1.json @@ -0,0 +1,111 @@ +{ + "manifest_version": 1, + "$schema_ref": "qualification/evidence_schema.v1.json", + "issue": "CHE-537", + "coordinating_parent": "CHE-388", + "journey": "qualification/journeys/pocket_actual_save_relaunch.md", + + "fork_sha": { + "value": "64e1b3226b9553bdf4e5a368f14b8a40c0af6111", + "source": "git", + "repo": "https://github.com/cheese-work/artemis", + "note": "Source baseline observed and pinned per CHE-537 delivery plan. Weekly upstream review and manual pin promotion after qualification -- see FORK_MAINTENANCE.md. This value must be updated (new revision) if the pinned SHA changes; execution stages must re-read it at admission rather than caching an older value." + }, + + "gate1_manifest_mechanism": { + "source": "https://github.com/cheese-work/artemis/pull/1", + "head_sha": "2b58709ed00b0015055a99f9e644857ba9d05ee0", + "status": "OPEN (draft, not merged) as observed at CHE-537 authoring time", + "note": "Reused by reference, not rebuilt or ported here. Per-attempt LLM identity manifests (artemis/config/attempt_manifest.py) and reconciliation (artemis/config/attempt_reconciliation.py) are produced at real-run time by this mechanism once it is merged onto the pinned fork_sha (or its own base is advanced past fork_sha, whichever the executing stage's admitted revision resolves to). This qualification manifest does not claim Gate 1 has passed." + }, + + "candidate_app": { + "app_sha": null, + "apk_digest_sha256": null, + "package_name": null, + "note": "Not pinned by this issue. CHE-537 is scoped to authoring the qualification input contract only -- no device operations, no APK install, no provider calls. The execution stage (CHE-540+) fills these fields for the exact candidate build under test and must not proceed without them; a null value here is a hold, not a default." + }, + + "dependency_lock": { + "file": "uv.lock", + "digest_sha256": null, + "note": "Populate via `sha256sum uv.lock` at the exact fork_sha checkout used for the qualification run. Left null here since this issue performs no install." + }, + + "accessibility_helper": { + "name": "Artemis Accessibility Helper (UIAutomator2 fallback)", + "digest_sha256": null, + "note": "Per-host on-device helper referenced in CHE-388's install surface. Digest recorded by the host install stage, not authored here." + }, + + "model_targets": [ + { + "tier": "luna", + "provider": "google", + "model": "gemini-3.5-flash-lite", + "provider_id": "google:gemini-3.5-flash-lite", + "role": "cheap tier under CHE-388's 'compare Luna and Terra model tiers' requirement" + }, + { + "tier": "terra", + "provider": "google", + "model": "gemini-3.8-flash", + "provider_id": "google:gemini-3.8-flash", + "role": "mid tier under CHE-388's 'compare Luna and Terra model tiers' requirement" + } + ], + "tier_vocabulary_source": "artemis/config/attempt_manifest.py TIER_MODELS (PR #1 / CHE-491) -- do not diverge from that mapping; if it changes, this manifest must be revised to match rather than declaring a third mapping.", + "sol_tier_excluded": { + "tier": "sol", + "provider": "openai", + "model": "gpt-5.6-sol", + "reason": "Excluded from model_targets, not merely deprioritized. The 'sol' Gate 1 tier resolves to an OpenAI-backed model, and the workspace announcement is 'Stop all routing to OpenAI based agents. No set end date' (reaffirmed same-day on this issue: 'no openai based agent runs. they're all dead'). A batch of N=10 positive runs + 1 negative control against this target cannot execute under current policy, so pinning it as an active qualification target would freeze a dead-on-arrival gate.", + "unblock_condition": "Add 'sol' back to model_targets only after the OpenAI routing-stop announcement is explicitly lifted or replaced by a routable provider/model pair under the same 'sol' tier label. Until then, config/artemis.jsonc's top-level default node (openai/gpt-5.6-sol) stays unqualified by this manifest even though it is the fork's real configured default -- this is a known, deliberate gap, not an oversight." + }, + + "profile": "pro", + "profile_note": "Pro selected over Flash for this journey: the assertion step (read back a persisted value against an expected/negative token) needs the Checker's verified-checkpoint path (CHE-388 install-surface note: Flash has no final report and no verification). verification_level left to the execution stage's per-run config, not pinned here.", + + "entry_point": { + "route": "mcp", + "tool": "mobile_run_task", + "fallback_route": "cli", + "fallback_command": "artemis run", + "note": "CHE-537 plan: 'Prefer the existing MCP route with foreground terminal collection; select native CLI/SDK only where exact installed behavior proves necessary.' mobile_run_task params referenced: task_desc, model (tier profile), locked_app_package, app_path, expected_output_desc, verification_level." + }, + + "fixture": { + "journey_ref": "qualification/journeys/pocket_actual_save_relaunch.md", + "reset_procedure": "adb shell am force-stop && adb shell pm clear , cold-start verified before every attempt", + "positive_runs_required": 10, + "negative_controls_required": 1, + "note": "N=10 positive runs per target/tier plus one separate negative control per CHE-388 plan v2. Counts are per (tier, profile) combination in model_targets, not a single global N=10." + }, + + "evidence_schema": { + "file": "qualification/evidence_schema.v1.json", + "version": 1 + }, + + "revision_policy": { + "rule": "A changed candidate revision (app_sha, apk_digest_sha256) or a changed fork_sha forces a new manifest revision and invalidates all evidence collected against the prior revision. Evidence is not transferable across revisions.", + "source": "CHE-388 review contract invariant: 'a changed candidate revision forces fresh step-2 review AND invalidates prior evidence (new run required)'." + }, + + "review_contract": { + "chain_as_originally_settled": "Terra authors -> one candidate reviewer approves -> Luna executes -> Opus AND Sol independently review the run evidence -> Terra verifies", + "chain_status": "Superseded 2026-09-16 (issue comments 01a0a83b, 01a0a83f): Terra/Sol/Luna/Astra are permanently unroutable (OpenAI routing stopped workspace-wide, no set end date). Chain is re-staffed Claude-only; independence is now host-scoped (different agent instance on a different runtime), not model-scoped -- a recorded, accepted reduction in gate strength, not equivalent to the original design.", + "candidate_reviewer_this_issue": "mbp-claude-sonnet-5 (read-only review, comment 01a0a844; did not author this candidate, independence holds on host-scoped grounds). x99-codex-5.6-sol was named at authoring time but is permanently unroutable and never actually reviewed this candidate.", + "evidence_reviewers_current": "CHE-540/541/542 evidence review: reviewer A x99-claude-opus-5, reviewer B mbp-claude-opus-5 (different runtime). The human-decision gate on these stages was lifted 2026-09-16; they now carry only technical/dependency gates (CHE-537 published inputs, CHE-538/539 pinned runtime install).", + "no_self_review": true + }, + + "out_of_scope_for_this_manifest_authoring": [ + "host install", + "provider calls", + "device operations", + "binding changes", + "agent merge", + "claiming Gate 1 or any qualification gate has passed" + ] +} diff --git a/tests/unit/config/test_qualification_manifest_che537.py b/tests/unit/config/test_qualification_manifest_che537.py new file mode 100644 index 00000000..5a092e4e --- /dev/null +++ b/tests/unit/config/test_qualification_manifest_che537.py @@ -0,0 +1,146 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Structural checks for the CHE-537 pinned qualification manifest. + +These are plain dict/field assertions, not JSON Schema validation (no +`jsonschema` dependency introduced for one manifest). The manifest and its +schema doc are authored, not executed, by this issue -- these tests only +guard against the manifest silently drifting out of the shape the +qualification execution stage (CHE-540+) and the Gate 1 tier vocabulary +(PR #1 / CHE-491) depend on. +""" + +import json +from pathlib import Path +import re + +REPO_ROOT = Path(__file__).resolve().parents[3] +MANIFEST_PATH = REPO_ROOT / "qualification/manifests/pocket_actual_save_relaunch.v1.json" +JOURNEY_PATH = REPO_ROOT / "qualification/journeys/pocket_actual_save_relaunch.md" +EVIDENCE_SCHEMA_PATH = REPO_ROOT / "qualification/evidence_schema.v1.json" + +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + +# Must stay in lockstep with artemis/config/attempt_manifest.py's TIER_MODELS +# (PR #1 / CHE-491). This test intentionally hardcodes the same tuple rather +# than importing that (unmerged, draft) module, so it fails loudly if this +# manifest's tiers/providers/models drift from what Gate 1 actually declares. +_KNOWN_TIER_MODELS = { + "luna": ("google", "gemini-3.5-flash-lite"), + "terra": ("google", "gemini-3.8-flash"), + "sol": ("openai", "gpt-5.6-sol"), +} + + +def _load_manifest() -> dict: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def test_manifest_and_companion_files_exist(): + assert MANIFEST_PATH.is_file() + assert JOURNEY_PATH.is_file() + assert EVIDENCE_SCHEMA_PATH.is_file() + + +def test_fork_sha_is_a_valid_git_sha_and_matches_source_baseline(): + manifest = _load_manifest() + fork_sha = manifest["fork_sha"]["value"] + assert _SHA_RE.match(fork_sha), f"fork_sha must be a 40-hex-char git SHA, got {fork_sha!r}" + assert fork_sha == "64e1b3226b9553bdf4e5a368f14b8a40c0af6111", ( + "fork_sha must match the CHE-537 observed source baseline; if the pin " + "legitimately moved, this test and FORK_MAINTENANCE.md's promotion " + "record must be updated together, not silently." + ) + + +def test_candidate_app_fields_are_a_hold_not_a_default(): + manifest = _load_manifest() + candidate = manifest["candidate_app"] + assert candidate["app_sha"] is None + assert candidate["apk_digest_sha256"] is None + assert candidate["package_name"] is None + + +def test_model_targets_match_gate1_tier_vocabulary(): + manifest = _load_manifest() + targets = manifest["model_targets"] + assert len(targets) >= 2, "CHE-388 requires baselining at least two model tiers" + for target in targets: + tier = target["tier"] + assert tier in _KNOWN_TIER_MODELS, f"unknown tier {tier!r} not in Gate 1 TIER_MODELS" + expected_provider, expected_model = _KNOWN_TIER_MODELS[tier] + assert target["provider"] == expected_provider + assert target["model"] == expected_model + assert target["provider_id"] == f"{expected_provider}:{expected_model}" + + +def test_model_targets_are_luna_and_terra_per_che388_plan_not_sol(): + manifest = _load_manifest() + tiers = {t["tier"] for t in manifest["model_targets"]} + assert tiers == {"luna", "terra"}, ( + "CHE-388's plan text says 'compare Luna and Terra model tiers'; " + "'sol' resolves to an OpenAI-backed model and the workspace announcement " + "bars all OpenAI routing indefinitely, so it must be excluded here, not pinned." + ) + excluded = manifest["sol_tier_excluded"] + assert excluded["tier"] == "sol" + assert "unblock_condition" in excluded + + +def test_fixture_counts_match_che388_plan_v2(): + manifest = _load_manifest() + fixture = manifest["fixture"] + assert fixture["positive_runs_required"] == 10 + assert fixture["negative_controls_required"] == 1 + + +def test_revision_policy_and_review_contract_present(): + manifest = _load_manifest() + assert "rule" in manifest["revision_policy"] + review = manifest["review_contract"] + assert review["no_self_review"] is True + assert "Opus AND Sol" in review["chain_as_originally_settled"] + assert "mbp-claude-sonnet-5" in review["candidate_reviewer_this_issue"] + + +def test_evidence_schema_declares_verdict_enum_covering_infra_and_device_failures(): + schema = json.loads(EVIDENCE_SCHEMA_PATH.read_text(encoding="utf-8")) + verdict_enum = set(schema["properties"]["verdict"]["enum"]) + assert verdict_enum == {"pass", "fail_assertion", "fail_infrastructure", "device_unavailable"} + + +def test_evidence_schema_reject_reasons_match_gate1_reconciliation_vocabulary(): + schema = json.loads(EVIDENCE_SCHEMA_PATH.read_text(encoding="utf-8")) + reject_reasons = set( + schema["properties"]["reconciliation"]["properties"]["reject_reasons"]["items"]["enum"] + ) + # Mirrors artemis/config/attempt_reconciliation.py's REJECT_REASONS (PR #1 / CHE-491). + assert reject_reasons == { + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", + } + + +def test_journey_defines_fixture_reset_and_negative_control(): + text = JOURNEY_PATH.read_text(encoding="utf-8") + assert "pm clear" in text + assert "force-stop" in text + assert "Negative control" in text + assert "WRONG-SUFFIX" in text From b87d5495d7fe2afa44a6c1884c646e3fe83fe1ce Mon Sep 17 00:00:00 2001 From: Cong Vu Chi <129714106+congvc-dev@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:42:07 +0700 Subject: [PATCH 10/13] feat(artemis): pin candidate app for pocket-actual save/relaunch (CHE-540) (#12) Manifest revision v2 per CHE-540 dispatch: fills candidate_app.app_sha, apk_digest_sha256, package_name and dependency_lock.digest_sha256, held null by CHE-537 pending the execution stage. Candidate is cheese-work/pocket-actual @ 2c42481778e6f9b88a9502f55decdbe4e1a76e83 (main HEAD), built via ./gradlew :composeApp:assembleDebug on X99, no device operations performed. Updates the CHE-537-era test asserting these fields as a null hold to assert the pinned values instead. Co-authored-by: congvc-bot[bot] <3634010+congvc-bot[bot]@users.noreply.github.com> --- .../pocket_actual_save_relaunch.v1.json | 12 +++---- .../test_qualification_manifest_che537.py | 32 ++++++++++++++++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/qualification/manifests/pocket_actual_save_relaunch.v1.json b/qualification/manifests/pocket_actual_save_relaunch.v1.json index e4b1946a..4410f42e 100644 --- a/qualification/manifests/pocket_actual_save_relaunch.v1.json +++ b/qualification/manifests/pocket_actual_save_relaunch.v1.json @@ -20,16 +20,16 @@ }, "candidate_app": { - "app_sha": null, - "apk_digest_sha256": null, - "package_name": null, - "note": "Not pinned by this issue. CHE-537 is scoped to authoring the qualification input contract only -- no device operations, no APK install, no provider calls. The execution stage (CHE-540+) fills these fields for the exact candidate build under test and must not proceed without them; a null value here is a hold, not a default." + "app_sha": "2c42481778e6f9b88a9502f55decdbe4e1a76e83", + "apk_digest_sha256": "04795d5f3995c8e895f6440a9763c8a003c8b6ff955541e3965f5e71564e2e12", + "package_name": "dev.cheese.pocketactual", + "note": "Pinned by CHE-540 (manifest revision v2). Source: https://github.com/cheese-work/pocket-actual at its main HEAD `2c42481778e6f9b88a9502f55decdbe4e1a76e83`, observed and built at authoring time (no device install performed). apk_digest_sha256 is `sha256sum` of `composeApp/build/outputs/apk/debug/composeApp-debug.apk` produced by `./gradlew :composeApp:assembleDebug` at that exact commit on X99. Historical Maestro-era evidence exists on pocket-actual PR #5 (app_sha `29ddf249c573ec6e07effa0c52708b7007d12ea9`) but is superseded, not reused, per revision_policy -- that build predates the Maestro-to-Artemis cutover." }, "dependency_lock": { "file": "uv.lock", - "digest_sha256": null, - "note": "Populate via `sha256sum uv.lock` at the exact fork_sha checkout used for the qualification run. Left null here since this issue performs no install." + "digest_sha256": "2c63863dd5827482d7e206a887219c6438ce61dc494d9cc596f862378decc19a", + "note": "sha256sum of this fork's own uv.lock (the Artemis qualification harness's dependency lock, not the candidate app's) at the pinned fork_sha `64e1b3226b9553bdf4e5a368f14b8a40c0af6111` checkout -- confirmed unchanged between fork_sha and this manifest revision's HEAD via `git diff fork_sha HEAD -- uv.lock` (empty)." }, "accessibility_helper": { diff --git a/tests/unit/config/test_qualification_manifest_che537.py b/tests/unit/config/test_qualification_manifest_che537.py index 5a092e4e..1ddffa10 100644 --- a/tests/unit/config/test_qualification_manifest_che537.py +++ b/tests/unit/config/test_qualification_manifest_che537.py @@ -32,6 +32,7 @@ EVIDENCE_SCHEMA_PATH = REPO_ROOT / "qualification/evidence_schema.v1.json" _SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") # Must stay in lockstep with artemis/config/attempt_manifest.py's TIER_MODELS # (PR #1 / CHE-491). This test intentionally hardcodes the same tuple rather @@ -65,12 +66,35 @@ def test_fork_sha_is_a_valid_git_sha_and_matches_source_baseline(): ) -def test_candidate_app_fields_are_a_hold_not_a_default(): +def test_candidate_app_fields_are_pinned_by_che540(): + """CHE-537 left these fields null as a hold; CHE-540 (manifest revision v2) + pins the exact pocket-actual build under test. If the pin legitimately + moves, this test and the manifest's candidate_app.note must be updated + together (revision_policy: a changed app_sha/apk_digest_sha256 forces a + new manifest revision and invalidates prior evidence). + """ manifest = _load_manifest() candidate = manifest["candidate_app"] - assert candidate["app_sha"] is None - assert candidate["apk_digest_sha256"] is None - assert candidate["package_name"] is None + assert _SHA_RE.match(candidate["app_sha"]), ( + f"app_sha must be a 40-hex-char git SHA, got {candidate['app_sha']!r}" + ) + assert candidate["app_sha"] == "2c42481778e6f9b88a9502f55decdbe4e1a76e83" + assert _SHA256_RE.match(candidate["apk_digest_sha256"]), ( + f"apk_digest_sha256 must be a 64-hex-char sha256 digest, got {candidate['apk_digest_sha256']!r}" + ) + assert candidate["apk_digest_sha256"] == ( + "04795d5f3995c8e895f6440a9763c8a003c8b6ff955541e3965f5e71564e2e12" + ) + assert candidate["package_name"] == "dev.cheese.pocketactual" + + +def test_dependency_lock_digest_is_pinned_by_che540(): + manifest = _load_manifest() + digest = manifest["dependency_lock"]["digest_sha256"] + assert _SHA256_RE.match(digest), ( + f"digest_sha256 must be a 64-hex-char sha256 digest, got {digest!r}" + ) + assert digest == "2c63863dd5827482d7e206a887219c6438ce61dc494d9cc596f862378decc19a" def test_model_targets_match_gate1_tier_vocabulary(): From 01d51c86c779de6e70873369efa14dce45aeb85e Mon Sep 17 00:00:00 2001 From: "congvc-bot-x99[bot]" <282619718+congvc-bot-x99[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:45:35 +0700 Subject: [PATCH 11/13] fix(llm): work around langchain-anthropic model_dump crash on streamed context_management (CHE-643) (#11) ChatAnthropic._make_message_chunk_from_anthropic_event reads event.context_management off message_delta stream events and calls .model_dump() on it unconditionally. That name is only declared on ChatAnthropic itself (a plain request-side dict); Anthropic's RawMessageDeltaEvent has no typed field for it, so when the API includes that block in a response it arrives as a raw dict via Pydantic's extra="allow" and .model_dump() raises AttributeError, discarding the whole in-flight response. Confirmed present on langchain-anthropic's latest release, so a version bump would not fix it. Patches _make_message_chunk_from_anthropic_event at Anthropic model construction time to normalize any raw-dict context_management/container into something model_dump()-safe before delegating to the original method. Co-authored-by: congvc-bot[bot] <3634010+congvc-bot[bot]@users.noreply.github.com> --- artemis/llm/router.py | 47 ++++++ .../unit/test_llm_router_anthropic_stream.py | 135 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 tests/unit/test_llm_router_anthropic_stream.py diff --git a/artemis/llm/router.py b/artemis/llm/router.py index e45c3edb..24efbfff 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -22,6 +22,7 @@ from enum import StrEnum import hashlib import os +from types import SimpleNamespace from typing import Any from langchain_core.language_models.chat_models import BaseChatModel @@ -124,6 +125,50 @@ def anthropic_rejects_temperature(model_name: Any) -> bool: return name.startswith(_ANTHROPIC_TEMPERATURE_REJECTING_PREFIXES) +_anthropic_stream_event_patched = False + + +def _patch_anthropic_stream_context_management() -> None: + """Works around a langchain-anthropic bug that crashes every stream. + + ``ChatAnthropic._make_message_chunk_from_anthropic_event`` reads + ``event.context_management`` off `message_delta` stream events and calls + ``.model_dump()`` on it unconditionally. That attribute is declared on + `ChatAnthropic` itself as a plain ``dict`` (its own request-side memory/ + context-management config), not as a typed field on the Anthropic SDK's + `RawMessageDeltaEvent`. Because that event's Pydantic model allows extra + fields, an API response carrying a top-level `context_management` block + (a newer, non-beta feature) arrives as a raw `dict`, not a submodel, and + `.model_dump()` raises `AttributeError: 'dict' object has no attribute + 'model_dump'` mid-stream, discarding the whole response. Confirmed still + present on langchain-anthropic's latest release; upgrading does not fix + it. Idempotent — safe to call on every Anthropic model construction. + """ + global _anthropic_stream_event_patched + if _anthropic_stream_event_patched: + return + + from langchain_anthropic import ChatAnthropic + + original = ChatAnthropic._make_message_chunk_from_anthropic_event + + def _dumpable(value: Any) -> Any: + if isinstance(value, dict): + return SimpleNamespace(model_dump=lambda mode=None: value) + return value + + def _patched(self, event, **kwargs): + if getattr(event, "context_management", None) is not None: + event.context_management = _dumpable(event.context_management) + delta = getattr(event, "delta", None) + if delta is not None and getattr(delta, "container", None) is not None: + delta.container = _dumpable(delta.container) + return original(self, event, **kwargs) + + ChatAnthropic._make_message_chunk_from_anthropic_event = _patched + _anthropic_stream_event_patched = True + + class ModelEndpoint(BaseModel): """Configuration definition for an LLM/VLM model endpoint.""" @@ -361,6 +406,8 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: elif provider == ModelProvider.ANTHROPIC: from langchain_anthropic import ChatAnthropic + _patch_anthropic_stream_context_management() + api_key = ( endpoint.api_key or ( diff --git a/tests/unit/test_llm_router_anthropic_stream.py b/tests/unit/test_llm_router_anthropic_stream.py new file mode 100644 index 00000000..c94b604c --- /dev/null +++ b/tests/unit/test_llm_router_anthropic_stream.py @@ -0,0 +1,135 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression test for the Anthropic streaming `model_dump` crash (CHE-643). + +``langchain_anthropic``'s ``ChatAnthropic._make_message_chunk_from_anthropic_event`` +reads ``event.context_management`` off `message_delta` stream events and calls +``.model_dump()`` on it unconditionally. The Anthropic SDK's `RawMessageDeltaEvent` +has no typed field for `context_management` (that name is only declared on +`ChatAnthropic` itself, as a plain request-side dict), so when the API includes +that block in a response, Pydantic's `extra="allow"` stores it as a raw `dict` +instead of a submodel — and `.model_dump()` raises +`AttributeError: 'dict' object has no attribute 'model_dump'`, discarding the +whole in-flight response (see `artemis/services/llm.py::_complete_attempt`). +""" + +import inspect +from unittest.mock import patch + +from anthropic.types.raw_message_delta_event import Delta, RawMessageDeltaEvent +from anthropic.types.usage import Usage +import pytest + +from artemis.llm.router import _patch_anthropic_stream_context_management + + +def _unpatched_make_message_chunk(): + """The original langchain-anthropic method, regardless of test order. + + Other tests/modules construct real Anthropic models (e.g. + ``test_llm_grounding.py``), which applies this module's process-global + patch as a side effect. Walk the `__wrapped__`-style closure our patch + builds to recover the pristine method so this test proves the upstream + bug independent of suite ordering. + """ + from langchain_anthropic import ChatAnthropic + + current = ChatAnthropic._make_message_chunk_from_anthropic_event + closure = inspect.getclosurevars(current).nonlocals + return closure.get("original", current) + + +def _delta_event_with_raw_context_management() -> RawMessageDeltaEvent: + """Builds the exact malformed event shape the live Anthropic API sent.""" + delta = Delta.model_construct( + stop_reason="end_turn", + stop_sequence=None, + container=None, + ) + return RawMessageDeltaEvent.model_construct( + type="message_delta", + delta=delta, + usage=Usage(input_tokens=1, output_tokens=1), + context_management={"applied_edits": []}, + ) + + +def test_raw_dict_context_management_crashes_without_the_patch(): + """Reproduces the upstream bug so the workaround has something to guard against.""" + from langchain_anthropic import ChatAnthropic + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + event = _delta_event_with_raw_context_management() + unpatched = _unpatched_make_message_chunk() + + with patch.object(ChatAnthropic, "_make_message_chunk_from_anthropic_event", unpatched): + with pytest.raises(AttributeError, match="model_dump"): + model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + +def test_patch_normalizes_raw_dict_context_management(): + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + event = _delta_event_with_raw_context_management() + + chunk, _ = model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + assert chunk is not None + assert chunk.response_metadata["context_management"] == {"applied_edits": []} + + +def test_patch_normalizes_raw_dict_container(): + """The same unconditional `.model_dump()` pattern applies to `delta.container`.""" + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + delta = Delta.model_construct(stop_reason="end_turn", stop_sequence=None) + delta.container = {"id": "container_123"} + event = RawMessageDeltaEvent.model_construct( + type="message_delta", + delta=delta, + usage=Usage(input_tokens=1, output_tokens=1), + ) + + chunk, _ = model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + assert chunk is not None + assert chunk.response_metadata["container"] == {"id": "container_123"} + + +def test_patch_is_idempotent_across_multiple_model_constructions(): + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + patched_once = ChatAnthropic._make_message_chunk_from_anthropic_event + + # Re-applying (as every Anthropic model construction does) must not wrap + # the wrapper again — otherwise each new stream event pays for one more + # layer of indirection for the lifetime of the process. + _patch_anthropic_stream_context_management() + _patch_anthropic_stream_context_management() + + assert ChatAnthropic._make_message_chunk_from_anthropic_event is patched_once From 144006b1b2e6888e31dc8d76887a7da5e8839e99 Mon Sep 17 00:00:00 2001 From: Cong Vu Chi <129714106+congvc-dev@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:57:05 +0700 Subject: [PATCH 12/13] fix(object_detector): resolve the detector model from utils config instead of silently using the operator model (CHE-646) (#13) * fix(object_detector): resolve detector LLM from utils config instead of silently using the operator model (CHE-646) object_detector.py called get_llm(ctx, name="object_detector") without is_utils=True, so it always raised AttributeError (object_detector is a field of LLMConfigUtils, not LLMConfig) and a bare except Exception silently fell back to the operator model on every call, even when a detector was properly configured. Pass is_utils=True to resolve the correct field, and narrow the except clause to (ValueError, AttributeError) with a logged warning so an unconfigured detector still falls back but is no longer silent. precondition_pixel.py had a structurally identical but functionally dead fallback in _init_llm_and_prompt: "validator" is not a real LLMConfig field, so the fallback branch could only ever raise itself. Removed the unreachable branch and documented why validator_pixel_safety_net's own get_agent default makes it unnecessary. Added unit tests covering: configured detector resolves via utils (not operator), unconfigured detector logs a warning and falls back to operator, and the pre-fix is_utils=False path still raises AttributeError (regression guard). * fix(image_processor): stop hiding the operator model behind an impossible config lookup (CHE-646) The audit for CHE-646 found the same shape at ImageProcessor.run: get_llm(name="image_processor") wrapped in a bare except that resolved the operator model. There is no image_processor node on LLMConfig or LLMConfigUtils, so the lookup raised AttributeError on every call and the agent has only ever run on the operator model. Behaviour is unchanged: call the operator node directly and document why. Giving the image processor its own configurable node is a config-schema change and is tracked separately. --------- Co-authored-by: congvc-bot[bot] <3634010+congvc-bot[bot]@users.noreply.github.com> --- .../agents/image_processor/image_processor.py | 13 +- .../agents/object_detector/object_detector.py | 9 +- .../agents/validator/precondition_pixel.py | 15 +- .../test_object_detector_llm_resolution.py | 148 ++++++++++++++++++ 4 files changed, 174 insertions(+), 11 deletions(-) create mode 100644 tests/unit/agents/test_object_detector_llm_resolution.py diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 05e37abb..c2538576 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -79,10 +79,15 @@ def __init__(self, ctx: ArtemisContext): @trace(type="agent", name="image_processor") async def run(self, instruction: str, target_image_path: str) -> dict: - try: - llm = get_llm(self.ctx, name="image_processor") - except Exception: - llm = get_llm(self.ctx, name="operator") + # There is no ``image_processor`` node on ``LLMConfig`` or + # ``LLMConfigUtils``, so the former ``get_llm(name="image_processor")`` + # attempt raised ``AttributeError`` on every call and a bare handler + # silently resolved the operator model instead. The operator model is + # what this agent has always actually used; say so rather than hiding + # it behind a lookup that cannot succeed. Giving the image processor + # its own configurable node is a config-schema change, tracked + # separately. + llm = get_llm(self.ctx, name="operator") base_dir = settings.TRACES_PATH image_processor_dir = base_dir / "images" / "image_processor" diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index 22b34f2e..9c464c84 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -124,8 +124,13 @@ async def _run_object_detection( queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] try: - llm = get_llm(ctx, name="object_detector") - except Exception: + llm = get_llm(ctx, name="object_detector", is_utils=True) + except (ValueError, AttributeError) as exc: + logger.warning( + f"object_detector is not configured ({exc}); falling back to the operator " + "model, which is slower and more expensive. Configure 'object_detector' " + "under the llm utils config to use the intended detector." + ) llm = get_llm(ctx, name="operator") raw_timeout = getattr(getattr(ctx, "llm_config", None), "timeout", None) diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index b53c8834..bd4be13d 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -55,11 +55,16 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn): - """Resolves the VLM and loads the prompt template (once per validation).""" - try: - llm = get_llm_fn(ctx, name="validator_pixel_safety_net") - except Exception: - llm = get_llm_fn(ctx, name="validator") + """Resolves the VLM and loads the prompt template (once per validation). + + ``validator_pixel_safety_net`` is an optional ``LLMConfig`` field whose + ``get_agent`` accessor returns a built-in lightweight-judge default when + unset (see ``LLMConfig.get_agent``), so this call never raises for a + missing config entry. There is no fallback branch: ``validator`` is not a + real ``LLMConfig`` field, so a fallback attempt could only ever raise + itself -- it was unreachable dead code. + """ + llm = get_llm_fn(ctx, name="validator_pixel_safety_net") prompt_path = Path(__file__).parent.joinpath("pixel_safety_net.md") prompt = prompt_path.read_text(encoding="utf-8") diff --git a/tests/unit/agents/test_object_detector_llm_resolution.py b/tests/unit/agents/test_object_detector_llm_resolution.py new file mode 100644 index 00000000..edd25d65 --- /dev/null +++ b/tests/unit/agents/test_object_detector_llm_resolution.py @@ -0,0 +1,148 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CHE-646: object_detector must resolve its own configured model from the +llm utils config, and must never silently fall back to the operator model +without a visible log signal. +""" + +import logging +from unittest.mock import AsyncMock, MagicMock + +from artemis.agents.object_detector import object_detector as od +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +import pytest + + +def _empty_response_llm() -> MagicMock: + """A fake LLM whose ainvoke resolves to an empty-list JSON response, so + _detect_single_label completes cleanly without hitting the network.""" + llm = MagicMock() + llm.ainvoke = AsyncMock(return_value=MagicMock(content="[]")) + return llm + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +def _config(object_detector: LLMWithFallback | None) -> LLMConfig: + """A fully-populated LLMConfig; utils.object_detector varies per test.""" + node = _llm() + return LLMConfig( + **{n: node for n in _REQUIRED_NODES}, + utils=LLMConfigUtils( + outputter=node, + hopper=node, + object_detector=object_detector, + ), + ) + + +@pytest.mark.asyncio +async def test_configured_object_detector_resolves_via_utils_not_operator(monkeypatch): + """When utils.object_detector IS configured, the detector must resolve to + that configured entry via get_llm(name='object_detector', is_utils=True), + never to the operator entry.""" + detector_llm = _llm(model="detector-model") + ctx = MagicMock() + ctx.llm_config = _config(object_detector=detector_llm) + + calls = [] + + def fake_get_llm(ctx_arg, name, is_utils=False, **kwargs): + calls.append({"name": name, "is_utils": is_utils}) + return _empty_response_llm() + + monkeypatch.setattr(od, "get_llm", fake_get_llm) + + result = await od._run_object_detection( + ctx, image_bytes=b"fake-image-bytes", queries=["button"] + ) + + assert result["detected"] == [] + # Exactly one get_llm call, for the configured detector node -- never operator. + assert calls == [{"name": "object_detector", "is_utils": True}] + + +@pytest.mark.asyncio +async def test_unconfigured_object_detector_logs_warning_and_falls_back_to_operator( + monkeypatch, caplog +): + """When utils.object_detector is None, get_llm(is_utils=True) raises ValueError + (see LLMConfig.get_utils). The detector must log a visible warning AND fall + back to the operator model -- the fallback must remain functional, but the + condition must no longer be silently swallowed.""" + ctx = MagicMock() + ctx.llm_config = _config(object_detector=None) + + calls = [] + + def fake_get_llm(ctx_arg, name, is_utils=False, **kwargs): + calls.append({"name": name, "is_utils": is_utils}) + if name == "object_detector": + raise ValueError("Utils 'object_detector' is not configured.") + return _empty_response_llm() + + monkeypatch.setattr(od, "get_llm", fake_get_llm) + + with caplog.at_level(logging.WARNING, logger="artemis.agents.object_detector.object_detector"): + result = await od._run_object_detection( + ctx, image_bytes=b"fake-image-bytes", queries=["button"] + ) + + assert result["detected"] == [] + # Both calls happened: the failed primary attempt, then the operator fallback. + assert calls == [ + {"name": "object_detector", "is_utils": True}, + {"name": "operator", "is_utils": False}, + ] + # The fallback condition must be visible in logs, not silently swallowed. + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert any( + "object_detector" in r.getMessage() and "operator" in r.getMessage() for r in warnings + ) + + +def test_prefix_path_without_is_utils_raises_attribute_error(): + """Regression guard: object_detector is a field of LLMConfigUtils, not of + LLMConfig. get_agent('object_detector') (the pre-fix, is_utils=False path) + must still raise AttributeError -- proving the production call site must + pass is_utils=True and cannot silently rely on get_agent's defaulting.""" + config = _config(object_detector=_llm()) + + with pytest.raises(AttributeError): + config.get_agent("object_detector") # type: ignore[arg-type] + + # The correct (is_utils=True) path resolves cleanly instead. + resolved = config.get_utils("object_detector") + assert resolved.model == "gpt-5.6-sol" From 01bbf72cdae1b51f26113655d5ab0b894f150bfc Mon Sep 17 00:00:00 2001 From: "congvc-bot[bot]" <3634010+congvc-bot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 23:39:09 +0700 Subject: [PATCH 13/13] test(config): cover utils.* node reconciliation as match, not unmapped_call (CHE-656) --- tests/unit/config/test_attempt_manifest.py | 24 +++++++++++++++++++ .../config/test_attempt_reconciliation.py | 22 +++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/unit/config/test_attempt_manifest.py b/tests/unit/config/test_attempt_manifest.py index b63e0f3b..99acd633 100644 --- a/tests/unit/config/test_attempt_manifest.py +++ b/tests/unit/config/test_attempt_manifest.py @@ -237,6 +237,30 @@ def test_build_attempt_manifest_rejects_unknown_tier(self): now=1.0, ) + def test_enabled_utils_nodes_get_same_tier_matching_as_agent_nodes(self): + """outputter/hopper are the two required (non-nullable) utils nodes; + configuring them must give them the exact same enabled/tier-matched + treatment as a required agent node -- not a second-class entry. + """ + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="nova", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + for util in ("outputter", "hopper"): + entry = manifest["utils"][util] + assert entry["enabled"] is True + assert entry["resolved_tier"] == "nova" + assert entry["config"]["provider"] == "anthropic" + assert entry["config"]["model"] == "claude-sonnet-5" + assert manifest["untiered_enabled_nodes"] == [] + class TestNullableNodesRepresentedExplicitly: def test_disabled_nodes_show_would_resolve_default_without_being_active(self): diff --git a/tests/unit/config/test_attempt_reconciliation.py b/tests/unit/config/test_attempt_reconciliation.py index 21c3d7a0..3385791c 100644 --- a/tests/unit/config/test_attempt_reconciliation.py +++ b/tests/unit/config/test_attempt_reconciliation.py @@ -132,6 +132,28 @@ def test_usage_for_node_absent_from_manifest_is_unmapped_call(self): assert unmapped.verdict == "unmapped_call" assert result.has_unmapped_call + def test_utils_node_usage_reconciles_as_match_not_unmapped_call(self): + """outputter/hopper are declared in manifest["utils"], not manifest["nodes"]. + + reconcile_attempt merges both dicts by node name before matching, so a + real llm_usage event recorded under node="outputter" (e.g. the Flash + runner's Outputter synthesis call, which fires on essentially every + real run) must reconcile as "match" against the manifest's utils + entry -- never as unmapped_call just because it lives under a + different top-level manifest key than the twelve agent nodes. + """ + manifest = _manifest() + result = reconcile_attempt( + "a1", + manifest, + [_usage("outputter"), _usage("hopper")], + ) + outputter = next(n for n in result.nodes if n.node == "outputter") + hopper = next(n for n in result.nodes if n.node == "hopper") + assert outputter.verdict == "match" + assert hopper.verdict == "match" + assert not result.has_unmapped_call + def test_enabled_node_never_invoked_is_not_invoked_informational(self): manifest = _manifest() result = reconcile_attempt("a1", manifest, [])