diff --git a/.env.example b/.env.example index 8b23d0a5..6e85be45 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,20 @@ ANTHROPIC_API_KEY= OPEN_ROUTER_API_KEY= XAI_API_KEY= +# Optional provider-compatible endpoints. OpenAI-compatible URLs include `/v1`; +# Anthropic URLs are the API origin because the SDK appends `/v1`. Gemini and +# Vertex values override the SDK service endpoint. Per-model `api_base` in +# artemis.jsonc overrides these values. +# GOOGLE_BASE_URL=https://generativelanguage.googleapis.com +# VERTEX_AI_BASE_URL=https://us-central1-aiplatform.googleapis.com +# OPENAI_BASE_URL=https://openai-proxy.example/v1 +# ANTHROPIC_BASE_URL=https://anthropic-proxy.example +# OPEN_ROUTER_BASE_URL=https://openrouter.ai/api/v1 +# XAI_BASE_URL=https://api.x.ai/v1 +# OLLAMA_BASE_URL=http://localhost:11434/v1 +# VLLM_BASE_URL=http://localhost:8000/v1 +# CUSTOM_BASE_URL=http://localhost:8000/v1 + # Google Cloud Vision OCR (Optional - for advanced OCR processing) OCR_API_KEY= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2be0ff5d..8ba51bb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ env: UV_NO_PROGRESS: "1" jobs: + # Windows leg removed org-wide per CHE-522 (Cheese: Linux CI only, no + # Windows jobs or artifacts). python-quality: name: Python 3.12 / ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -25,8 +27,7 @@ jobs: fail-fast: false matrix: os: - - ubuntu-latest - - windows-latest + - blacksmith-2vcpu-ubuntu-2404 steps: - name: Check out repository @@ -78,7 +79,7 @@ jobs: frontend: name: Frontend tests and production build - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 20 defaults: run: @@ -106,7 +107,7 @@ jobs: package: name: Build and smoke-test Python packages - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 30 steps: diff --git a/.quality-baseline.json b/.quality-baseline.json index 5b53f91a..9cf1508f 100644 --- a/.quality-baseline.json +++ b/.quality-baseline.json @@ -1,5 +1,5 @@ { - "broad_exception_handlers": 754, + "broad_exception_handlers": 759, "silent_broad_exception_handlers": 0, "type_ignore_comments": 18 } diff --git a/apps/admin_console/database/repositories/session_repository.py b/apps/admin_console/database/repositories/session_repository.py index a12e8692..fba0fbab 100644 --- a/apps/admin_console/database/repositories/session_repository.py +++ b/apps/admin_console/database/repositories/session_repository.py @@ -510,6 +510,40 @@ def get_session_status(self, session_id: str) -> str | None: except Exception: return None + def create_queued_session( + self, + session_id: str, + goal: str, + profile: str, + device_serial: str | None, + start_time: float | None = None, + ) -> bool: + """Persist a queue item before its worker starts a session.""" + try: + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO sessions " + "(session_id, initial_goal, start_time, end_time, status, device_info, pid) " + "VALUES (?, ?, ?, NULL, 'queued', ?, NULL)", + ( + str(session_id), + goal, + start_time if start_time is not None else time.time(), + json.dumps( + { + "profile": profile, + "device_id": device_serial, + } + ), + ), + ) + conn.commit() + return cursor.rowcount > 0 + except Exception: + logger.exception("Could not create queued session %s", session_id) + return False + def update_session_status( self, session_id: str, status: str, end_time: float | None = None ) -> bool: diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index 5127eca6..f9159653 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -422,9 +422,12 @@ def mask_key(k: str | None) -> str | None: anthropic_real = get_real_env_value("ANTHROPIC_API_KEY", "anthropic") openrouter_real = get_real_env_value("OPEN_ROUTER_API_KEY", "openrouter") xai_real = get_real_env_value("XAI_API_KEY", "xai") - base_url_val = settings.OPENAI_BASE_URL or os.environ.get("OPENAI_BASE_URL") - if base_url_val and is_placeholder_key(base_url_val): - base_url_val = None + openai_base_url = settings.OPENAI_BASE_URL or os.environ.get("OPENAI_BASE_URL") + anthropic_base_url = settings.ANTHROPIC_BASE_URL or os.environ.get("ANTHROPIC_BASE_URL") + if openai_base_url and is_placeholder_key(openai_base_url): + openai_base_url = None + if anthropic_base_url and is_placeholder_key(anthropic_base_url): + anthropic_base_url = None ocr_real = get_real_env_value("OCR_API_KEY", "ocr") or get_real_env_value( "VISION_API_KEY", "ocr" ) @@ -467,10 +470,17 @@ def mask_key(k: str | None) -> str | None: }, { "name": "OPENAI_BASE_URL", - "provider": "custom", - "is_set": bool(base_url_val), - "preview": base_url_val, - "description": "Custom API endpoint (for local Ollama, vLLM, DeepSeek, or proxies)", + "provider": "openai", + "is_set": bool(openai_base_url), + "preview": openai_base_url, + "description": "Custom OpenAI-compatible API endpoint", + }, + { + "name": "ANTHROPIC_BASE_URL", + "provider": "anthropic", + "is_set": bool(anthropic_base_url), + "preview": anthropic_base_url, + "description": "Custom Anthropic-compatible API endpoint", }, { "name": "VISION_API_KEY", diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 26fca461..1cf4f158 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -178,6 +178,7 @@ async def run_task(request: RunRequest): ingress=request.ingress or "frontend", session_id=request.session_id, conversation_id=request.conversation_id, + run_id=request.run_id, ) diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 144a9720..09c90f30 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -32,6 +32,7 @@ class RunRequest(BaseModel): ingress: str | None = "frontend" session_id: str | None = None conversation_id: str | None = None + run_id: str | None = None class ReplayRequest(BaseModel): diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b99b4c91..203bd8ff 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -507,6 +507,7 @@ def _build_worker_invocation( explorer_mode = task_item.get("explorer_mode") locked_app = task_item.get("locked_app_package") or task_item.get("locked_app") app_path = task_item.get("app_path") + run_id = task_item.get("run_id") test_name = f"web_{int(time.time())}_{run_key[:8]}" env = os.environ.copy() @@ -543,6 +544,8 @@ def _build_worker_invocation( ] if sess_id: cmd.extend(["--session-id", str(sess_id)]) + if run_id: + cmd.extend(["--run-id", str(run_id)]) if expected_output: cmd.extend(["--output-description", str(expected_output)]) if enable_outputter is not None: @@ -1004,6 +1007,7 @@ def _create_queue_item( conversation_id: str | None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Reserve a device slot and build one pending queue item for a goal.""" sess_id = single_session_id if single_session_id else str(uuid.uuid4()) @@ -1031,6 +1035,7 @@ def _create_queue_item( "adb_endpoint": endpoint.to_dict(), "ingress": ingress, "conversation_id": conversation_id, + "run_id": run_id, "status": "pending", "queue_ticket": queue_ticket, "created_at": now + index * 0.001, @@ -1052,12 +1057,18 @@ async def enqueue_tasks( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Enqueues one or more goals and wakes up the background worker. ``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. + + ``run_id`` is the Gate 1 batch-grouping key (see + ``artemis.config.attempt_lifecycle_hooks``); it is forwarded to the + spawned worker as ``--run-id`` so daemon-dispatched attempts get the + same manifest/reconciliation evidence as standalone runs. """ verification_level = ( str(verification_level).strip().lower() or None if verification_level else None @@ -1066,6 +1077,7 @@ async def enqueue_tasks( cls.ensure_worker_running() enqueued_tasks = [] + created_trace_session_ids: set[str] = set() now = time.time() endpoint = current_adb_endpoint() @@ -1105,7 +1117,64 @@ async def enqueue_tasks( conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + run_id=run_id, + ) + session_id = str(task_item["session_id"]) + existing_trace = trace_store.read_status(session_id) + trace_created = existing_trace is None + existing_trace_is_terminal = bool( + existing_trace + and existing_trace.get("status") in {"completed", "failed", "cancelled", "success"} ) + try: + if existing_trace_is_terminal: + raise RuntimeError(f"Session {session_id} already has a terminal trace") + if trace_created: + trace_store.init_trace( + session_id, + goal, + task_item["profile"], + task_item.get("conversation_id"), + task_item.get("device_serial"), + ) + created_trace_session_ids.add(session_id) + if not session_repo.create_queued_session( + session_id, + goal, + task_item["profile"], + task_item.get("device_serial"), + task_item.get("start_time"), + ): + raise RuntimeError(f"Could not persist queued session {session_id}") + except (OSError, RuntimeError) as exc: + DeviceExecutionLock.cancel_reservation(task_item.get("queue_ticket")) + if trace_created or ( + not existing_trace_is_terminal + and session_repo.get_session_by_id(session_id) is None + ): + try: + trace_store.update_trace_status(session_id, "failed", error=str(exc)) + except OSError: + logger.exception( + "Could not mark queue setup failure for session %s", session_id + ) + for enqueued_task in enqueued_tasks: + enqueued_session_id = str(enqueued_task["session_id"]) + session_repo.update_session_status(enqueued_session_id, "failed", time.time()) + if enqueued_session_id in created_trace_session_ids: + try: + trace_store.update_trace_status( + enqueued_session_id, + "failed", + error="Task batch could not be queued.", + ) + except OSError: + logger.exception( + "Could not mark rolled-back queue session %s failed", + enqueued_session_id, + ) + cls._remove_task(enqueued_session_id) + raise state.queue_items.append(task_item) enqueued_tasks.append(task_item) cls._broadcast_startup_progress( diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 936f6065..27e03351 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -81,9 +81,7 @@ from artemis.mcp.observation import observe from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger, mark_ephemeral from artemis.services.llm import ( - RobustChatModelWrapper, acomplete, - get_google_llm, get_llm, invoke_llm_with_timeout_message, ) @@ -279,12 +277,7 @@ def _build_ledger(self) -> TranscriptLedger: def _init_llm(self): """Initializes the Universal LLM via the Service Layer.""" - try: - return get_llm(self.ctx, name="operator") - except Exception as e: - logger.warning(f"Failed to get operator LLM from config, using default: {e}") - - return RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx) + return get_llm(self.ctx, name="operator") def _render_system_prompt(self, tools_declaration: list) -> str: """Renders the system prompt from the flash_runner.md template. diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 56c5e098..12fa6be4 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -36,7 +36,7 @@ from artemis.context import ArtemisContext from artemis.memory.step_memory import JobKey, StepMemoryService -from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm +from artemis.services.llm import RobustChatModelWrapper, get_llm from artemis.services.token_meter import record_llm_usage from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_actions_clean @@ -161,18 +161,22 @@ def __init__( flush_timeout_s=flush_timeout_s, ) - # Initialize lightweight VLM: prioritize explicit model_name - target_model = model_name or "gemini-2.5-flash-lite" - self._model_name = target_model - try: - if model_name: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) - else: - self._llm = get_llm(ctx, name="summarizer", is_utils=True) - except Exception: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) + # The summarizer role carries the provider and default model. This profile + # knob only overrides that model when the caller explicitly supplies one. + self._model_name = model_name or "" + self._llm = get_llm( + ctx, + name="summarizer", + temperature=0.0, + model_name=model_name or None, + ) try: - configured = getattr(self._llm, "model", None) or getattr(self._llm, "model_name", None) + endpoint = getattr(self._llm, "endpoint", None) + configured = getattr(endpoint, "model_name", None) + if not configured: + configured = getattr(self._llm, "model", None) or getattr( + self._llm, "model_name", None + ) if isinstance(configured, str) and configured: self._model_name = configured except Exception as exc: diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 05e37abb..c2538576 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -79,10 +79,15 @@ def __init__(self, ctx: ArtemisContext): @trace(type="agent", name="image_processor") async def run(self, instruction: str, target_image_path: str) -> dict: - try: - llm = get_llm(self.ctx, name="image_processor") - except Exception: - llm = get_llm(self.ctx, name="operator") + # There is no ``image_processor`` node on ``LLMConfig`` or + # ``LLMConfigUtils``, so the former ``get_llm(name="image_processor")`` + # attempt raised ``AttributeError`` on every call and a bare handler + # silently resolved the operator model instead. The operator model is + # what this agent has always actually used; say so rather than hiding + # it behind a lookup that cannot succeed. Giving the image processor + # its own configurable node is a config-schema change, tracked + # separately. + llm = get_llm(self.ctx, name="operator") base_dir = settings.TRACES_PATH image_processor_dir = base_dir / "images" / "image_processor" diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index 22b34f2e..9c464c84 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -124,8 +124,13 @@ async def _run_object_detection( queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] try: - llm = get_llm(ctx, name="object_detector") - except Exception: + llm = get_llm(ctx, name="object_detector", is_utils=True) + except (ValueError, AttributeError) as exc: + logger.warning( + f"object_detector is not configured ({exc}); falling back to the operator " + "model, which is slower and more expensive. Configure 'object_detector' " + "under the llm utils config to use the intended detector." + ) llm = get_llm(ctx, name="operator") raw_timeout = getattr(getattr(ctx, "llm_config", None), "timeout", None) diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index b53c8834..bd4be13d 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -55,11 +55,16 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn): - """Resolves the VLM and loads the prompt template (once per validation).""" - try: - llm = get_llm_fn(ctx, name="validator_pixel_safety_net") - except Exception: - llm = get_llm_fn(ctx, name="validator") + """Resolves the VLM and loads the prompt template (once per validation). + + ``validator_pixel_safety_net`` is an optional ``LLMConfig`` field whose + ``get_agent`` accessor returns a built-in lightweight-judge default when + unset (see ``LLMConfig.get_agent``), so this call never raises for a + missing config entry. There is no fallback branch: ``validator`` is not a + real ``LLMConfig`` field, so a fallback attempt could only ever raise + itself -- it was unreachable dead code. + """ + llm = get_llm_fn(ctx, name="validator_pixel_safety_net") prompt_path = Path(__file__).parent.joinpath("pixel_safety_net.md") prompt = prompt_path.read_text(encoding="utf-8") diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index e539f933..681a04e5 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -58,11 +58,13 @@ ENV_ADB_PORT, ENV_ADB_SERVER_SOCKET, ENV_ANTHROPIC_API_KEY, + ENV_ANTHROPIC_BASE_URL, ENV_ANTIGRAVITY_APP_DIR, ENV_ANTIGRAVITY_LS_ADDRESS, ENV_DATA_ENGINE_DB_PATH, ENV_EVENTS_OUTPUT_PATH, ENV_GCP_API_KEY, + ENV_GOOGLE_BASE_URL, ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_KEEP_VIDEOS, @@ -84,10 +86,16 @@ ENV_ARTEMIS_USE_FILE_API, ENV_ARTEMIS_USE_USER_DIR, ENV_OPEN_ROUTER_API_KEY, + ENV_OPEN_ROUTER_BASE_URL, ENV_OPENAI_API_KEY, ENV_OPENAI_BASE_URL, ENV_RESULTS_OUTPUT_PATH, ENV_XAI_API_KEY, + ENV_XAI_BASE_URL, + ENV_VERTEX_AI_BASE_URL, + ENV_OLLAMA_BASE_URL, + ENV_VLLM_BASE_URL, + ENV_CUSTOM_BASE_URL, IPC_PORT_FILENAME, LLM_CONFIG_FILENAME, LLM_CONFIG_OVERRIDE_FILENAME, @@ -293,11 +301,19 @@ "ENV_GOOGLE_API_KEY", "ENV_GEMINI_API_KEY", "ENV_GCP_API_KEY", + "ENV_GOOGLE_BASE_URL", "ENV_OPENAI_API_KEY", "ENV_OPENAI_BASE_URL", "ENV_ANTHROPIC_API_KEY", + "ENV_ANTHROPIC_BASE_URL", "ENV_OPEN_ROUTER_API_KEY", + "ENV_OPEN_ROUTER_BASE_URL", "ENV_XAI_API_KEY", + "ENV_XAI_BASE_URL", + "ENV_VERTEX_AI_BASE_URL", + "ENV_OLLAMA_BASE_URL", + "ENV_VLLM_BASE_URL", + "ENV_CUSTOM_BASE_URL", "ENV_ARTEMIS_EXPLORER_VERSION", "ENV_ARTEMIS_DEFAULT_PROFILE", "ENV_ARTEMIS_DEFAULT_MODEL", diff --git a/artemis/config/attempt_lifecycle_hooks.py b/artemis/config/attempt_lifecycle_hooks.py new file mode 100644 index 00000000..fac734a7 --- /dev/null +++ b/artemis/config/attempt_lifecycle_hooks.py @@ -0,0 +1,185 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared best-effort Gate 1 evidence hooks for every real execution path. + +Both the standalone/worker-subprocess runner (``mcp_server.background.task_runner``) +and the daemon-dispatched / direct-CLI runner (``artemis.interfaces.cli.commands.run``) +build an ``Agent`` from an ``LLMConfig`` and call ``agent.run_task`` under a shared +``trace_id``. This module gives both call sites the exact same +record-manifest-before / reconcile-after contract so a Gate 1 evidence gap is never a +function of which entry point started the run. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def record_attempt_manifest( + *, + trace_id: str, + checkpoint: str, + llm_config: Any, + run_id: str | None = None, + parent_attempt_id: str | None = None, +) -> None: + """Best-effort Gate 1 evidence hook: resolve, hash and store one attempt + manifest checkpoint beside this trace before the real inference run. + + Mirrors ``token_meter.record_llm_usage``'s own contract: this must never + raise into the task execution path. A tier that cannot be inferred from + ``llm_config`` (this fork's ``config/artemis.jsonc`` does not yet declare + an explicit tier table -- see ``attempt_manifest.TIER_MODELS``) or a + checkpoint that collides with an already-stored one (create-only storage) + are both recorded as skips, not failures, so a manifest gap is visible in + the caller's own log without ever aborting a live mobile task. + + ``run_id`` defaults to ``trace_id`` when not given, preserving the exact + 1:1 ``run_id``<->``trace_id`` behavior every existing caller relies on. A + caller that wants several real attempts to be reconcilable together as + one batch (see ``attempt_reconciliation.reconcile_attempt_batch_by_run_id``) + can pass a ``run_id`` shared across multiple calls instead. + """ + try: + from artemis.config.attempt_manifest import ( + TIER_MODELS, + ManifestAlreadyExistsError, + build_attempt_manifest, + store_attempt_manifest, + ) + import os + + tier = None + for candidate_tier, (provider, model_name) in TIER_MODELS.items(): + if llm_config.planner.provider == provider and llm_config.planner.model == model_name: + tier = candidate_tier + break + if tier is None: + print( + f"Attempt manifest [{checkpoint}] skipped: planner " + f"{llm_config.planner.provider}/{llm_config.planner.model} does not match " + "a declared tier in attempt_manifest.TIER_MODELS." + ) + return + + manifest = build_attempt_manifest( + run_id=run_id or trace_id, + attempt_id=trace_id, + trace_id=trace_id, + tier=tier, + llm_config=llm_config, + env=dict(os.environ), + checkpoint=checkpoint, + parent_attempt_id=parent_attempt_id, + ) + manifest_path, digest = store_attempt_manifest(manifest) + print(f"Attempt manifest [{checkpoint}] stored at {manifest_path} (sha256={digest}).") + except ManifestAlreadyExistsError as exc: + print(f"Attempt manifest [{checkpoint}] skipped: {exc}") + except Exception as exc: + print(f"Attempt manifest [{checkpoint}] skipped (best-effort, non-fatal): {exc}") + + +def reconcile_and_store_verdict( + *, trace_id: str, checkpoint: str = "launch", run_id: str | None = None +) -> None: + """Best-effort Gate 1 evidence hook: reconcile this attempt's stored + manifest against its native ``llm_usage`` receipts and preserve the + verdict beside the trace. + + Runs after the real inference run returns, regardless of task outcome -- + identity verification is orthogonal to whether the mobile task itself + passed or failed. Never raises into the task path (same best-effort + contract as :func:`record_attempt_manifest`). Preserves the original + ``llm_usage`` receipts unmodified: this only adds a derived verdict file, + it never rewrites or drops the native rows the DataEngine already wrote. + + When ``run_id`` is given and genuinely differs from ``trace_id`` (i.e. + this attempt was launched as part of an explicit multi-attempt batch, not + the default 1:1 case), this also runs + ``attempt_reconciliation.reconcile_attempt_batch_by_run_id`` across every + stored attempt sharing that ``run_id`` and stores that batch verdict as a + sibling file, ``attempt_reconciliation_batch_verdict.json``, next to the + single-attempt verdict -- kept separate so neither file's meaning is + ambiguous. The batch call is wrapped in its own best-effort guard so a + batch-reconciliation failure can never suppress the single-attempt + verdict this function already stored. + """ + from artemis.runtime import trace_store + + try: + from artemis.config.attempt_reconciliation import reconcile_finished_attempt + from artemis.config.paths import get_data_engine_db_path, get_traces_dir + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint=checkpoint, + db_path=get_data_engine_db_path(), + traces_dir=get_traces_dir(), + ) + if verdict is None: + print( + f"Attempt reconciliation skipped: no stored manifest for trace {trace_id!r} " + f"checkpoint {checkpoint!r}." + ) + else: + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + verdict_path = trace_dir / "attempt_reconciliation_verdict.json" + verdict_payload = { + "accepted": verdict.accepted, + "reason": verdict.reason, + "detail": verdict.detail, + "invalid_attempt_count": len(verdict.invalid_attempts), + } + verdict_path.write_text(json.dumps(verdict_payload, indent=2), encoding="utf-8") + print( + f"Attempt reconciliation verdict stored at {verdict_path}: " + f"accepted={verdict.accepted} reason={verdict.reason}." + ) + except Exception as exc: + print(f"Attempt reconciliation skipped (best-effort, non-fatal): {exc}") + + if run_id is not None and run_id != trace_id: + try: + from artemis.config.attempt_reconciliation import reconcile_attempt_batch_by_run_id + from artemis.config.paths import get_data_engine_db_path, get_traces_dir + + batch_verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint=checkpoint, + db_path=get_data_engine_db_path(), + traces_dir=get_traces_dir(), + traces_root=Path(trace_store.TRACES_DIR), + ) + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + batch_verdict_path = trace_dir / "attempt_reconciliation_batch_verdict.json" + batch_verdict_payload = { + "accepted": batch_verdict.accepted, + "reason": batch_verdict.reason, + "detail": batch_verdict.detail, + "invalid_attempt_count": len(batch_verdict.invalid_attempts), + } + batch_verdict_path.write_text( + json.dumps(batch_verdict_payload, indent=2), encoding="utf-8" + ) + print( + f"Attempt batch reconciliation verdict stored at {batch_verdict_path}: " + f"accepted={batch_verdict.accepted} reason={batch_verdict.reason}." + ) + except Exception as exc: + print(f"Attempt batch reconciliation skipped (best-effort, non-fatal): {exc}") diff --git a/artemis/config/attempt_manifest.py b/artemis/config/attempt_manifest.py new file mode 100644 index 00000000..65a732b0 --- /dev/null +++ b/artemis/config/attempt_manifest.py @@ -0,0 +1,555 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-attempt, credential-free LLM identity manifest (Gate 1 mechanism). + +Builds one complete, provable record of exactly which model/provider/tier +resolved for every model-bearing node before an attempt starts inference, +serializes it to a deterministic canonical byte form, hashes it, and writes +it create-only beside the attempt's trace directory +(:func:`artemis.runtime.trace_store.get_trace_dir`). + +This module implements the *mechanism* only. It does not itself run a +qualification pilot, and nothing here should be read as evidence that any +Gate 1 qualification has passed — see ``artemis.config.attempt_reconciliation`` +for the usage-vs-manifest reconciliation and batch accept/reject rule that +consume this manifest's output. + +Isolation scope note: :func:`build_attempt_manifest` is a pure function of +its explicit arguments (an ``LLMConfig``, a tier label, and an explicit env +snapshot dict) plus process-global constants it does not mutate. It does not +read ``os.environ`` directly and does not touch ``ModelFactory``'s +process-wide model cache. That makes it possible to prove, in-process, that +two calls with different explicit inputs never share state — see the unit +tests for the specific isolation boundary this actually exercises versus the +real MCP/device process boundary, which this module does not touch. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any, Literal + +from artemis.config.constants import AgentNode, LLMUtilsNode +from artemis.config.llm import LLM, LLMConfig, LLMWithFallback, lightweight_judge_default +from artemis.runtime import trace_store + +# ============================================================================== +# Tier mapping +# ============================================================================== + +#: Internal model-tier labels (Luna/Terra/Sol cheap/mid/premium) used across +#: Cheese Work squads. Every enabled model-bearing node in a manifest must +#: resolve to exactly one of these tiers (mixed tiers => ``mixed_tier`` +#: rejection, see ``attempt_reconciliation.validate_batch``). +Tier = Literal["luna", "terra", "sol", "nova"] + +TIERS: tuple[Tier, ...] = ("luna", "terra", "sol", "nova") + +# Provider/model pairs that identify each tier. This fork's config/artemis.jsonc +# does not (yet) declare an explicit tier->model table; its "default" node +# resolves to openai/gpt-5.6-sol today, so "sol" is mapped to that pair as the +# only tier presently exercised end-to-end. "luna"/"terra" are declared for +# forward compatibility with the escalation controller (Gate 2, out of scope +# here) and must be updated here (not inferred) if/when config/artemis.jsonc +# grows an explicit tier table. "nova" is the Anthropic tier added for the +# CHE-491 qualification pilot against the sub2api gateway (CHE-639). +TIER_MODELS: dict[Tier, tuple[str, str]] = { + "luna": ("google", "gemini-3.5-flash-lite"), + "terra": ("google", "gemini-3.8-flash"), + "sol": ("openai", "gpt-5.6-sol"), + "nova": ("anthropic", "claude-sonnet-5"), +} + +# Environment variables whose presence/value can influence LLMConfig +# resolution and must be captured (never their raw secret value) in the +# manifest's provenance section. +_RELEVANT_ENV_PREFIXES = ("ARTEMIS_",) +_FAKE_LLM_ENV_VAR = "ARTEMIS_FAKE_LLM" + +# Substrings that mark an ARTEMIS_* variable name as credential-shaped (e.g. +# ARTEMIS_TENANT_TOKEN, ARTEMIS_LIFECYCLE_TOKEN). Matched case-insensitively +# against the whole var name. Any ARTEMIS_* var matching one of these must +# never have its raw value copied into the manifest's env_overrides -- it is +# reported the same non-secret way as _credential_reference (set + last4) +# instead. This is deliberately a denylist of credential-shaped substrings, +# not an allowlist of every non-secret ARTEMIS_* knob, since new resolution +# knobs are added far more often than new credential-bearing vars. +_CREDENTIAL_SHAPED_NAME_MARKERS = ( + "TOKEN", + "KEY", + "SECRET", + "PASSWORD", + "CREDENTIAL", + "AUTH", +) + + +def _is_credential_shaped_env_name(name: str) -> bool: + """True if ``name`` looks like it carries a secret (by name only).""" + upper = name.upper() + return any(marker in upper for marker in _CREDENTIAL_SHAPED_NAME_MARKERS) + + +def _sensitive_env_reference(value: str) -> dict[str, Any]: + """Non-secret presence reference for one credential-shaped env var value. + + Mirrors :func:`_credential_reference`'s shape exactly: never includes the + raw value, and omits ``last4`` entirely when the value is empty or + shorter than 4 characters. + """ + ref: dict[str, Any] = {"set": bool(value)} + if value and len(value) >= 4: + ref["last4"] = value[-4:] + return ref + + +# Every LLMConfig field that can carry an LLMWithFallback, in the exact +# vocabulary of the spec. Fields not present in LLMConfig.model_fields are a +# programming error (caught by the assertion in build_attempt_manifest). +_TOP_LEVEL_NODES: tuple[AgentNode, ...] = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + "history_analyzer", + "validator_pixel_safety_net", + "planner_validation", + "output_analyzer", +) + +# Nodes that are soft-defaulted by LLMConfig.get_agent() when absent, and the +# node they inherit from / default factory they use. Mirrors the branches in +# LLMConfig.get_agent verbatim so the manifest's "would resolve to" values +# never drift from the real resolution code. +_SOFT_DEFAULT_INHERITS_FROM: dict[str, str] = { + "history_analyzer": "operator", + "output_analyzer": "log_analyzer", +} +_SOFT_DEFAULT_FACTORY_NODES: frozenset[str] = frozenset( + {"validator_pixel_safety_net", "planner_validation"} +) + +_UTILS_NODES: tuple[LLMUtilsNode, ...] = ( + "outputter", + "hopper", + "video_analyzer", + "object_detector", +) + +# Nullable utils nodes have no soft-default factory (LLMConfig.get_utils raises +# instead); the manifest still records what they would need to be configured +# as, without pretending they are active. +_NULLABLE_UTILS_NODES: frozenset[str] = frozenset({"video_analyzer", "object_detector"}) + + +def _canonical_repo_sha(repo_root: Path | None) -> tuple[str, str]: + """Returns ``(sha, source)`` for the artemis repo's current commit. + + ``source`` is ``"git"`` on success. On any failure (not a git checkout, + git missing, detached worktree issue, etc.) falls back to + ``"unknown"`` with a fixed sentinel SHA rather than raising — a Gate 1 + manifest must always build, even in a source tree that lost its .git + directory (e.g. an extracted archive or bundled wheel), but the fallback + must be unambiguous in the manifest, never mistaken for a real SHA. + """ + root = repo_root or Path(__file__).resolve().parent.parent.parent + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(root), + capture_output=True, + text=True, + timeout=10, + check=True, + ) + sha = result.stdout.strip() + if sha: + return sha, "git" + except (OSError, subprocess.SubprocessError): + pass + return "0" * 40, "unknown-not-a-git-checkout" + + +def _relevant_env_snapshot(env: dict[str, str]) -> dict[str, Any]: + """Filters an explicit env mapping down to Artemis-relevant keys only. + + Never includes raw credential values. Most ``ARTEMIS_*`` vars are + resolution knobs (e.g. ``ARTEMIS_FAKE_LLM``, ``ARTEMIS_CONFIG_DIR``) and + pass through as-is. Any ``ARTEMIS_*`` var whose *name* is credential-shaped + (see ``_CREDENTIAL_SHAPED_NAME_MARKERS`` -- e.g. ``ARTEMIS_TENANT_TOKEN``) + is never copied by raw value: it is replaced with the same non-secret + ``{"set": bool, "last4": ...}`` shape ``_credential_reference`` already + uses for provider API keys, so a manifest can never leak a raw + credential into ``env_overrides`` even though provider credentials are + also reported separately in the ``credentials`` section. + """ + out: dict[str, Any] = {} + for k, v in env.items(): + if not k.startswith(_RELEVANT_ENV_PREFIXES): + continue + if _is_credential_shaped_env_name(k): + out[k] = _sensitive_env_reference(v) + else: + out[k] = v + return out + + +def _credential_reference(env: dict[str, str], provider: str) -> dict[str, Any]: + """Non-secret credential presence reference for one provider: set + last-4. + + Never includes the raw key. ``last4`` is omitted entirely when the key is + unset or shorter than 4 characters, rather than emitting an empty string. + """ + env_var = { + "openai": "OPENAI_API_KEY", + "google": "GOOGLE_API_KEY", + "vertexai": None, + "anthropic": "ANTHROPIC_API_KEY", + "openrouter": "OPEN_ROUTER_API_KEY", + "xai": "XAI_API_KEY", + "ollama": None, + "vllm": None, + "custom": None, + }.get(provider) + if env_var is None: + return {"provider": provider, "credential_env_var": None, "set": None} + value = env.get(env_var) + ref: dict[str, Any] = { + "provider": provider, + "credential_env_var": env_var, + "set": bool(value), + } + if value and len(value) >= 4: + ref["last4"] = value[-4:] + return ref + + +def _llm_node_snapshot(llm: LLM | LLMWithFallback) -> dict[str, Any]: + """Sanitized, non-secret snapshot of one LLM/LLMWithFallback record.""" + snap: dict[str, Any] = { + "provider": llm.provider, + "model": llm.model, + "api_base": llm.api_base, + "temperature": llm.temperature, + "thinking_budget": llm.thinking_budget, + "thinking_level": llm.thinking_level, + "reasoning_effort": llm.reasoning_effort, + "include_thoughts": llm.include_thoughts, + "enable_grounding": llm.enable_grounding, + } + if isinstance(llm, LLMWithFallback): + snap["fallback"] = _llm_node_snapshot(llm.fallback) + snap["fix_model"] = llm.fix_model + snap["timeout"] = llm.timeout + return snap + + +def _node_tier(llm: LLM | LLMWithFallback) -> Tier | None: + """Which declared tier (if any) this node's primary provider/model matches.""" + for tier, (provider, model) in TIER_MODELS.items(): + if llm.provider == provider and llm.model == model: + return tier + return None + + +@dataclasses.dataclass(frozen=True) +class NodeManifestEntry: + """One node's manifest entry: enabled/disabled state, snapshot, and tier match.""" + + node: str + enabled: bool + provenance: str + config: dict[str, Any] | None + resolved_tier: Tier | None + # For disabled (None-valued, soft-defaulted) nodes only: what get_agent() + # would actually return if this node were invoked right now, so the + # manifest never silently omits the effective default. + would_resolve_to: dict[str, Any] | None = None + would_resolve_provenance: str | None = None + + +def _build_node_entry( + node: str, + raw_value: LLMWithFallback | None, + config: LLMConfig, + layer_hint: str, +) -> NodeManifestEntry: + if raw_value is not None: + snapshot = _llm_node_snapshot(raw_value) + return NodeManifestEntry( + node=node, + enabled=True, + provenance=layer_hint, + config=snapshot, + resolved_tier=_node_tier(raw_value), + ) + + # Explicitly disabled/nullable node. Represent it as disabled, and also + # show what get_agent() would resolve it to if invoked, without treating + # that hypothetical resolution as an active manifest entry. + would_resolve_to: dict[str, Any] | None = None + would_resolve_provenance: str | None = None + if node in _SOFT_DEFAULT_INHERITS_FROM: + inherited_from = _SOFT_DEFAULT_INHERITS_FROM[node] + inherited_value = getattr(config, inherited_from) + would_resolve_to = _llm_node_snapshot(inherited_value) + would_resolve_provenance = f"soft-default: inherits '{inherited_from}'" + elif node in _SOFT_DEFAULT_FACTORY_NODES: + would_resolve_to = _llm_node_snapshot(lightweight_judge_default()) + would_resolve_provenance = "soft-default: lightweight_judge_default()" + elif node in _NULLABLE_UTILS_NODES: + would_resolve_provenance = "no soft default: get_utils() raises if invoked while unset" + + return NodeManifestEntry( + node=node, + enabled=False, + provenance="disabled (None in LLMConfig)", + config=None, + resolved_tier=None, + would_resolve_to=would_resolve_to, + would_resolve_provenance=would_resolve_provenance, + ) + + +def _node_entries_to_dict(entries: list[NodeManifestEntry]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for entry in entries: + out[entry.node] = { + "enabled": entry.enabled, + "provenance": entry.provenance, + "config": entry.config, + "resolved_tier": entry.resolved_tier, + "would_resolve_to": entry.would_resolve_to, + "would_resolve_provenance": entry.would_resolve_provenance, + } + return out + + +def build_attempt_manifest( + *, + run_id: str, + attempt_id: str, + trace_id: str, + tier: Tier, + llm_config: LLMConfig, + env: dict[str, str], + checkpoint: Literal["launch", "worker_start", "termination"], + parent_attempt_id: str | None = None, + config_layer_hint: str = "resolved LLMConfig (layer not separately tracked by caller)", + repo_root: Path | None = None, + now: float | None = None, +) -> dict[str, Any]: + """Builds one complete, credential-free attempt manifest. + + Pure function of its explicit arguments: it never reads ``os.environ`` + itself (the caller must snapshot it into ``env``) and never touches + ``ModelFactory``'s process-wide model cache. Two calls with different + ``env``/``llm_config``/``tier`` inputs are guaranteed to diverge; two + calls with identical inputs (module state held constant) are guaranteed + to produce logically identical manifests, modulo ``now``/checkpoint. + + ``config_layer_hint`` documents which resolution layer produced + ``llm_config`` as a whole (e.g. "env override: ARTEMIS_CONFIG_DIR", + "config/artemis.jsonc", "hardcoded default in parse_llm_config"); callers + that track richer per-field provenance may pass a more specific string, + or post-process the returned dict's per-node ``provenance`` values. + + Raises ``ValueError`` if ``tier`` is not one of the declared tiers, or if + the resulting manifest mixes tiers across enabled nodes (fail fast at + build time rather than deferring the mixed_tier check to batch + validation, since a single manifest that already disagrees with itself + should never be persisted as if it were valid). + """ + if tier not in TIER_MODELS: + raise ValueError(f"Unknown tier {tier!r}; expected one of {TIERS}") + + sha, sha_source = _canonical_repo_sha(repo_root) + relevant_env = _relevant_env_snapshot(env) + fake_llm_value = env.get(_FAKE_LLM_ENV_VAR) + fake_llm_enabled = fake_llm_value == "1" + + node_entries = [ + _build_node_entry(node, getattr(llm_config, node), llm_config, config_layer_hint) + for node in _TOP_LEVEL_NODES + ] + util_entries = [ + _build_node_entry(util, getattr(llm_config.utils, util), llm_config, config_layer_hint) + for util in _UTILS_NODES + ] + + enabled_tiers = { + e.resolved_tier for e in (*node_entries, *util_entries) if e.enabled and e.resolved_tier + } + untiered_enabled = [ + e.node for e in (*node_entries, *util_entries) if e.enabled and e.resolved_tier is None + ] + if enabled_tiers - {tier}: + raise ValueError( + f"Attempt manifest requested tier {tier!r} but enabled node(s) resolve to " + f"other tier(s) {sorted(enabled_tiers - {tier})}; refusing to build a " + "self-contradicting manifest. This is the mixed_tier condition." + ) + + credentials = { + provider: _credential_reference(env, provider) + for provider in ("openai", "google", "anthropic", "openrouter", "xai") + } + + manifest: dict[str, Any] = { + "manifest_schema_version": 1, + "run_id": run_id, + "attempt_id": attempt_id, + "trace_id": trace_id, + "parent_attempt_id": parent_attempt_id, + "checkpoint": checkpoint, + "recorded_at": now if now is not None else time.time(), + "tier": tier, + "source_sha": sha, + "source_sha_provenance": sha_source, + "fake_llm_enabled": fake_llm_enabled, + "fake_llm_env_var": _FAKE_LLM_ENV_VAR, + "env_overrides": relevant_env, + "credentials": credentials, + "nodes": _node_entries_to_dict(node_entries), + "utils": _node_entries_to_dict(util_entries), + "untiered_enabled_nodes": sorted(untiered_enabled), + } + return manifest + + +# ============================================================================== +# Canonical serialization + hashing +# ============================================================================== + + +def canonical_bytes(manifest: dict[str, Any]) -> bytes: + """Deterministic UTF-8 canonical JSON: sorted keys, no whitespace ambiguity. + + Uses ``json.dumps(..., sort_keys=True, ensure_ascii=False, + separators=(",", ":"))``. Two dicts that are equal (regardless of key + insertion order, since Python dict equality ignores order) always + serialize to byte-identical output under this scheme. + """ + return json.dumps( + manifest, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + + +def digest_of(manifest: dict[str, Any]) -> str: + """SHA-256 hex digest over ``canonical_bytes(manifest)``.""" + return hashlib.sha256(canonical_bytes(manifest)).hexdigest() + + +# ============================================================================== +# Create-only storage beside the trace directory +# ============================================================================== + + +class ManifestAlreadyExistsError(FileExistsError): + """Raised when an attempt manifest write would overwrite an existing file.""" + + +def _manifest_paths(trace_id: str, checkpoint: str) -> tuple[Path, Path]: + trace_dir = Path(trace_store.get_trace_dir(trace_id)) + manifest_path = trace_dir / f"attempt_manifest.{checkpoint}.json" + digest_path = trace_dir / f"attempt_manifest.{checkpoint}.sha256" + return manifest_path, digest_path + + +def _create_only_write(path: Path, data: bytes) -> None: + """Writes ``data`` to ``path`` iff the path does not already exist. + + Uses ``O_CREAT | O_EXCL`` so the existence check and the write are one + atomic kernel operation — no TOCTOU window between a prior ``.exists()`` + check and the write itself. Raises :class:`ManifestAlreadyExistsError` if + the path already exists (races included: two concurrent writers targeting + the same path always leave exactly one winner and one raised error, never + a silent overwrite). + """ + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY + try: + fd = os.open(path, flags, 0o644) + except FileExistsError as e: + raise ManifestAlreadyExistsError( + f"{path} already exists; refusing to overwrite a prior attempt's record." + ) from e + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + except BaseException: + path.unlink(missing_ok=True) + raise + + +def store_attempt_manifest(manifest: dict[str, Any]) -> tuple[Path, str]: + """Writes the manifest JSON bytes + its digest beside the trace directory. + + Create-only under concurrency: both the manifest and digest files are + written via ``O_CREAT | O_EXCL`` (:func:`_create_only_write`), so a + concurrent writer targeting the same ``(trace_id, checkpoint)`` can never + silently overwrite an earlier attempt's stored manifest (requirement: + "never overwrite a prior attempt's manifest/trace file") — the loser of + the race raises :class:`ManifestAlreadyExistsError` instead. One file is + written per checkpoint (``launch`` / ``worker_start`` / ``termination``), + keyed by ``trace_id``+``checkpoint``, so all three checkpoints for one + attempt can coexist and each is independently immutable once written. + + Returns ``(manifest_path, digest_hex)``. + """ + trace_id = manifest["trace_id"] + checkpoint = manifest["checkpoint"] + manifest_path, digest_path = _manifest_paths(trace_id, checkpoint) + + manifest_path.parent.mkdir(parents=True, exist_ok=True) + payload = canonical_bytes(manifest) + digest = hashlib.sha256(payload).hexdigest() + + _create_only_write(manifest_path, payload) + try: + _create_only_write(digest_path, digest.encode("utf-8")) + except ManifestAlreadyExistsError: + # The manifest write above already won its race and is on disk and + # immutable; only the digest file lost its race. Leave both files as + # they are — the manifest write must not be undone (a concurrent + # reader may already observe it), and re-raising surfaces the + # conflict to the caller instead of masking it. + raise + + return manifest_path, digest + + +def read_stored_manifest(trace_id: str, checkpoint: str) -> tuple[bytes, str] | None: + """Reads back a previously stored manifest's exact bytes and digest, or None.""" + manifest_path, digest_path = _manifest_paths(trace_id, checkpoint) + if not manifest_path.exists() or not digest_path.exists(): + return None + return manifest_path.read_bytes(), digest_path.read_text(encoding="utf-8").strip() diff --git a/artemis/config/attempt_reconciliation.py b/artemis/config/attempt_reconciliation.py new file mode 100644 index 00000000..9fd8e15d --- /dev/null +++ b/artemis/config/attempt_reconciliation.py @@ -0,0 +1,583 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reconciles native ``llm_usage`` trace events against an attempt manifest, +and applies the Gate 1 batch accept/reject rule across one or more attempts. + +This module implements the *mechanism* only (Gate 1). It does not run or +claim to have run a qualification pilot; see +``artemis.config.attempt_manifest`` module docstring for the same caveat. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from typing import Any, Literal + +from artemis.config.attempt_manifest import TIER_MODELS, Tier, digest_of, read_stored_manifest +from artemis.config.attempt_usage_reader import read_llm_usage_events +from artemis.runtime import trace_store + +#: Machine-readable batch-reject reasons, verbatim per the Gate 1 spec. +RejectReason = Literal[ + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", +] + +REJECT_REASONS: tuple[RejectReason, ...] = ( + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", +) + +# Maps a usage-side node name (the `node` field a `@trace`-scoped call site +# actually records via CURRENT_NODE_NAME, see +# artemis.services.token_meter.record_llm_usage) to the manifest node name it +# should reconcile against, for the rare cases where the enclosing @trace +# scope's name legitimately differs from the LLMConfig field name the node +# resolves through. +# +# validator_pixel_safety_net is the one confirmed case today: +# artemis.agents.validator.validator._validate_action_precondition_pixel is +# traced as "safety_net_pixel_validation" (see task_tree.py's custom +# rendering, which depends on that exact trace name for unrelated UI +# purposes -- do not rename the @trace scope to "fix" this), but internally +# calls get_llm_fn(ctx, name="validator_pixel_safety_net"), which is the +# LLMConfig field / manifest node name. Without this alias, a real, +# correctly-identified LLM call reconciles as unmapped_call purely because +# the trace scope name and the manifest node name disagree. +_NODE_ALIASES: dict[str, str] = { + "safety_net_pixel_validation": "validator_pixel_safety_net", +} + + +@dataclasses.dataclass(frozen=True) +class NodeReconciliation: + """Reconciliation verdict for one manifest node against usage events.""" + + node: str + manifest_enabled: bool + manifest_source: str | None # "provider:model" the manifest expects, if enabled + invoked: bool + usage_sources: tuple[str, ...] # distinct `source` values seen for this node + verdict: Literal[ + "match", # invoked, source matches manifest + "mismatch", # invoked, source present but does not match manifest + "unverified_identity", # invoked, but usage event(s) missing `source` + "not_invoked", # enabled in manifest, no usage event this attempt (informational) + "disabled_not_invoked", # explicitly disabled in manifest, and never invoked (expected) + "unmapped_call", # invoked, but node absent from manifest entirely + ] + + +@dataclasses.dataclass(frozen=True) +class ReconciliationResult: + """Full reconciliation outcome for one attempt: per-node verdicts + flags.""" + + attempt_id: str + nodes: tuple[NodeReconciliation, ...] + + @property + def has_mismatch(self) -> bool: + return any(n.verdict == "mismatch" for n in self.nodes) + + @property + def has_unverified_identity(self) -> bool: + return any(n.verdict == "unverified_identity" for n in self.nodes) + + @property + def has_unmapped_call(self) -> bool: + return any(n.verdict == "unmapped_call" for n in self.nodes) + + +def _tier_of_provider_model(provider: str | None, model: str | None) -> Tier | None: + """Which declared tier (if any) a raw ``(provider, model)`` pair matches. + + Dict-based counterpart of ``attempt_manifest._node_tier`` (which takes an + ``LLM``/``LLMWithFallback`` instance): this module only ever sees the + already-serialized ``config``/``would_resolve_to`` dicts a stored manifest + carries, never the original ``LLM`` objects, so it re-does the same + ``TIER_MODELS`` lookup directly against those two plain strings. + """ + for tier, (tier_provider, tier_model) in TIER_MODELS.items(): + if provider == tier_provider and model == tier_model: + return tier + return None + + +def _manifest_expected_source(node_entry: dict[str, Any], manifest_tier: Tier | None) -> str | None: + """The `provider:model` string a manifest node is expected to produce as `source`. + + For an enabled node this is its own ``config``. For a *disabled* + (soft-defaulted) node -- e.g. ``validator_pixel_safety_net`` / + ``planner_validation`` left unset in ``LLMConfig``, which + ``LLMConfig.get_agent()`` actually resolves to + ``lightweight_judge_default()`` (or an inherited node) when invoked, see + ``attempt_manifest._build_node_entry`` -- the expected source is derived + from ``would_resolve_to`` instead, since that is what the node will + genuinely produce if it fires. A disabled node with no ``would_resolve_to`` + (no soft default applies; it is simply off) still returns ``None``: an + unexpected receipt from an entirely-disabled node must still surface as a + problem, not be silently accepted. + + Tier pinning applies to soft-defaulted nodes exactly as it already applies + to enabled ones: an attempt manifest built for ``tier="sol"`` asserts that + *every* model-bearing node this attempt can invoke -- including a node + that only fires via its soft default, e.g. ``validator_pixel_safety_net`` + -- resolves within the Sol tier. ``would_resolve_to`` is a fixed function + of the node (``lightweight_judge_default()`` or an inherited sibling + node), not of ``manifest_tier``, so when its provider/model maps to a + *different* declared tier than ``manifest_tier``, trusting it as the + "expected" source would let an attempt pinned to one tier silently accept + a receipt from another tier's model -- exactly the cross-tier mixing + Gate 1 exists to catch. In that case this returns ``None`` (not the + would-resolve-to string), which routes the caller's mismatch/match branch + in ``reconcile_attempt`` to ``mismatch`` for any receipt, and to + ``disabled_not_invoked`` if the node never actually fires -- never to a + silent ``match``. A would-resolve-to whose provider/model maps to no + declared tier at all is treated the same conservative way, for the same + reason ``untiered_enabled_nodes`` is never silently trusted for enabled + nodes (see ``attempt_manifest.build_attempt_manifest`` / + ``validate_batch``'s ``missing_identity`` check): an unpinned model + identity is never assumed safe. + """ + config = node_entry.get("config") + if node_entry.get("enabled"): + config = config or {} + else: + config = node_entry.get("would_resolve_to") or None + if config is None: + return None + would_resolve_tier = _tier_of_provider_model(config.get("provider"), config.get("model")) + if would_resolve_tier != manifest_tier: + return None + provider = config.get("provider") + model = config.get("model") + if provider is None or model is None: + return None + return f"{provider}:{model}" + + +def reconcile_attempt( + attempt_id: str, + manifest: dict[str, Any], + usage_events: list[dict[str, Any]], +) -> ReconciliationResult: + """Reconciles one attempt's manifest against its raw ``llm_usage`` payloads. + + ``usage_events`` are the dicts produced by + ``artemis.services.token_meter.record_llm_usage`` (carrying at least + ``node``; ``source`` is only present when the call site passed one — see + ``RobustChatModelWrapper._endpoint_key()`` for the ``provider:model`` + shape, and the ``lens:*`` labels used by raw-model bypass call sites in + ``memory/chunking.py`` / ``agents/flash/summarizer.py``, which are + reported as-is and will not match any manifest node's ``provider:model`` + expectation — they surface as ``mismatch`` rather than being silently + accepted, since Gate 1 requires every invoked identity to be provable, + not merely present). + + Never fabricates a usage record for a node that did not fire: nodes with + no usage event are reported as ``not_invoked``/``disabled_not_invoked``, + not synthesized as zero-usage matches. + """ + manifest_nodes: dict[str, dict[str, Any]] = { + **manifest.get("nodes", {}), + **manifest.get("utils", {}), + } + + events_by_node: dict[str, list[dict[str, Any]]] = {} + for event in usage_events: + node = event.get("node") + if node is None: + # A usage event with no node context at all cannot be attributed + # to any manifest entry; treat its node key as the literal + # sentinel so it still surfaces as unmapped rather than being + # dropped silently. + node = "" + node = str(node) + # Resolve to the manifest node name when the usage-side node is a + # known trace-scope alias (see _NODE_ALIASES); otherwise group under + # the raw node name as before. + resolved_node = _NODE_ALIASES.get(node, node) + events_by_node.setdefault(resolved_node, []).append(event) + + manifest_tier: Tier | None = manifest.get("tier") + + results: list[NodeReconciliation] = [] + seen_nodes: set[str] = set() + + for node, entry in manifest_nodes.items(): + seen_nodes.add(node) + expected_source = _manifest_expected_source(entry, manifest_tier) + events = events_by_node.get(node, []) + if not events: + verdict = "not_invoked" if entry.get("enabled") else "disabled_not_invoked" + results.append( + NodeReconciliation( + node=node, + manifest_enabled=bool(entry.get("enabled")), + manifest_source=expected_source, + invoked=False, + usage_sources=(), + verdict=verdict, + ) + ) + continue + + sources = tuple(e.get("source") for e in events) + if any(s is None for s in sources): + verdict = "unverified_identity" + elif expected_source is not None and any(s != expected_source for s in sources): + verdict = "mismatch" + elif expected_source is None: + # Node fired but the manifest has no resolvable provider:model to + # expect for it -- either an enabled entry missing config, a + # disabled node with no soft default, or a soft-defaulted disabled + # node whose `would_resolve_to` tier disagrees with (or is + # entirely unmapped from) the manifest's own declared `tier` (see + # `_manifest_expected_source`). A soft-defaulted disabled node + # (e.g. validator_pixel_safety_net left unset) only has a non-None + # `expected_source` here when its `would_resolve_to` tier matches + # this attempt's own tier -- only that case is *not* covered by + # this branch and can legitimately reach "match" below. Surface + # this branch as mismatch rather than pretending an entirely- or + # cross-tier-defaulted node firing is fine. + verdict = "mismatch" + else: + verdict = "match" + + results.append( + NodeReconciliation( + node=node, + manifest_enabled=bool(entry.get("enabled")), + manifest_source=expected_source, + invoked=True, + usage_sources=tuple(s for s in sources if s is not None), + verdict=verdict, + ) + ) + + # Usage events for nodes the manifest never declared at all (e.g. a + # renamed node, a lens/utility label, or an entirely unmapped call site). + for node, events in events_by_node.items(): + if node in seen_nodes: + continue + sources = tuple(e.get("source") for e in events if e.get("source") is not None) + results.append( + NodeReconciliation( + node=node, + manifest_enabled=False, + manifest_source=None, + invoked=True, + usage_sources=sources, + verdict="unmapped_call", + ) + ) + + return ReconciliationResult(attempt_id=attempt_id, nodes=tuple(results)) + + +# ============================================================================== +# Batch validation / rejection rule +# ============================================================================== + + +@dataclasses.dataclass(frozen=True) +class AttemptRecord: + """One attempt's manifest + reconciliation, as handed to batch validation.""" + + attempt_id: str + manifest: dict[str, Any] + reconciliation: ReconciliationResult + # Digest computed at build/store time (from attempt_manifest.digest_of), + # supplied by the caller so batch validation can detect drift between + # what was stored and what is being validated now, without recomputing + # trust in a possibly-tampered manifest dict. + stored_digest: str | None = None + + +@dataclasses.dataclass(frozen=True) +class BatchVerdict: + """ACCEPT, or REJECT with a specific machine-readable reason. + + On reject, ``invalid_attempts`` preserves the exact records that failed + (manifest + reconciliation unmodified) so a caller can retain them for + audit rather than discarding them from totals. + """ + + accepted: bool + reason: RejectReason | None + detail: str + invalid_attempts: tuple[AttemptRecord, ...] = () + + +def _reject(reason: RejectReason, detail: str, invalid: tuple[AttemptRecord, ...]) -> BatchVerdict: + return BatchVerdict(accepted=False, reason=reason, detail=detail, invalid_attempts=invalid) + + +def validate_batch(attempts: list[AttemptRecord]) -> BatchVerdict: + """Validates one homogeneous batch of attempts (e.g. one journey x device cell). + + Checks run in a fixed order so the first violation found is reported; + every check preserves the offending attempt record(s) unmodified in + ``invalid_attempts`` rather than discarding them. An empty batch is + reported as ``missing_snapshot`` (nothing to validate is itself a defect + for a caller expecting N repeats). + """ + if not attempts: + return _reject("missing_snapshot", "Batch is empty: no attempt manifests supplied.", ()) + + # 1. missing_identity: any attempt whose manifest lacks a usable + # source_sha, whose fake_llm flag/tier is absent entirely, whose + # source_sha was never actually verified against a real git checkout + # (source_sha_provenance != "git" -- the "0"*40/"unknown-not-a-git- + # checkout" fallback sentinel is truthy but not a provable identity), + # or that has any enabled node resolving to no declared tier at all + # (untiered_enabled_nodes non-empty -- a model-bearing node that isn't + # pinned to a declared tier). + missing_identity = [ + a + for a in attempts + if not a.manifest.get("source_sha") + or a.manifest.get("tier") not in TIER_MODELS + or a.manifest.get("fake_llm_enabled") is None + or a.manifest.get("source_sha_provenance") != "git" + or a.manifest.get("untiered_enabled_nodes") + ] + if missing_identity: + return _reject( + "missing_identity", + f"{len(missing_identity)} attempt(s) have an incomplete manifest identity " + "(missing source_sha, tier, or fake_llm_enabled; an unverified/non-git " + "source_sha; or an enabled node with no declared tier).", + tuple(missing_identity), + ) + + # 2. fake_mode_enabled: any attempt that ran with ARTEMIS_FAKE_LLM=1. + fake_enabled = [a for a in attempts if a.manifest.get("fake_llm_enabled")] + if fake_enabled: + env_var = fake_enabled[0].manifest.get("fake_llm_env_var", "ARTEMIS_FAKE_LLM") + return _reject( + "fake_mode_enabled", + f"{len(fake_enabled)} attempt(s) ran with fake-LLM mode enabled ({env_var}=1).", + tuple(fake_enabled), + ) + + # 3. missing_snapshot: any attempt with no stored digest to compare against. + missing_snapshot = [a for a in attempts if not a.stored_digest] + if missing_snapshot: + return _reject( + "missing_snapshot", + f"{len(missing_snapshot)} attempt(s) have no stored manifest digest to verify.", + tuple(missing_snapshot), + ) + + # 4. digest_drift: recomputed digest of the manifest dict no longer + # matches the digest recorded at store time. + drifted = [a for a in attempts if digest_of(a.manifest) != a.stored_digest] + if drifted: + return _reject( + "digest_drift", + f"{len(drifted)} attempt(s) have a manifest whose recomputed digest no longer " + "matches its stored digest (possible post-hoc mutation).", + tuple(drifted), + ) + + # 5. mixed_tier: not every attempt in the batch declares the same tier. + tiers = {a.manifest.get("tier") for a in attempts} + if len(tiers) > 1: + return _reject( + "mixed_tier", + f"Batch mixes tiers across attempts: {sorted(t for t in tiers if t)}.", + tuple(attempts), + ) + + # 6. unmapped_call / unexpected_model_or_endpoint: derived from each + # attempt's reconciliation result. + unmapped = [a for a in attempts if a.reconciliation.has_unmapped_call] + if unmapped: + return _reject( + "unmapped_call", + f"{len(unmapped)} attempt(s) invoked a node absent from their manifest.", + tuple(unmapped), + ) + + unexpected = [ + a + for a in attempts + if a.reconciliation.has_mismatch or a.reconciliation.has_unverified_identity + ] + if unexpected: + return _reject( + "unexpected_model_or_endpoint", + f"{len(unexpected)} attempt(s) invoked a node whose observed `source` does not " + "match the manifest, or is missing entirely (unverified identity).", + tuple(unexpected), + ) + + return BatchVerdict(accepted=True, reason=None, detail=f"{len(attempts)} attempt(s) accepted.") + + +# ============================================================================== +# Post-run orchestration: stored manifest + native usage receipts -> verdict +# ============================================================================== + + +def reconcile_finished_attempt( + *, + trace_id: str, + session_id: str, + checkpoint: str, + db_path: str | Path, + traces_dir: str | Path, +) -> BatchVerdict | None: + """Reconciles one finished attempt against its own stored manifest + DB receipts. + + Reads back the manifest :func:`artemis.config.attempt_manifest.store_attempt_manifest` + wrote for ``(trace_id, checkpoint)``, reads every native ``llm_usage`` row + the DataEngine recorded for ``session_id`` (via + :func:`artemis.config.attempt_usage_reader.read_llm_usage_events`, never + mutated or summarized), reconciles them, and runs the single-attempt batch + rule. Returns ``None`` (not a verdict) when no manifest was stored for + this checkpoint — that is a Gate 1 wiring gap to fix, not something to + misrepresent as a passing or failing verdict. + + This is intentionally a one-attempt batch: multi-attempt batch validation + (e.g. a five-repeat journey/device cell) is the qualification pilot's own + concern, not something this per-run hook can see on its own. + + Because this always validates a batch of exactly one attempt, the + ``mixed_tier`` rejection path is not reachable from this function; see + :func:`reconcile_attempt_batch_by_run_id` for the production-reachable + multi-attempt path. + """ + stored = read_stored_manifest(trace_id, checkpoint) + if stored is None: + return None + manifest_bytes, stored_digest = stored + manifest = json.loads(manifest_bytes.decode("utf-8")) + + usage_events = read_llm_usage_events(db_path, traces_dir, session_id) + reconciliation = reconcile_attempt(trace_id, manifest, usage_events) + record = AttemptRecord( + attempt_id=trace_id, + manifest=manifest, + reconciliation=reconciliation, + stored_digest=stored_digest, + ) + return validate_batch([record]) + + +def reconcile_attempt_batch_by_run_id( + *, + run_id: str, + checkpoint: str, + db_path: str | Path, + traces_dir: str | Path, + traces_root: str | Path | None = None, +) -> BatchVerdict: + """Reconciles every stored attempt sharing ``run_id`` as one batch. + + This is the production-reachable home for cross-attempt validation + (including the ``mixed_tier`` rejection, which :func:`reconcile_finished_attempt` + can never exercise since it always validates a batch of exactly one + attempt). ``mcp_server.background.task_runner.run_task`` accepts an + optional ``run_id`` parameter (threaded through to + ``_record_attempt_manifest``'s ``run_id=`` argument) that a caller can set + to a value shared across multiple real ``run_task`` invocations, so the + grouping key this function reads (each stored manifest's own ``"run_id"`` + field) is real, settable plumbing today, not synthetic test-only state. + Today's default remains unchanged when ``run_id`` is not passed: + ``run_id=trace_id`` 1:1, so an unmodified caller still produces one + attempt manifest per ``run_id`` and this function degenerates to a + one-attempt batch for it. No caller in this codebase actually invokes + ``run_task`` more than once with a shared ``run_id`` yet -- there is no + automatic retry orchestrator wired up, and building one is explicitly out + of scope here. This function exists as a real, tested, importable, + general-purpose multi-attempt reconciler that a future caller (e.g. a + qualification-pilot orchestrator running repeated attempts of one + journey/device cell under a shared ``run_id``) can call using the + plumbing that already exists, without needing to add any. + + Discovers candidate attempts by listing the immediate subdirectories of + ``traces_root`` (default: :data:`artemis.runtime.trace_store.TRACES_DIR`) + -- each subdirectory name is a ``trace_id`` + (:func:`artemis.runtime.trace_store.get_trace_dir`) -- reading each one's + stored manifest at ``checkpoint`` via + :func:`artemis.config.attempt_manifest.read_stored_manifest`, and keeping + only the attempts whose manifest ``"run_id"`` field equals the requested + ``run_id``. For each matching attempt, native ``llm_usage`` events are + read via :func:`artemis.config.attempt_usage_reader.read_llm_usage_events` + using the attempt's own ``trace_id`` as ``session_id`` (matching the + ``session_id=trace_id`` convention ``_record_attempt_manifest`` / + ``Agent(config=config, session_id=trace_id)`` already use), reconciled, + and assembled into an :class:`AttemptRecord` exactly as + :func:`reconcile_finished_attempt` does per-attempt. + + Read-only: never writes, mutates, or deletes any manifest or trace file. + A directory whose stored manifest cannot be parsed as JSON is skipped + rather than aborting the whole scan -- one corrupt/unreadable record must + not hide every other attempt's evidence, mirroring + ``read_llm_usage_events``'s own skip-corrupt-rows behavior. + + ``validate_batch`` is called over the full collected list, which may be + empty (no attempt manifests found for ``run_id``); an empty batch + correctly hits ``validate_batch``'s existing ``missing_snapshot`` + rejection rather than being special-cased here. + """ + root = Path(traces_root) if traces_root is not None else Path(trace_store.TRACES_DIR) + + records: list[AttemptRecord] = [] + if root.is_dir(): + for trace_dir in sorted(root.iterdir()): + if not trace_dir.is_dir(): + continue + trace_id = trace_dir.name + + stored = read_stored_manifest(trace_id, checkpoint) + if stored is None: + continue + manifest_bytes, stored_digest = stored + try: + manifest = json.loads(manifest_bytes.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + # A corrupt stored manifest for one attempt must not abort + # discovery of every other attempt sharing this run_id. + continue + if manifest.get("run_id") != run_id: + continue + + usage_events = read_llm_usage_events(db_path, traces_dir, trace_id) + reconciliation = reconcile_attempt(trace_id, manifest, usage_events) + records.append( + AttemptRecord( + attempt_id=trace_id, + manifest=manifest, + reconciliation=reconciliation, + stored_digest=stored_digest, + ) + ) + + return validate_batch(records) diff --git a/artemis/config/attempt_usage_reader.py b/artemis/config/attempt_usage_reader.py new file mode 100644 index 00000000..f54c9907 --- /dev/null +++ b/artemis/config/attempt_usage_reader.py @@ -0,0 +1,83 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only access to a session's native ``llm_usage`` trace receipts. + +Gate 1 reconciliation (:mod:`artemis.config.attempt_reconciliation`) needs the +raw ``llm_usage`` payloads :func:`artemis.services.token_meter.record_llm_usage` +already writes into the DataEngine SQLite store during a live run. That store +only exposes trace lookup scoped to a step +(:meth:`artemis.data_engine.storage.StorageManager.get_steps_with_traces`), +but ``llm_usage`` traces are not guaranteed to carry a ``step_id`` (background +lens/utility calls may record with ``step_id=None``); walking steps would +silently miss those. This module queries the ``traces`` table directly by +``session_id``, following the same read-only-offline-reader pattern already +established by :class:`artemis.data_engine.history_reader.OfflineHistoryReader` +(``StorageManager(db_path, traces_dir, read_only=True)``), without touching +any write path or the live in-process engine. + +Never mutates, summarizes, or drops rows: every ``llm_usage`` payload found +for the session is returned as-is, so reconciliation always sees the original +receipts rather than a derived view. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + +from artemis.data_engine.storage import StorageManager + + +def read_llm_usage_events( + db_path: str | Path, + traces_dir: str | Path, + session_id: str, +) -> list[dict[str, Any]]: + """Returns every stored ``llm_usage`` trace payload for ``session_id``. + + Read-only: opens the DataEngine database with ``read_only=True`` (refuses + to create a missing database or table, matching + :class:`~artemis.data_engine.history_reader.OfflineHistoryReader`). Rows + with unparsable JSON payloads are skipped rather than raising, since a + single corrupt trace row must not hide every other attempt's evidence; + the caller (reconciliation) already treats a missing/absent event as + ``not_invoked``/``unverified_identity`` rather than inferring identity. + + Returns an empty list, never raises, if the database does not exist yet + (e.g. a session that recorded no traces at all). + """ + resolved_db_path = Path(db_path) + if not resolved_db_path.exists(): + return [] + + storage = StorageManager(resolved_db_path, traces_dir, read_only=True) + events: list[dict[str, Any]] = [] + with storage._get_connection() as conn: + cursor = conn.execute( + "SELECT payload FROM traces WHERE session_id = ? AND name = 'llm_usage' " + "ORDER BY timestamp ASC", + (str(session_id),), + ) + for row in cursor.fetchall(): + raw_payload = row["payload"] + if not raw_payload: + continue + try: + events.append(json.loads(raw_payload)) + except (json.JSONDecodeError, TypeError): + continue + return events diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 972bc6df..d0b768e8 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -62,8 +62,16 @@ ENV_OPENAI_API_KEY = "OPENAI_API_KEY" ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" +ENV_ANTHROPIC_BASE_URL = "ANTHROPIC_BASE_URL" +ENV_GOOGLE_BASE_URL = "GOOGLE_BASE_URL" +ENV_VERTEX_AI_BASE_URL = "VERTEX_AI_BASE_URL" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" +ENV_OPEN_ROUTER_BASE_URL = "OPEN_ROUTER_BASE_URL" ENV_XAI_API_KEY = "XAI_API_KEY" +ENV_XAI_BASE_URL = "XAI_BASE_URL" +ENV_OLLAMA_BASE_URL = "OLLAMA_BASE_URL" +ENV_VLLM_BASE_URL = "VLLM_BASE_URL" +ENV_CUSTOM_BASE_URL = "CUSTOM_BASE_URL" # Explorer & Agent Defaults ENV_ARTEMIS_EXPLORER_VERSION = "ARTEMIS_EXPLORER_VERSION" diff --git a/artemis/config/llm.py b/artemis/config/llm.py index 36915dd9..1f2faaef 100644 --- a/artemis/config/llm.py +++ b/artemis/config/llm.py @@ -74,6 +74,7 @@ class LLM(BaseModel): model_config = {"ignored_types": (CyFunctionDetector,)} provider: LLMProvider model: str + api_base: str | None = None temperature: float | None = None thinking_budget: int | None = None thinking_level: Literal["minimal", "low", "medium", "high"] | None = None diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 7129dd8a..5c0fe393 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -107,7 +107,15 @@ class Settings(BaseSettings): API_KEY: SecretStr | None = None # Custom Provider Endpoints + GOOGLE_BASE_URL: str | None = None + VERTEX_AI_BASE_URL: str | None = None OPENAI_BASE_URL: str | None = None + ANTHROPIC_BASE_URL: str | None = None + OPEN_ROUTER_BASE_URL: str | None = None + XAI_BASE_URL: str | None = None + OLLAMA_BASE_URL: str | None = None + VLLM_BASE_URL: str | None = None + CUSTOM_BASE_URL: str | None = None # Android ADB Connectivity ADB_HOST: str | None = Field(default=DEFAULT_ADB_HOST) diff --git a/artemis/core/diagnostics/probes/credentials_probe.py b/artemis/core/diagnostics/probes/credentials_probe.py index 683e216c..eb374bda 100644 --- a/artemis/core/diagnostics/probes/credentials_probe.py +++ b/artemis/core/diagnostics/probes/credentials_probe.py @@ -82,6 +82,7 @@ async def probe(self) -> ProbeResult: "masked": self._mask_key(o_val), "raw_key": o_val, "key": o_val, + "base_url": settings.OPENAI_BASE_URL, } ) api_keys_map["openai"] = o_val @@ -94,6 +95,7 @@ async def probe(self) -> ProbeResult: "masked": self._mask_key(c_val), "raw_key": c_val, "key": c_val, + "base_url": settings.ANTHROPIC_BASE_URL, } ) api_keys_map["anthropic"] = c_val @@ -126,7 +128,6 @@ async def probe(self) -> ProbeResult: for env_var, label, prov_id in [ ("DEEPSEEK_API_KEY", "DeepSeek", "deepseek"), ("GROQ_API_KEY", "Groq", "groq"), - ("OPENAI_BASE_URL", "Custom OpenAI Endpoint", "custom"), ("OLLAMA_BASE_URL", "Local Ollama", "ollama"), ("VLLM_BASE_URL", "vLLM Endpoint", "vllm"), ("VERTEX_AI_PROJECT", "Google Cloud Vertex AI", "vertexai"), @@ -144,6 +145,26 @@ async def probe(self) -> ProbeResult: ) api_keys_map[prov_id] = val.strip() + # Preserve endpoint-only diagnosis when a compatible URL exists without + # its provider key; keyed providers are verified against their own URL above. + openai_base_url = os.environ.get("OPENAI_BASE_URL") + if ( + openai_base_url + and openai_base_url.strip() + and not is_placeholder_key(openai_base_url.strip()) + and "openai" not in api_keys_map + ): + configured_providers.append( + { + "provider": "custom", + "label": "Custom OpenAI Endpoint", + "masked": self._mask_key(openai_base_url.strip()), + "raw_key": openai_base_url.strip(), + "key": openai_base_url.strip(), + } + ) + api_keys_map["custom"] = openai_base_url.strip() + if ocr_key and not is_placeholder_key(ocr_key): api_keys_map["ocr"] = ocr_key.get_secret_value() diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 91a5d044..59ebb24d 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -16,6 +16,7 @@ import asyncio import os +import uuid from pathlib import Path from shutil import which from typing import Annotated @@ -23,6 +24,11 @@ from adbutils import AdbClient from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, initialize_llm_config, settings +from artemis.config.attempt_lifecycle_hooks import ( + record_attempt_manifest, + reconcile_and_store_verdict, +) +from artemis.runtime import trace_store from artemis.utils.startup_progress import publish_startup_progress from artemis import Agent, Builders from artemis.sdk.types.task import AgentProfile @@ -59,6 +65,7 @@ async def execute_task( explorer_flash_mode: str | None = None, explorer_pro_mode: str | None = None, verification_level: str | None = None, + run_id: str | None = None, ) -> None: """Executes a single mobile automation task end-to-end. @@ -79,110 +86,159 @@ async def execute_task( explorer_pro_mode: Override the Explorer tier for the Pro execution profile. verification_level: Coarse Checker preset ('off', 'final', 'checkpoints', 'strict'); applied before the explicit ``enable_checker`` switch. + run_id: Gate 1 batch-grouping key for this attempt's manifest (see + ``artemis.config.attempt_lifecycle_hooks``); defaults to the + effective session id when omitted. """ effective_sid = ( - session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + session_id + or os.getenv("ARTEMIS_SESSION_ID") + or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + or str(uuid.uuid4()) ) - if effective_sid: - os.environ["ARTEMIS_SESSION_ID"] = str(effective_sid) + # ARTEMIS_SESSION_ID is a process-global env var, but effective_sid is a + # per-invocation identity -- an explicit session_id arg (or a generated + # fallback) must never bleed into a later execute_task() call in the same + # process. Snapshot whatever was there before this call and restore it + # (or delete the key if it wasn't set) in the finally below, which covers + # both the initialize_llm_config() failure path and the main run. + _prior_session_id_env = os.environ.get("ARTEMIS_SESSION_ID") + os.environ["ARTEMIS_SESSION_ID"] = str(effective_sid) if not os.environ.get("ARTEMIS_TASK_INGRESS"): os.environ["ARTEMIS_TASK_INGRESS"] = "cli" - publish_startup_progress( - "configuration", - "Loading the run configuration", - session_id=str(effective_sid) if effective_sid else None, - ) + try: + publish_startup_progress( + "configuration", + "Loading the run configuration", + session_id=str(effective_sid) if effective_sid else None, + ) - llm_config = initialize_llm_config() - agent_profile = AgentProfile(name="default", llm_config=llm_config) - config = Builders.AgentConfig.with_default_profile(profile=agent_profile) + try: + llm_config = initialize_llm_config() + except Exception as exc: + if effective_sid: + try: + trace_store.update_trace_status(str(effective_sid), "failed", error=str(exc)) + except OSError: + logger.exception( + "Could not record configuration failure for session %s", effective_sid + ) + raise + agent_profile = AgentProfile(name="default", llm_config=llm_config) + config = Builders.AgentConfig.with_default_profile(profile=agent_profile) - if video_recording_tools_enabled is not None: - config.with_video_recording_tools(enabled=video_recording_tools_enabled) + if video_recording_tools_enabled is not None: + config.with_video_recording_tools(enabled=video_recording_tools_enabled) - if enable_planner_validation is not None: - config.with_planner_validation(enabled=enable_planner_validation) + if enable_planner_validation is not None: + config.with_planner_validation(enabled=enable_planner_validation) - if enable_committee is not None: - config.with_committee(enabled=enable_committee) + if enable_committee is not None: + config.with_committee(enabled=enable_committee) - if verification_level is not None: - config.with_checker(**checker_overrides_for_level(verification_level)) + if verification_level is not None: + config.with_checker(**checker_overrides_for_level(verification_level)) - if enable_checker is not None: - config.with_checker(enabled=enable_checker) + if enable_checker is not None: + config.with_checker(enabled=enable_checker) - if enable_step_summarizer is not None: - config.with_flash_step_summarizer(enabled=enable_step_summarizer) + if enable_step_summarizer is not None: + config.with_flash_step_summarizer(enabled=enable_step_summarizer) - if enable_outputter is not None or force_output_synthesis is not None: - config.with_outputter( - enabled=enable_outputter if enable_outputter is not None else True, - force_synthesis=bool(force_output_synthesis), - ) + if enable_outputter is not None or force_output_synthesis is not None: + config.with_outputter( + enabled=enable_outputter if enable_outputter is not None else True, + force_synthesis=bool(force_output_synthesis), + ) - if ( - explorer_version is not None - or explorer_flash_mode is not None - or explorer_pro_mode is not None - ): - config.with_explorer( - version=explorer_version, - flash_mode=explorer_flash_mode, - pro_mode=explorer_pro_mode, - ) + if ( + explorer_version is not None + or explorer_flash_mode is not None + or explorer_pro_mode is not None + ): + config.with_explorer( + version=explorer_version, + flash_mode=explorer_flash_mode, + pro_mode=explorer_pro_mode, + ) - if settings.ADB_HOST: - config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) + if settings.ADB_HOST: + config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) - target_serial = ( - device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") - ) - if not target_serial: - try: - from artemis.runtime import device_pool + target_serial = ( + device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") + ) + if not target_serial: + try: + from artemis.runtime import device_pool - target_serial = device_pool.select_device() - except Exception: - target_serial = None + target_serial = device_pool.select_device() + except Exception: + target_serial = None - if target_serial: - from artemis.context import DevicePlatform + if target_serial: + from artemis.context import DevicePlatform - config.for_device(DevicePlatform.ANDROID, target_serial) + config.for_device(DevicePlatform.ANDROID, target_serial) - if graph_config_callbacks: - config.with_graph_config_callbacks(graph_config_callbacks) + if graph_config_callbacks: + config.with_graph_config_callbacks(graph_config_callbacks) - agent: Agent | None = None - try: - agent = Agent(config=config.build(), session_id=effective_sid) - await agent.init( - retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), - retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), - ) + agent: Agent | None = None + try: + record_attempt_manifest( + trace_id=str(effective_sid), + checkpoint="launch", + llm_config=llm_config, + run_id=run_id, + ) - task = agent.new_task(goal) - if locked_app_package: - task.with_locked_app_package(locked_app_package) - if test_name: - trace_path = traces_output_path_str or str(settings.TRACES_PATH) - task.with_name(test_name).with_trace_recording(path=trace_path) - if output_description: - task.with_output_description(output_description) - if profile: - task.using_profile(profile) - if app_path: - task.with_app_path(Path(app_path)) - - llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) - if llm_result_path: - task.with_llm_output_saving(path=llm_result_path) - - await agent.run_task(request=task.build()) + agent = Agent(config=config.build(), session_id=effective_sid) + await agent.init( + retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), + retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), + ) + + task = agent.new_task(goal) + if locked_app_package: + task.with_locked_app_package(locked_app_package) + if test_name: + trace_path = traces_output_path_str or str(settings.TRACES_PATH) + task.with_name(test_name).with_trace_recording(path=trace_path) + if output_description: + task.with_output_description(output_description) + if profile: + task.using_profile(profile) + if app_path: + task.with_app_path(Path(app_path)) + + llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) + if llm_result_path: + task.with_llm_output_saving(path=llm_result_path) + + await agent.run_task(request=task.build()) + finally: + # Reconciliation runs regardless of task outcome (success, failed + # status, exception, or cancellation) and before cleanup, so an + # Agent.clean() failure (e.g. device disconnect) can never + # suppress the reconciliation verdict -- mirrors + # mcp_server.background.task_runner's finally-block ordering. + reconcile_and_store_verdict( + trace_id=str(effective_sid), checkpoint="launch", run_id=run_id + ) + if agent is not None: + try: + await agent.clean() + except Exception as exc: + logger.error(f"Error cleaning agent: {exc}") finally: - if agent is not None: - await agent.clean() + # Restore whatever ARTEMIS_SESSION_ID held before this call so an + # explicit session_id (or a generated fallback) never bleeds into a + # later execute_task() call in the same process. + if _prior_session_id_env is None: + os.environ.pop("ARTEMIS_SESSION_ID", None) + else: + os.environ["ARTEMIS_SESSION_ID"] = _prior_session_id_env def run_command( @@ -337,6 +393,16 @@ def run_command( help="Canonical session UUID for trace and stream telemetry.", ), ] = None, + run_id: Annotated[ + str | None, + typer.Option( + "--run-id", + help=( + "Gate 1 batch-grouping key for this attempt's manifest " + "(defaults to the session id when omitted)." + ), + ), + ] = None, standalone: Annotated[ bool, typer.Option( @@ -385,6 +451,7 @@ def run_command( app_path=app_path, session_id=target_sid, ingress="cli", + run_id=run_id, base_url=base_url, ) if resp and resp.get("tasks"): @@ -488,6 +555,7 @@ def on_status(sess_info): explorer_flash_mode=explorer_flash_mode, explorer_pro_mode=explorer_pro_mode, verification_level=verification_level, + run_id=run_id, ) ) except (KeyboardInterrupt, asyncio.CancelledError): diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 8b44743b..24efbfff 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -22,6 +22,7 @@ from enum import StrEnum import hashlib import os +from types import SimpleNamespace from typing import Any from langchain_core.language_models.chat_models import BaseChatModel @@ -84,6 +85,90 @@ def from_string(cls, val: Any) -> "ModelProvider": return provider +_PROVIDER_BASE_URL_SETTINGS = { + ModelProvider.GOOGLE: "GOOGLE_BASE_URL", + ModelProvider.VERTEX_AI: "VERTEX_AI_BASE_URL", + ModelProvider.OPENAI: "OPENAI_BASE_URL", + ModelProvider.ANTHROPIC: "ANTHROPIC_BASE_URL", + ModelProvider.OPENROUTER: "OPEN_ROUTER_BASE_URL", + ModelProvider.XAI: "XAI_BASE_URL", + ModelProvider.OLLAMA: "OLLAMA_BASE_URL", + ModelProvider.VLLM: "VLLM_BASE_URL", + ModelProvider.CUSTOM: "CUSTOM_BASE_URL", +} + +_PROVIDER_DEFAULT_BASE_URLS = { + ModelProvider.OPENROUTER: "https://openrouter.ai/api/v1", + ModelProvider.XAI: "https://api.x.ai/v1", + ModelProvider.OLLAMA: "http://localhost:11434/v1", + ModelProvider.VLLM: "http://localhost:8000/v1", + ModelProvider.CUSTOM: "http://localhost:8000/v1", +} + +# Anthropic model name prefixes that hard-reject the `temperature` sampling +# parameter (400 invalid_request_error: "temperature is deprecated for this +# model"), starting with the generational (non-dated) naming scheme. Add new +# rejecting families here as Anthropic expands the deprecation. +_ANTHROPIC_TEMPERATURE_REJECTING_PREFIXES = ( + "claude-sonnet-5", + "claude-opus-5", + "claude-haiku-4-5", + "claude-fable-5", + "claude-opus-4-7", + "claude-opus-4-8", +) + + +def anthropic_rejects_temperature(model_name: Any) -> bool: + """Whether this Anthropic model 400s on any `temperature` value, including 0.0/1.0.""" + name = str(model_name).lower() + return name.startswith(_ANTHROPIC_TEMPERATURE_REJECTING_PREFIXES) + + +_anthropic_stream_event_patched = False + + +def _patch_anthropic_stream_context_management() -> None: + """Works around a langchain-anthropic bug that crashes every stream. + + ``ChatAnthropic._make_message_chunk_from_anthropic_event`` reads + ``event.context_management`` off `message_delta` stream events and calls + ``.model_dump()`` on it unconditionally. That attribute is declared on + `ChatAnthropic` itself as a plain ``dict`` (its own request-side memory/ + context-management config), not as a typed field on the Anthropic SDK's + `RawMessageDeltaEvent`. Because that event's Pydantic model allows extra + fields, an API response carrying a top-level `context_management` block + (a newer, non-beta feature) arrives as a raw `dict`, not a submodel, and + `.model_dump()` raises `AttributeError: 'dict' object has no attribute + 'model_dump'` mid-stream, discarding the whole response. Confirmed still + present on langchain-anthropic's latest release; upgrading does not fix + it. Idempotent — safe to call on every Anthropic model construction. + """ + global _anthropic_stream_event_patched + if _anthropic_stream_event_patched: + return + + from langchain_anthropic import ChatAnthropic + + original = ChatAnthropic._make_message_chunk_from_anthropic_event + + def _dumpable(value: Any) -> Any: + if isinstance(value, dict): + return SimpleNamespace(model_dump=lambda mode=None: value) + return value + + def _patched(self, event, **kwargs): + if getattr(event, "context_management", None) is not None: + event.context_management = _dumpable(event.context_management) + delta = getattr(event, "delta", None) + if delta is not None and getattr(delta, "container", None) is not None: + delta.container = _dumpable(delta.container) + return original(self, event, **kwargs) + + ChatAnthropic._make_message_chunk_from_anthropic_event = _patched + _anthropic_stream_event_patched = True + + class ModelEndpoint(BaseModel): """Configuration definition for an LLM/VLM model endpoint.""" @@ -133,6 +218,21 @@ def cache_key(self) -> tuple: ) +def resolve_provider_base_url(endpoint: ModelEndpoint) -> str | None: + """Return the endpoint URL with model, settings, then provider-default precedence.""" + + def usable_url(value: object) -> str | None: + value = str(value).strip() if value is not None else "" + return value or None + + provider = ModelProvider.from_string(endpoint.provider) + return ( + usable_url(endpoint.api_base) + or usable_url(getattr(settings, _PROVIDER_BASE_URL_SETTINGS[provider])) + or _PROVIDER_DEFAULT_BASE_URLS.get(provider) + ) + + def _patch_langchain_google_genai(): """Patches ChatGoogleGenerativeAI._process_tool_config to preserve and ensure include_server_side_tool_invocations.""" try: @@ -245,6 +345,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "temperature": endpoint.temperature, "max_output_tokens": endpoint.max_tokens, "api_key": api_key, + "base_url": resolve_provider_base_url(endpoint), "timeout": endpoint.timeout_seconds, "thinking_budget": endpoint.thinking_budget, "thinking_level": thinking_level, @@ -269,6 +370,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "model_name": endpoint.model_name, "temperature": endpoint.temperature, "max_output_tokens": endpoint.max_tokens, + "base_url": resolve_provider_base_url(endpoint), "timeout": endpoint.timeout_seconds, "thinking_budget": endpoint.thinking_budget, "safety_settings": { @@ -288,9 +390,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: or (settings.OPENAI_API_KEY.get_secret_value() if settings.OPENAI_API_KEY else None) or os.environ.get("OPENAI_API_KEY", "EMPTY") ) - base_url = endpoint.api_base or ( - str(settings.OPENAI_BASE_URL) if settings.OPENAI_BASE_URL else None - ) + base_url = resolve_provider_base_url(endpoint) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, @@ -306,6 +406,8 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: elif provider == ModelProvider.ANTHROPIC: from langchain_anthropic import ChatAnthropic + _patch_anthropic_stream_context_management() + api_key = ( endpoint.api_key or ( @@ -315,19 +417,24 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: ) or os.environ.get("ANTHROPIC_API_KEY") ) - kwargs = { + base_url = resolve_provider_base_url(endpoint) + rejects_temperature = anthropic_rejects_temperature(endpoint.model_name) + kwargs: dict[str, Any] = { "model": endpoint.model_name, - "temperature": endpoint.temperature, "api_key": api_key, + "base_url": base_url, "timeout": endpoint.timeout_seconds, } + if not rejects_temperature: + kwargs["temperature"] = endpoint.temperature budget = endpoint.thinking_budget if not budget and endpoint.reasoning_effort: effort_map = {"low": 2048, "medium": 8192, "high": 32768} budget = effort_map.get(endpoint.reasoning_effort.lower()) if budget: kwargs["thinking"] = {"type": "enabled", "budget_tokens": budget} - kwargs["temperature"] = 1.0 + if not rejects_temperature: + kwargs["temperature"] = 1.0 return ChatAnthropic(**{k: v for k, v in kwargs.items() if v is not None}) elif provider == ModelProvider.OPENROUTER: @@ -346,7 +453,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: model=endpoint.model_name, temperature=endpoint.temperature, api_key=api_key, - base_url=endpoint.api_base or "https://openrouter.ai/api/v1", + base_url=resolve_provider_base_url(endpoint), timeout=endpoint.timeout_seconds, ) @@ -362,7 +469,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: model=endpoint.model_name, temperature=endpoint.temperature, api_key=api_key, - base_url=endpoint.api_base or "https://api.x.ai/v1", + base_url=resolve_provider_base_url(endpoint), timeout=endpoint.timeout_seconds, ) @@ -370,9 +477,7 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: from langchain_openai import ChatOpenAI api_key = endpoint.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY") - base_url = endpoint.api_base or os.environ.get( - "OPENAI_BASE_URL", "http://localhost:8000/v1" - ) + base_url = resolve_provider_base_url(endpoint) kwargs = { "model": endpoint.model_name, "temperature": endpoint.temperature, diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py index 4e564540..ab8e332d 100644 --- a/artemis/memory/chunking.py +++ b/artemis/memory/chunking.py @@ -45,7 +45,6 @@ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage -from artemis.llm.google import is_google_provider from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService from artemis.memory.transcript import format_session_offset from artemis.utils.logger import get_logger @@ -323,17 +322,23 @@ def __init__( def _get_llm(self): if self._llm is None: - from artemis.services.llm import get_google_llm + from artemis.services.llm import get_llm - self._llm = get_google_llm(model_name=self._model_name, temperature=0.0) + self._llm = get_llm( + self._ctx, name="summarizer", temperature=0.0, model_name=self._model_name + ) return self._llm def _get_fallback_llm(self): if self._fallback_llm is None and self._fallback_model_name: - from artemis.services.llm import get_google_llm - - self._fallback_llm = get_google_llm( - model_name=self._fallback_model_name, temperature=0.0 + from artemis.services.llm import get_llm + + self._fallback_llm = get_llm( + self._ctx, + name="summarizer", + use_fallback=True, + temperature=0.0, + model_name=self._fallback_model_name, ) return self._fallback_llm @@ -952,9 +957,8 @@ def _build_capsule_service(self, ctx: Any) -> StepMemoryService: def _resolve_capsule_fallback_model(self, ctx: Any) -> str | None: """Fallback model for capsule generation when `chunking.model` is down. - Resolved from the LLM config's summarizer role (which inherits the - global default fallback unless overridden). Only same-provider (google) - fallbacks apply — the capsule lens rides the raw google model path. + Resolved from the LLM config's summarizer role, which inherits the + global default fallback unless overridden. """ try: llm_cfg = getattr(ctx, "llm_config", None) if ctx is not None else None @@ -963,9 +967,8 @@ def _resolve_capsule_fallback_model(self, ctx: Any) -> str | None: llm_cfg = get_default_llm_config() fallback = getattr(getattr(llm_cfg, "summarizer", None), "fallback", None) - provider = str(getattr(fallback, "provider", "") or "") model = getattr(fallback, "model", None) - if model and is_google_provider(provider) and model != self._model_name: + if model and model != self._model_name: return str(model) except Exception as exc: logger.debug(f"Capsule fallback model resolution skipped: {exc}", exc_info=True) diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 517d7384..d50b5d6e 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -9,6 +9,9 @@ "default": { "provider": "google", "model": "gemini-3.8-flash", + // For OpenAI/Anthropic models, optional per-model API root. OpenAI-compatible endpoints include /v1; + // Anthropic endpoints normally omit /v1 because the SDK appends it. + // "api_base": "https://openai-proxy.example/v1", "thinking_level": "medium", "fallback": { "provider": "google", diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 67736e65..3751c718 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -221,6 +221,7 @@ def submit_task_to_daemon( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: @@ -230,6 +231,10 @@ def submit_task_to_daemon( ``explorer_mode`` ('flash' | 'pro' | 'ultra') are the Pro-profile tuning knobs of ``/api/run``; they are forwarded verbatim and ignored by Flash. + ``run_id`` is the Gate 1 batch-grouping key for this attempt's manifest + (see ``artemis.config.attempt_lifecycle_hooks``); it defaults to the + attempt's own trace/session id when omitted. + Returns the response JSON dict if successfully enqueued, or None on error. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" @@ -246,6 +251,7 @@ def submit_task_to_daemon( "session_id": session_id, "ingress": ingress, "conversation_id": conversation_id, + "run_id": run_id, } try: diff --git a/artemis/services/llm.py b/artemis/services/llm.py index 784e971f..8449ad95 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -1074,6 +1074,7 @@ def _get_val(obj, attr, expected_type): reasoning_effort=_get_val(cfg, "reasoning_effort", str), include_thoughts=_get_val(cfg, "include_thoughts", bool), enable_grounding=_get_val(cfg, "enable_grounding", bool) or False, + api_base=_get_val(cfg, "api_base", str), ) @@ -1084,6 +1085,7 @@ def get_llm( *, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1094,6 +1096,7 @@ def get_llm( *, is_utils: Literal[True], temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1105,6 +1108,7 @@ def get_llm( is_utils: Literal[True], use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @@ -1114,11 +1118,14 @@ def get_llm( is_utils: bool = False, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: """Resolves and instantiates the appropriate LLM wrapper for the given agent role.""" endpoint = _resolve_endpoint(ctx, str(name), is_utils=is_utils, use_fallback=use_fallback) if temperature is not None: endpoint = endpoint.model_copy(update={"temperature": temperature}) + if model_name is not None: + endpoint = endpoint.model_copy(update={"model_name": model_name}) raw_model = ModelFactory.get_model(endpoint) handler = DataEngineCallbackHandler(ctx) diff --git a/artemis/utils/credentials_validator.py b/artemis/utils/credentials_validator.py index e71e8ba4..48c63d15 100644 --- a/artemis/utils/credentials_validator.py +++ b/artemis/utils/credentials_validator.py @@ -76,6 +76,14 @@ async def validate_api_key( clean_provider = provider.strip().lower() clean_key = api_key.strip() + if base_url is None: + from artemis.config import settings + + if clean_provider == "openai": + base_url = settings.OPENAI_BASE_URL + elif clean_provider in ("anthropic", "claude"): + base_url = settings.ANTHROPIC_BASE_URL + if not clean_key: return False, "API key cannot be empty." @@ -136,7 +144,11 @@ async def validate_api_key( return False, f"OpenAI API verification failed ({resp.status_code}): {err_msg}" elif clean_provider in ("anthropic", "claude"): - url = "https://api.anthropic.com/v1/models" + url = ( + f"{base_url.rstrip('/')}/v1/models" + if base_url + else "https://api.anthropic.com/v1/models" + ) resp = await client.get( url, headers={ diff --git a/artemis/utils/file.py b/artemis/utils/file.py index 7b8d97d6..27caf293 100644 --- a/artemis/utils/file.py +++ b/artemis/utils/file.py @@ -18,9 +18,13 @@ def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Strip JSONC comments while preserving quoted strings and escapes.""" + pattern = r'//.*?$|/\*.*?\*/|"(?:\\.|[^\\"])*"' + + def replace_comment(match: re.Match[str]) -> str: + return "" if match.group(0).startswith("/") else match.group(0) + + return re.sub(pattern, replace_comment, text, flags=re.DOTALL | re.MULTILINE) def load_jsonc(file: IO) -> dict: diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 517d7384..20b148be 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -7,13 +7,14 @@ // 1. Global Default Model (Applied to all sub-agents unless overridden below) // ------------------------------------------------------------------------------ "default": { - "provider": "google", - "model": "gemini-3.8-flash", - "thinking_level": "medium", + "provider": "openai", + "model": "gpt-5.6-sol", + // API roots come from OPENAI_BASE_URL or ANTHROPIC_BASE_URL in the environment. + "reasoning_effort": "medium", "fallback": { - "provider": "google", - "model": "gemini-3.7-flash", - "thinking_level": "medium" + "provider": "openai", + "model": "gpt-5.6-sol", + "reasoning_effort": "medium" } }, @@ -71,14 +72,12 @@ "video_analyzer": {}, // 👁️ Object Detector & Explorer (Flash Mode): Visual coordinates grounding and bounding boxes - // CRITICAL REQUIREMENT: Standard LLMs (including standard Gemini Flash, GPT-4o, Claude 3.7) lack sub-pixel spatial coordinate fine-tuning. - // To achieve accurate [x, y] point and bounding box localization, this node MUST use specialized Gemini ER (Embodied Reasoning / Robotics) models: - // e.g., "gemini-robotics-er-2-preview". Non-ER models will fail spatial coordinate detection. "object_detector": { - "model": "gemini-robotics-er-2-preview", + "provider": "openai", + "model": "gpt-5.6-sol", "fallback": { - "provider": "google", - "model": "gemini-robotics-er-2-preview" + "provider": "openai", + "model": "gpt-5.6-sol" } }, @@ -87,7 +86,7 @@ // - "flash" version delegates directly to object_detector (requires Gemini ER model above). // - "pro" / "ultra" versions run multi-turn ReAct loops (supports both Native Gemini and Universal Multi-Model LangChain engines). "explorer": { - "thinking_level": "medium" + "reasoning_effort": "medium" }, // 📋 Summarizer: Aggregates step history and generates structured outcome @@ -95,10 +94,11 @@ // 🔽 Hopper: Fast lightweight package name locator "hopper": { - "model": "gemini-3.5-flash-lite", + "provider": "openai", + "model": "gpt-5.6-sol", "fallback": { - "provider": "google", - "model": "gemini-3.1-flash-lite" + "provider": "openai", + "model": "gpt-5.6-sol" } }, @@ -139,13 +139,13 @@ // Execution: Observe-Think-Act single LLM reactive loop without graph overhead. "flash": { "max_turns": 0, // Maximum reactive turns before stopping (0 = unlimited; history is bounded by transcript compression, not by a turn cap) - "explorer_mode": "flash", // Explorer tier for the Flash runner: 'flash' | 'pro' | 'ultra' (see top-level "explorer") + "explorer_mode": "pro", // Explorer tier for the Flash runner: 'flash' | 'pro' | 'ultra' (see top-level "explorer") // 🧠 Step Summarizer: Asynchronous visual context compressor // Summarizes intermediate steps in background without blocking the runner, // replacing images only after strictly objective, high-density summaries are ready. "step_summarizer": { "enabled": true, // Whether background step summarization is active - "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference + "model": "gpt-5.6-sol", // Lightweight model for fast background inference "prune_history_xml": true // Prune outdated heavy XML trees from historical steps } }, @@ -158,7 +158,7 @@ "pro": { // 🧭 Explorer: Perception sub-agent under Pro profile ('flash', 'pro', or 'ultra') "explorer": { - "mode": "flash" // Explorer tier for Pro nodes (Operator/Validator): 'flash' | 'pro' | 'ultra' + "mode": "pro" // Explorer tier for Pro nodes (Operator/Validator): 'flash' | 'pro' | 'ultra' }, // 🎯 Planner Validation: Async verification when task plan milestones change "planner_validation": { @@ -246,7 +246,7 @@ // call each); milestone closes are exempt, the hard line waives it. "min_steps": 3, "target_source_tokens": 2000, - "model": "gemini-3.8-flash", + "model": "gpt-5.6-sol", "max_chunks": 8 // "max_eras": 8 // independent era cap; omitted = follows max_chunks }, diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index a6f377ce..0151048b 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -41,6 +41,12 @@ except Exception: load_dotenv(os.path.join(PROJECT_ROOT, ".env")) +from artemis.config.attempt_lifecycle_hooks import ( + record_attempt_manifest as _record_attempt_manifest, +) +from artemis.config.attempt_lifecycle_hooks import ( + reconcile_and_store_verdict as _reconcile_and_store_verdict, +) from artemis.runtime import trace_store from mcp_server.notifiers import notify from mcp_server.utils import device_utils @@ -121,6 +127,7 @@ async def run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_pro_mode: str | None = None, + run_id: str | None = None, ): """Executes the mobile automation agent task and logs all actions/results. @@ -128,6 +135,17 @@ async def run_task( ``explorer_pro_mode`` ('flash' | 'pro' | 'ultra') are Pro-profile tuning knobs mirroring ``artemis run --verification-level / --explorer-pro-mode``; the Flash profile ignores them. + + ``run_id`` is plumbing only: when not given, the attempt manifests this + call records default to ``run_id=trace_id`` (today's exact behavior, and + still the case for every real caller today -- see + ``mcp_server.background.task_runner`` module usage). No caller in this + codebase currently invokes ``run_task`` more than once for a shared + ``run_id``; a future caller that does (e.g. a retry driver re-running the + same logical task under multiple ``trace_id``s) can pass one shared + ``run_id`` across those invocations so their manifests become + reconcilable together as one batch via + ``attempt_reconciliation.reconcile_attempt_batch_by_run_id``. """ trace_dir = trace_store.get_trace_dir(trace_id) os.makedirs(trace_dir, exist_ok=True) @@ -213,6 +231,10 @@ async def run_task( else: profile = AgentProfile(name="default", llm_config=initialize_llm_config()) + _record_attempt_manifest( + trace_id=trace_id, checkpoint="launch", llm_config=profile.llm_config, run_id=run_id + ) + config_builder = Builders.AgentConfig.with_default_profile(profile) if verification_level: config_builder.with_verification_level(verification_level) @@ -228,7 +250,18 @@ async def run_task( config = config_builder.build() - agent = Agent(config=config) + # session_id=trace_id links this run's DataEngine session (where + # native llm_usage receipts land) to the same identifier the attempt + # manifest is keyed and stored by, so post-run reconciliation + # (_reconcile_and_store_verdict) can find one attempt's usage events + # without inventing a separate cross-process identity mapping. + agent = Agent(config=config, session_id=trace_id) + _record_attempt_manifest( + trace_id=trace_id, + checkpoint="worker_start", + llm_config=profile.llm_config, + run_id=run_id, + ) await _initialize_agent( agent, retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), @@ -400,6 +433,24 @@ async def run_task( finally: print("Cleaning up resources...") + attempt_profile = locals().get("profile") + if attempt_profile is not None: + _record_attempt_manifest( + trace_id=trace_id, + checkpoint="termination", + llm_config=attempt_profile.llm_config, + run_id=run_id, + ) + # Reconciliation runs regardless of task outcome (success, failed + # status, exception, or cancellation) — identity verification is + # orthogonal to whether the mobile task itself succeeded, and a + # failed attempt's usage receipts still need checking against its + # manifest. Reconciles against the "launch" manifest checkpoint, + # since that is the one guaranteed to have been stored before any + # inference call could have happened. Also runs batch + # reconciliation across run_id when a real (non-default) run_id + # was passed to this invocation. + _reconcile_and_store_verdict(trace_id=trace_id, checkpoint="launch", run_id=run_id) if agent: try: await agent.clean() @@ -412,7 +463,13 @@ async def run_task( log_file_err.close() -if __name__ == "__main__": +def _build_arg_parser() -> argparse.ArgumentParser: + """Builds the CLI parser for this module's ``__main__`` entry point. + + Factored out of ``if __name__ == "__main__":`` so tests can exercise the + real CLI surface (e.g. ``--run-id`` acceptance) via ``parse_args()`` + without spawning a subprocess. + """ parser = argparse.ArgumentParser(description="Artemis Background Task Runner") parser.add_argument("--trace-id", required=True, help="Unique trace identifier") parser.add_argument("--task-desc", required=True, help="Description of the task to run") @@ -434,8 +491,22 @@ async def run_task( "--explorer-pro-mode", help="Pro-profile Explorer perception version: 'flash', 'pro' or 'ultra'", ) + parser.add_argument( + "--run-id", + help=( + "Optional Gate 1 batch-grouping identifier shared across multiple " + "run_task invocations (e.g. repeated attempts of one journey/device " + "cell). Defaults to --trace-id when omitted, preserving today's " + "exact 1:1 run_id<->trace_id behavior; see " + "attempt_reconciliation.reconcile_attempt_batch_by_run_id." + ), + ) - args = parser.parse_args() + return parser + + +if __name__ == "__main__": + args = _build_arg_parser().parse_args() asyncio.run( run_task( @@ -449,5 +520,6 @@ async def run_task( device_serial=args.device_serial, verification_level=args.verification_level, explorer_pro_mode=args.explorer_pro_mode, + run_id=args.run_id, ) ) diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index c62b5226..71997be4 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -376,15 +376,23 @@ async def _verify_credentials(cred_result: ProbeResult | None) -> list[dict[str, return [] metadata = cred_result.metadata api_keys = metadata.get("api_keys") or {} - targets: list[tuple[str, str, str]] = [] + targets: list[tuple[str, str, str, str | None]] = [] for entry in metadata.get("providers") or []: provider = str(entry.get("provider") or "").strip() raw_key = entry.get("raw_key") or api_keys.get(provider) or "" if not provider or not raw_key: continue - targets.append((provider, str(entry.get("label") or provider), str(raw_key))) + base_url = entry.get("base_url") + targets.append( + ( + provider, + str(entry.get("label") or provider), + str(raw_key), + str(base_url) if base_url else None, + ) + ) if api_keys.get("ocr"): - targets.append(("ocr", "Vision OCR", str(api_keys["ocr"]))) + targets.append(("ocr", "Vision OCR", str(api_keys["ocr"]), None)) if not targets: return [] @@ -397,13 +405,18 @@ async def _verify_credentials(cred_result: ProbeResult | None) -> list[dict[str, timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS, ) if provider in _ENDPOINT_PROVIDERS - else validate_api_key(provider, key, timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS) - for provider, _label, key in targets + else validate_api_key( + provider, + key, + **({"base_url": base_url} if base_url else {}), + timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS, + ) + for provider, _label, key, base_url in targets ), return_exceptions=True, ) verified: list[dict[str, Any]] = [] - for (provider, label, key), outcome in zip(targets, outcomes): + for (provider, label, key, _base_url), outcome in zip(targets, outcomes): if isinstance(outcome, BaseException): valid, message = False, f"verification raised {outcome.__class__.__name__}: {outcome}" else: diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index d30c42eb..1b8fb5c9 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -212,6 +212,7 @@ def mobile_run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """Starts an autonomous mobile UI automation subagent on a connected Android device. @@ -277,6 +278,12 @@ def mobile_run_task( by the Operator: `"flash"` (1-shot detection, the default), `"pro"` (3-turn ReAct), `"ultra"` (deep pixel reasoning; slowest). Ignored for Flash. + run_id: Optional Gate 1 batch-grouping identifier. Plumbing for + grouping multiple real attempts (e.g. repeated attempts of one + journey/device cell) into one reconcilable batch at termination — + no built-in caller sets this today; omit unless you are + deliberately re-running the same logical task under a shared + identifier across multiple `mobile_run_task` calls. """ # 0. Validate and normalize model if model.lower() not in ("flash", "pro"): @@ -325,6 +332,7 @@ def mobile_run_task( conversation_id=conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + run_id=run_id, base_url=base_url, ) if resp and resp.get("status") == "rejected": @@ -471,6 +479,8 @@ def mobile_run_task( cmd.extend(["--verification-level", verification_level]) if explorer_mode: cmd.extend(["--explorer-pro-mode", explorer_mode]) + if run_id: + cmd.extend(["--run-id", run_id]) env = os.environ.copy() env["ARTEMIS_SESSION_ID"] = trace_id diff --git a/pyproject.toml b/pyproject.toml index e5cd095f..a36b8348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,9 @@ exclude = [ "node_modules", "site-packages", "venv", + # CHE-564: unmodified upstream directory fails the fork's UP ruleset; + # excluded to keep the fork's diff against upstream minimal. + "playground", ] # Same as Black. diff --git a/qualification/FORK_MAINTENANCE.md b/qualification/FORK_MAINTENANCE.md new file mode 100644 index 00000000..d9e4959b --- /dev/null +++ b/qualification/FORK_MAINTENANCE.md @@ -0,0 +1,55 @@ +# Cheese Work fork maintenance contract + +`cheese-work/artemis` tracks `google/artemis` (Apache-2.0, Pixel-Test-Engineering +"Fusion" team) under a **pinned-SHA policy**, not unpinned `main` tracking. +Upstream is early (repo created 2026-08-13, no tagged releases yet), so a +moving `main` is not a stable qualification target. + +## What is pinned, and where + +The single source of truth for the currently-qualified fork commit is +`qualification/manifests/pocket_actual_save_relaunch.v1.json`'s `fork_sha` +field (and any sibling manifest added for other qualification journeys). +CI, qualification runs, and rollout tooling read that field rather than +`origin/main` directly. + +## Weekly upstream review + +Once per week: + +1. Fetch upstream (`git fetch google main` with `google` remoting to + `https://github.com/google/artemis`, or the equivalent configured + remote). +2. Diff `google/main` against the currently pinned `fork_sha`. +3. Triage: security fixes and bug fixes affecting the qualified journey(s) + are candidates for promotion; unrelated feature work is not pulled in + opportunistically. +4. Record the review (date, commits considered, promote/skip decision) as + an issue comment on the fork-maintenance tracking issue (CHE-387) or a + dedicated weekly-review issue if volume warrants it — not silently in + git history alone. + +## Manual pin promotion + +Promoting the pinned SHA is manual and gated, never automatic: + +1. Open a PR that bumps `fork_sha` (and any file that embeds it) to the + new candidate commit. +2. The new SHA must pass the same independent-review chain as any other + qualification input change (see `manifests/pocket_actual_save_relaunch.v1.json`'s + `review_contract`): a changed `fork_sha` invalidates prior evidence per + the revision policy, so promotion always implies a fresh qualification + batch before the new pin is relied on for production rollout — not + before the PR merges. +3. Carry-forward Cheese Work patches (this fork's own diffs from upstream) + are rebase-checked against the candidate commit as part of the same PR; + a promotion that breaks a carried patch is not mergeable as-is. + +## Cheese Work patches vs upstream contributions + +- Cheese Work-specific changes (Gate 1 manifest/reconciliation mechanism, + qualification harness, this directory) live in the fork and are not + proposed upstream by default. +- If a fix is generally applicable (not Cheese-specific), it may be + proposed to `google/artemis` only through a Cheese-selected issue — + never opportunistically from an unrelated task. diff --git a/qualification/evidence_schema.v1.json b/qualification/evidence_schema.v1.json new file mode 100644 index 00000000..bf04c025 --- /dev/null +++ b/qualification/evidence_schema.v1.json @@ -0,0 +1,104 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "cheese-work/artemis/qualification/evidence_schema.v1.json", + "title": "Cheese Work Artemis qualification evidence record v1", + "description": "One record per qualification attempt (positive run or the negative control). Freeze N=10 positive runs per target/tier plus one negative control per CHE-388 plan v2; each attempt's evidence record must reference the exact qualification_manifest it ran against by digest.", + "type": "object", + "required": [ + "evidence_schema_version", + "qualification_manifest_digest", + "attempt_kind", + "target", + "tier", + "profile", + "host", + "device_serial", + "verdict", + "reconciliation", + "trace_id", + "recorded_at", + "cost", + "cleanup_verified" + ], + "properties": { + "evidence_schema_version": { "const": 1 }, + "qualification_manifest_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "SHA-256 digest of the pinned qualification manifest this attempt ran against (manifests/pocket_actual_save_relaunch.v1.json canonical_bytes). Never accept an attempt whose digest does not match the manifest under qualification." + }, + "attempt_kind": { + "type": "string", + "enum": ["positive", "negative_control"] + }, + "target": { + "type": "string", + "description": "Which model tier target this attempt qualifies, e.g. 'terra' or 'sol'." + }, + "tier": { + "type": "string", + "enum": ["luna", "terra", "sol"], + "description": "Gate 1 tier vocabulary (artemis/config/attempt_manifest.py TIER_MODELS, PR #1 / CHE-491)." + }, + "profile": { + "type": "string", + "enum": ["flash", "pro"] + }, + "host": { + "type": "string", + "enum": ["congvc-x99", "congvc-c00"] + }, + "device_serial": { "type": "string" }, + "per_attempt_manifest_digest": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{64}$", + "description": "Digest of the Gate 1 per-attempt LLM identity manifest (attempt_manifest.store_attempt_manifest output) for this specific attempt, once PR #1 lands and a real run produces one. Null while the mechanism referenced in PR #1 is not yet merged onto the base this attempt ran from." + }, + "verdict": { + "type": "string", + "enum": ["pass", "fail_assertion", "fail_infrastructure", "device_unavailable"], + "description": "Distinguish test failure, infrastructure failure, and unavailable-device coverage per CHE-388 acceptance criteria." + }, + "reconciliation": { + "type": "object", + "description": "Gate 1 batch accept/reject outcome for this attempt (artemis/config/attempt_reconciliation.py validate_batch, PR #1 / CHE-491).", + "required": ["accepted"], + "properties": { + "accepted": { "type": "boolean" }, + "reject_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot" + ] + } + } + } + }, + "trace_id": { "type": "string" }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "cost": { + "type": "object", + "required": ["usd", "step_count"], + "properties": { + "usd": { "type": "number", "minimum": 0 }, + "step_count": { "type": "integer", "minimum": 0 } + } + }, + "cleanup_verified": { + "type": "boolean", + "description": "True only if the fixture/reset step (see journeys/pocket_actual_save_relaunch.md) was confirmed to leave the candidate app and device in its pre-attempt state." + }, + "notes": { "type": "string" } + } +} diff --git a/qualification/journeys/pocket_actual_save_relaunch.md b/qualification/journeys/pocket_actual_save_relaunch.md new file mode 100644 index 00000000..34061c6d --- /dev/null +++ b/qualification/journeys/pocket_actual_save_relaunch.md @@ -0,0 +1,88 @@ +# Qualification journey: pocket-actual save/relaunch + +CHE-537 source packet. This is the smallest retrievable journey for CHE-388 +qualification: persist data via normal UI interaction, kill and relaunch the +app, and assert the persisted state survived — plus one deliberately false +assertion to prove the pipeline can fail for the right reason. + +This file is instructions and assertions only. It does not run anything, is +not evidence of a pilot pass, and does not implement Gate 1's manifest or +reconciliation mechanism (that mechanism is PR #1 / CHE-491, referenced by +digest in `manifests/pocket_actual_save_relaunch.v1.json`, and reused, not +rebuilt, here). + +## Fixture / reset + +Before every attempt (positive or negative control): + +1. Force-stop the candidate app: `adb -s shell am force-stop `. +2. Clear app data: `adb -s shell pm clear `. +3. Confirm cold-start state: relaunch once, verify the entry screen shows no + prior data (empty/default state). This is a scripted ADB check, not an + Artemis-driven step, and runs outside any budgeted attempt. + +`` and `` are supplied at run time by the +execution stage (CHE-540+); this journey does not pin them. + +## Positive run — natural language task instructions + +Goal text passed as `task_desc` (MCP `mobile_run_task`) / `goal` (CLI `artemis run`): + +> Open the app. Create a new entry with the label "qual-" and the +> value "" (substitute the actual run id and a freshly +> generated random token before dispatch — never reuse a token across +> attempts, so a stale relaunch can never be mistaken for a fresh save). +> Save it. Force the app to the background, then fully close it. Relaunch +> the app from the home screen. Open the same entry and read back its +> value. + +`expected_output_desc` / `output_description`: + +> Report the exact value read back for the entry labeled "qual-" +> after relaunch, and whether it matches "" exactly. + +### Assertions (positive run) + +1. **Save succeeded**: the app's own UI confirms the entry was created + (e.g. a visible list item, a save confirmation) before the + background/close step begins. +2. **Relaunch reached the same screen**: after relaunch, the agent + navigates back to the entry (via whatever path the app's UI requires) — + this is graded as part of the run, not scripted, since the navigation + path is exactly what a real user would need and is part of what + qualification is proving. +3. **Persisted value matches**: the value read back after relaunch equals + "" byte-for-byte. Anything else (empty, default, a + different prior value, an app crash) is `fail_assertion`. +4. **Verdict discipline**: an attempt that never reaches a relaunched, + readable state (app crash, ANR, device disconnect, ADB failure) is + `fail_infrastructure` or `device_unavailable`, never silently folded + into `fail_assertion` — see `evidence_schema.v1.json`'s `verdict` enum. + +## Negative control — deliberately false assertion + +Same fixture, same save step, but the checker is told to verify a value +that provenance guarantees is wrong: + +> ...(identical save/relaunch steps as the positive run)... Open the same +> entry and confirm its value reads "-WRONG-SUFFIX" exactly. + +This must produce `fail_assertion` (Checker or Outputter catches the +mismatch and reports failure) on every run. A negative control that comes +back `pass` invalidates the entire batch it belongs to — the pipeline is +not distinguishing true from false, so no positive result in that batch +can be trusted either. Record it as its own evidence entry +(`attempt_kind: "negative_control"`), not folded into the N=10 positive +count (CHE-388 plan v2: "freeze N=10 positive runs per target/tier plus +one separate negative control"). + +## Bounded execution and cleanup + +- One attempt = one `mobile_run_task` / `artemis run` invocation from a + freshly reset fixture to either a verdict or a hard timeout. +- After every attempt (pass, fail, or infrastructure failure), re-run the + fixture/reset steps and confirm cold-start state before releasing the + device lease — this is `cleanup_verified` in the evidence schema. +- No attempt escalates tier, retries silently, or mutates the manifest + mid-batch. A revised candidate or manifest requires a fresh batch (see + `manifests/pocket_actual_save_relaunch.v1.json`'s `revision_policy`). diff --git a/qualification/manifests/pocket_actual_save_relaunch.v1.json b/qualification/manifests/pocket_actual_save_relaunch.v1.json new file mode 100644 index 00000000..4410f42e --- /dev/null +++ b/qualification/manifests/pocket_actual_save_relaunch.v1.json @@ -0,0 +1,111 @@ +{ + "manifest_version": 1, + "$schema_ref": "qualification/evidence_schema.v1.json", + "issue": "CHE-537", + "coordinating_parent": "CHE-388", + "journey": "qualification/journeys/pocket_actual_save_relaunch.md", + + "fork_sha": { + "value": "64e1b3226b9553bdf4e5a368f14b8a40c0af6111", + "source": "git", + "repo": "https://github.com/cheese-work/artemis", + "note": "Source baseline observed and pinned per CHE-537 delivery plan. Weekly upstream review and manual pin promotion after qualification -- see FORK_MAINTENANCE.md. This value must be updated (new revision) if the pinned SHA changes; execution stages must re-read it at admission rather than caching an older value." + }, + + "gate1_manifest_mechanism": { + "source": "https://github.com/cheese-work/artemis/pull/1", + "head_sha": "2b58709ed00b0015055a99f9e644857ba9d05ee0", + "status": "OPEN (draft, not merged) as observed at CHE-537 authoring time", + "note": "Reused by reference, not rebuilt or ported here. Per-attempt LLM identity manifests (artemis/config/attempt_manifest.py) and reconciliation (artemis/config/attempt_reconciliation.py) are produced at real-run time by this mechanism once it is merged onto the pinned fork_sha (or its own base is advanced past fork_sha, whichever the executing stage's admitted revision resolves to). This qualification manifest does not claim Gate 1 has passed." + }, + + "candidate_app": { + "app_sha": "2c42481778e6f9b88a9502f55decdbe4e1a76e83", + "apk_digest_sha256": "04795d5f3995c8e895f6440a9763c8a003c8b6ff955541e3965f5e71564e2e12", + "package_name": "dev.cheese.pocketactual", + "note": "Pinned by CHE-540 (manifest revision v2). Source: https://github.com/cheese-work/pocket-actual at its main HEAD `2c42481778e6f9b88a9502f55decdbe4e1a76e83`, observed and built at authoring time (no device install performed). apk_digest_sha256 is `sha256sum` of `composeApp/build/outputs/apk/debug/composeApp-debug.apk` produced by `./gradlew :composeApp:assembleDebug` at that exact commit on X99. Historical Maestro-era evidence exists on pocket-actual PR #5 (app_sha `29ddf249c573ec6e07effa0c52708b7007d12ea9`) but is superseded, not reused, per revision_policy -- that build predates the Maestro-to-Artemis cutover." + }, + + "dependency_lock": { + "file": "uv.lock", + "digest_sha256": "2c63863dd5827482d7e206a887219c6438ce61dc494d9cc596f862378decc19a", + "note": "sha256sum of this fork's own uv.lock (the Artemis qualification harness's dependency lock, not the candidate app's) at the pinned fork_sha `64e1b3226b9553bdf4e5a368f14b8a40c0af6111` checkout -- confirmed unchanged between fork_sha and this manifest revision's HEAD via `git diff fork_sha HEAD -- uv.lock` (empty)." + }, + + "accessibility_helper": { + "name": "Artemis Accessibility Helper (UIAutomator2 fallback)", + "digest_sha256": null, + "note": "Per-host on-device helper referenced in CHE-388's install surface. Digest recorded by the host install stage, not authored here." + }, + + "model_targets": [ + { + "tier": "luna", + "provider": "google", + "model": "gemini-3.5-flash-lite", + "provider_id": "google:gemini-3.5-flash-lite", + "role": "cheap tier under CHE-388's 'compare Luna and Terra model tiers' requirement" + }, + { + "tier": "terra", + "provider": "google", + "model": "gemini-3.8-flash", + "provider_id": "google:gemini-3.8-flash", + "role": "mid tier under CHE-388's 'compare Luna and Terra model tiers' requirement" + } + ], + "tier_vocabulary_source": "artemis/config/attempt_manifest.py TIER_MODELS (PR #1 / CHE-491) -- do not diverge from that mapping; if it changes, this manifest must be revised to match rather than declaring a third mapping.", + "sol_tier_excluded": { + "tier": "sol", + "provider": "openai", + "model": "gpt-5.6-sol", + "reason": "Excluded from model_targets, not merely deprioritized. The 'sol' Gate 1 tier resolves to an OpenAI-backed model, and the workspace announcement is 'Stop all routing to OpenAI based agents. No set end date' (reaffirmed same-day on this issue: 'no openai based agent runs. they're all dead'). A batch of N=10 positive runs + 1 negative control against this target cannot execute under current policy, so pinning it as an active qualification target would freeze a dead-on-arrival gate.", + "unblock_condition": "Add 'sol' back to model_targets only after the OpenAI routing-stop announcement is explicitly lifted or replaced by a routable provider/model pair under the same 'sol' tier label. Until then, config/artemis.jsonc's top-level default node (openai/gpt-5.6-sol) stays unqualified by this manifest even though it is the fork's real configured default -- this is a known, deliberate gap, not an oversight." + }, + + "profile": "pro", + "profile_note": "Pro selected over Flash for this journey: the assertion step (read back a persisted value against an expected/negative token) needs the Checker's verified-checkpoint path (CHE-388 install-surface note: Flash has no final report and no verification). verification_level left to the execution stage's per-run config, not pinned here.", + + "entry_point": { + "route": "mcp", + "tool": "mobile_run_task", + "fallback_route": "cli", + "fallback_command": "artemis run", + "note": "CHE-537 plan: 'Prefer the existing MCP route with foreground terminal collection; select native CLI/SDK only where exact installed behavior proves necessary.' mobile_run_task params referenced: task_desc, model (tier profile), locked_app_package, app_path, expected_output_desc, verification_level." + }, + + "fixture": { + "journey_ref": "qualification/journeys/pocket_actual_save_relaunch.md", + "reset_procedure": "adb shell am force-stop && adb shell pm clear , cold-start verified before every attempt", + "positive_runs_required": 10, + "negative_controls_required": 1, + "note": "N=10 positive runs per target/tier plus one separate negative control per CHE-388 plan v2. Counts are per (tier, profile) combination in model_targets, not a single global N=10." + }, + + "evidence_schema": { + "file": "qualification/evidence_schema.v1.json", + "version": 1 + }, + + "revision_policy": { + "rule": "A changed candidate revision (app_sha, apk_digest_sha256) or a changed fork_sha forces a new manifest revision and invalidates all evidence collected against the prior revision. Evidence is not transferable across revisions.", + "source": "CHE-388 review contract invariant: 'a changed candidate revision forces fresh step-2 review AND invalidates prior evidence (new run required)'." + }, + + "review_contract": { + "chain_as_originally_settled": "Terra authors -> one candidate reviewer approves -> Luna executes -> Opus AND Sol independently review the run evidence -> Terra verifies", + "chain_status": "Superseded 2026-09-16 (issue comments 01a0a83b, 01a0a83f): Terra/Sol/Luna/Astra are permanently unroutable (OpenAI routing stopped workspace-wide, no set end date). Chain is re-staffed Claude-only; independence is now host-scoped (different agent instance on a different runtime), not model-scoped -- a recorded, accepted reduction in gate strength, not equivalent to the original design.", + "candidate_reviewer_this_issue": "mbp-claude-sonnet-5 (read-only review, comment 01a0a844; did not author this candidate, independence holds on host-scoped grounds). x99-codex-5.6-sol was named at authoring time but is permanently unroutable and never actually reviewed this candidate.", + "evidence_reviewers_current": "CHE-540/541/542 evidence review: reviewer A x99-claude-opus-5, reviewer B mbp-claude-opus-5 (different runtime). The human-decision gate on these stages was lifted 2026-09-16; they now carry only technical/dependency gates (CHE-537 published inputs, CHE-538/539 pinned runtime install).", + "no_self_review": true + }, + + "out_of_scope_for_this_manifest_authoring": [ + "host install", + "provider calls", + "device operations", + "binding changes", + "agent merge", + "claiming Gate 1 or any qualification gate has passed" + ] +} diff --git a/tests/unit/admin_console/test_session_status_normalization.py b/tests/unit/admin_console/test_session_status_normalization.py index 79094410..9c1a087f 100644 --- a/tests/unit/admin_console/test_session_status_normalization.py +++ b/tests/unit/admin_console/test_session_status_normalization.py @@ -58,3 +58,29 @@ def test_get_session_status_stays_raw_for_queue_reconcile(tmp_path): _insert_session(db_path, "legacy", "success") assert repository.get_session_status("legacy") == "success" + + +def test_create_queued_session_persists_queue_metadata(tmp_path): + repository = SessionRepository(tmp_path / "sessions.db") + + assert repository.create_queued_session( + "queued-session", "Open settings", "pro", "device-123", start_time=0.0 + ) + + row = repository.get_session_by_id("queued-session") + assert row is not None + assert row["status"] == "queued" + assert row["start_time"] == 0.0 + assert row["device_info"] == '{"profile": "pro", "device_id": "device-123"}' + + +def test_create_queued_session_preserves_existing_terminal_session(tmp_path): + db_path = tmp_path / "sessions.db" + repository = SessionRepository(db_path) + _insert_session(db_path, "terminal", "completed") + + assert not repository.create_queued_session("terminal", "New goal", "flash", None) + + row = repository.get_session_by_id("terminal") + assert row is not None + assert row["status"] == "completed" diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 24a572cd..ef057ad5 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -22,8 +22,10 @@ import pytest from apps.admin_console.core.state import state +from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.routers.tasks import get_status from apps.admin_console.services.task_queue_service import TaskQueueService, task_queue_service +from artemis.runtime import trace_store from artemis.runtime.device_lock import DeviceLockOwner from artemis.runtime.adb_endpoint import AdbEndpoint @@ -52,6 +54,8 @@ def clean_state(tmp_path, monkeypatch): "artemis.runtime.cancel_requests.get_temp_dir", lambda _subfolder=None: isolated_lock_dir, ) + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + monkeypatch.setattr(queue_module, "session_repo", SessionRepository(tmp_path / "sessions.db")) state.clear_queue() state.queue_items.clear() state.current_process = None @@ -132,6 +136,129 @@ async def test_enqueued_task_keeps_its_adb_endpoint_snapshot(): assert target.lock_key == f"{original.identity}/emulator-5554" +@pytest.mark.asyncio +async def test_pre_session_worker_failure_persists_failed_session_and_releases_ticket(tmp_path): + async def failed_worker(*_args, **_kwargs): + proc = MagicMock() + proc.pid = 99999999 + proc.returncode = 1 + proc.stdout = None + proc.wait = AsyncMock(return_value=1) + return proc + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + patch("asyncio.create_subprocess_exec", side_effect=failed_worker), + patch.object(TaskQueueService, "_recover_or_fail_recording", new=AsyncMock()), + ): + result = await TaskQueueService.enqueue_tasks( + ["Configuration fails before session startup"], + profile="pro", + device_serial="test-device", + ) + task_item = result["tasks"][0] + session_id = task_item["session_id"] + + trace = trace_store.read_status(session_id) + assert trace["status"] == "running" + assert trace["model"] == "pro" + assert trace["device_serial"] == "test-device" + + await TaskQueueService._execute_task_item(task_item) + + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + persisted = queue_module.session_repo.get_session_by_id(session_id) + assert persisted is not None + assert persisted["status"] == "failed" + assert persisted["end_time"] is not None + assert trace_store.read_status(session_id)["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_rejects_terminal_trace_without_overwriting_it(tmp_path): + session_id = "terminal-session" + trace_store.init_trace(session_id, "Finished goal", "flash") + trace_store.update_trace_status(session_id, "completed") + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="terminal trace"), + ): + await TaskQueueService.enqueue_tasks( + ["Do not overwrite"], session_id=session_id, device_serial="test-device" + ) + + assert trace_store.read_status(session_id)["status"] == "completed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_marks_existing_running_trace_failed_when_db_admission_fails( + tmp_path, monkeypatch +): + session_id = "mcp-session" + trace_store.init_trace(session_id, "MCP goal", "flash") + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + monkeypatch.setattr(queue_module.session_repo, "create_queued_session", lambda *_args: False) + + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="Could not persist queued session"), + ): + await TaskQueueService.enqueue_tasks( + ["MCP goal"], session_id=session_id, device_serial="test-device", ingress="mcp" + ) + + assert trace_store.read_status(session_id)["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + +@pytest.mark.asyncio +async def test_enqueue_rolls_back_earlier_items_when_later_setup_fails(tmp_path, monkeypatch): + queue_module = importlib.import_module("apps.admin_console.services.task_queue_service") + repository = queue_module.session_repo + create_queued_session = repository.create_queued_session + session_ids: list[str] = [] + + def fail_second_queued_session(*args, **kwargs): + session_ids.append(args[0]) + if len(session_ids) == 2: + return False + return create_queued_session(*args, **kwargs) + + monkeypatch.setattr(repository, "create_queued_session", fail_second_queued_session) + with ( + patch.object(TaskQueueService, "ensure_worker_running"), + patch.object( + TaskQueueService, "_reject_unavailable_device", new=AsyncMock(return_value=None) + ), + pytest.raises(RuntimeError, match="Could not persist queued session"), + ): + await TaskQueueService.enqueue_tasks( + ["First task", "Second task"], device_serial="test-device" + ) + + assert len(session_ids) == 2 + assert repository.get_session_status(session_ids[0]) == "failed" + assert trace_store.read_status(session_ids[0])["status"] == "failed" + assert trace_store.read_status(session_ids[1])["status"] == "failed" + assert state.queue_items == [] + assert not list((tmp_path / "device-locks" / "artemis-global-device.queue").glob("*.wait")) + + @pytest.mark.parametrize( ("current_status", "returncode", "stopped", "expected"), [ @@ -343,6 +470,60 @@ async def fake_subprocess_exec(*args, **kwargs): pass +@pytest.mark.asyncio +async def test_queue_worker_cmd_forwards_run_id(): + """A daemon-dispatched task carrying a Gate 1 run_id must reach the spawned + `artemis.main` worker as `--run-id` -- otherwise the daemon path never + records/reconciles an attempt manifest under that batch key (see + artemis.config.attempt_lifecycle_hooks and + artemis.interfaces.cli.commands.run.execute_task).""" + executed_cmds = [] + + async def fake_subprocess_exec(*args, **kwargs): + executed_cmds.append(list(args)) + proc = MagicMock() + proc.pid = 88889 + proc.wait = AsyncMock(return_value=0) + proc.returncode = 0 + return proc + + with ( + patch("asyncio.create_subprocess_exec", side_effect=fake_subprocess_exec), + patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo, + patch("apps.admin_console.services.task_queue_service.media_service"), + patch( + "artemis.runtime.device_pool.device_pool.select_device_async", + return_value="emulator-5554", + ), + ): + mock_repo.get_running_session_id.return_value = None + mock_repo.get_video_recording_for_session.return_value = {"status": "ready"} + + await task_queue_service.enqueue_tasks( + ["Test Goal with run_id"], + profile="flash", + run_id="daemon-batch-1", + ) + + for _ in range(30): + if len(executed_cmds) == 1 and len(state.queue_items) == 0: + break + await asyncio.sleep(0.05) + + assert len(executed_cmds) == 1 + cmd = executed_cmds[0] + assert "--run-id" in cmd + assert cmd[cmd.index("--run-id") + 1] == "daemon-batch-1" + + task = state.worker_task + if task and not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + @pytest.mark.asyncio async def test_forward_worker_output_preserves_split_utf8(capsys): stream = asyncio.StreamReader() diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index a26bc57b..d5909ddc 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -15,6 +15,7 @@ """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" from unittest.mock import ANY, AsyncMock, Mock, call +from types import SimpleNamespace from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage @@ -38,6 +39,34 @@ def mock_context(): return ctx +def test_summarizer_uses_configured_role_for_explicit_model(mock_context, monkeypatch): + configured_llm = Mock(model_name="gpt-5.6-sol") + get_llm = Mock(return_value=configured_llm) + monkeypatch.setattr("artemis.agents.flash.summarizer.get_llm", get_llm) + + summarizer = VisualStepSummarizer(mock_context, model_name="gpt-5.6-sol") + + assert summarizer._llm is configured_llm + get_llm.assert_called_once_with( + mock_context, name="summarizer", temperature=0.0, model_name="gpt-5.6-sol" + ) + + +@pytest.mark.parametrize("model_name", [None, ""]) +def test_summarizer_uses_role_default_without_model_override(mock_context, monkeypatch, model_name): + configured_llm = Mock() + configured_llm.endpoint = SimpleNamespace(model_name="gpt-5.6-sol") + get_llm = Mock(return_value=configured_llm) + monkeypatch.setattr("artemis.agents.flash.summarizer.get_llm", get_llm) + + summarizer = VisualStepSummarizer(mock_context, model_name=model_name) + + assert summarizer._model_name == "gpt-5.6-sol" + get_llm.assert_called_once_with( + mock_context, name="summarizer", temperature=0.0, model_name=None + ) + + @pytest.mark.asyncio async def test_summarizer_dispatch_and_caching(mock_context): """Verify non-blocking dispatch and summary generation.""" @@ -840,7 +869,7 @@ def test_flash_config_and_builder(): step_summarizer=True, step_summarizer_model="gemini-2.5-flash-lite", prune_history_xml=True, - ).build() + ).build(validate_profiles=False) assert cfg.flash.max_turns == 25 assert cfg.flash.explorer_mode == "flash" diff --git a/tests/unit/agents/test_object_detector_llm_resolution.py b/tests/unit/agents/test_object_detector_llm_resolution.py new file mode 100644 index 00000000..edd25d65 --- /dev/null +++ b/tests/unit/agents/test_object_detector_llm_resolution.py @@ -0,0 +1,148 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CHE-646: object_detector must resolve its own configured model from the +llm utils config, and must never silently fall back to the operator model +without a visible log signal. +""" + +import logging +from unittest.mock import AsyncMock, MagicMock + +from artemis.agents.object_detector import object_detector as od +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +import pytest + + +def _empty_response_llm() -> MagicMock: + """A fake LLM whose ainvoke resolves to an empty-list JSON response, so + _detect_single_label completes cleanly without hitting the network.""" + llm = MagicMock() + llm.ainvoke = AsyncMock(return_value=MagicMock(content="[]")) + return llm + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +def _config(object_detector: LLMWithFallback | None) -> LLMConfig: + """A fully-populated LLMConfig; utils.object_detector varies per test.""" + node = _llm() + return LLMConfig( + **{n: node for n in _REQUIRED_NODES}, + utils=LLMConfigUtils( + outputter=node, + hopper=node, + object_detector=object_detector, + ), + ) + + +@pytest.mark.asyncio +async def test_configured_object_detector_resolves_via_utils_not_operator(monkeypatch): + """When utils.object_detector IS configured, the detector must resolve to + that configured entry via get_llm(name='object_detector', is_utils=True), + never to the operator entry.""" + detector_llm = _llm(model="detector-model") + ctx = MagicMock() + ctx.llm_config = _config(object_detector=detector_llm) + + calls = [] + + def fake_get_llm(ctx_arg, name, is_utils=False, **kwargs): + calls.append({"name": name, "is_utils": is_utils}) + return _empty_response_llm() + + monkeypatch.setattr(od, "get_llm", fake_get_llm) + + result = await od._run_object_detection( + ctx, image_bytes=b"fake-image-bytes", queries=["button"] + ) + + assert result["detected"] == [] + # Exactly one get_llm call, for the configured detector node -- never operator. + assert calls == [{"name": "object_detector", "is_utils": True}] + + +@pytest.mark.asyncio +async def test_unconfigured_object_detector_logs_warning_and_falls_back_to_operator( + monkeypatch, caplog +): + """When utils.object_detector is None, get_llm(is_utils=True) raises ValueError + (see LLMConfig.get_utils). The detector must log a visible warning AND fall + back to the operator model -- the fallback must remain functional, but the + condition must no longer be silently swallowed.""" + ctx = MagicMock() + ctx.llm_config = _config(object_detector=None) + + calls = [] + + def fake_get_llm(ctx_arg, name, is_utils=False, **kwargs): + calls.append({"name": name, "is_utils": is_utils}) + if name == "object_detector": + raise ValueError("Utils 'object_detector' is not configured.") + return _empty_response_llm() + + monkeypatch.setattr(od, "get_llm", fake_get_llm) + + with caplog.at_level(logging.WARNING, logger="artemis.agents.object_detector.object_detector"): + result = await od._run_object_detection( + ctx, image_bytes=b"fake-image-bytes", queries=["button"] + ) + + assert result["detected"] == [] + # Both calls happened: the failed primary attempt, then the operator fallback. + assert calls == [ + {"name": "object_detector", "is_utils": True}, + {"name": "operator", "is_utils": False}, + ] + # The fallback condition must be visible in logs, not silently swallowed. + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert any( + "object_detector" in r.getMessage() and "operator" in r.getMessage() for r in warnings + ) + + +def test_prefix_path_without_is_utils_raises_attribute_error(): + """Regression guard: object_detector is a field of LLMConfigUtils, not of + LLMConfig. get_agent('object_detector') (the pre-fix, is_utils=False path) + must still raise AttributeError -- proving the production call site must + pass is_utils=True and cannot silently rely on get_agent's defaulting.""" + config = _config(object_detector=_llm()) + + with pytest.raises(AttributeError): + config.get_agent("object_detector") # type: ignore[arg-type] + + # The correct (is_utils=True) path resolves cleanly instead. + resolved = config.get_utils("object_detector") + assert resolved.model == "gpt-5.6-sol" diff --git a/tests/unit/cli/test_run_command_gate1.py b/tests/unit/cli/test_run_command_gate1.py new file mode 100644 index 00000000..5d944c78 --- /dev/null +++ b/tests/unit/cli/test_run_command_gate1.py @@ -0,0 +1,224 @@ +"""Gate 1 evidence regression tests for the daemon-dispatched / direct-CLI +``execute_task`` path (``artemis.interfaces.cli.commands.run``). + +Covers three defects found in independent review of the initial daemon-path +wiring: + +1. The default ``--standalone`` route (no explicit ``--session-id``) must + still generate a canonical trace identity so a real inference run is + never Gate-1-evidence-free by default. +2. ``reconcile_and_store_verdict`` must run before -- and regardless of -- + ``agent.clean()``, since ``Agent.clean()`` can raise (e.g. on device + disconnect) and must never be able to suppress the reconciliation + verdict for a real completed run. +3. A generated fallback identity from one default-route call must never + leak into a later default-route call in the same process via the + process-global ``ARTEMIS_SESSION_ID`` env var -- each attempt must get + its own identity and its own stored manifest. +4. An explicit ``--session-id`` on one call must not contaminate a *later* + default-route call either -- ``ARTEMIS_SESSION_ID`` must be restored to + whatever it held before the call (or cleared), not just skipped for the + generated-fallback case. +""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.interfaces.cli.commands import run as run_module + + +def _fake_llm_config(): + """A real, minimal LLMConfig (AgentProfile validates this via pydantic, + so a bare MagicMock is rejected) -- provider/model values don't matter + here since record_attempt_manifest/reconcile_and_store_verdict are + patched out in every test below.""" + from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback + + node = LLMWithFallback( + provider="google", + model="gemini-3.8-flash", + fallback=LLM(provider="google", model="gemini-3.8-flash"), + ) + required_nodes = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + ) + return LLMConfig( + **{n: node for n in required_nodes}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +def _fake_agent(clean_side_effect=None): + agent = MagicMock() + agent.init = AsyncMock() + agent.run_task = AsyncMock(return_value="ok") + agent.clean = AsyncMock(side_effect=clean_side_effect) + task = MagicMock() + task.build.return_value = "built-task" + agent.new_task.return_value = task + return agent + + +@pytest.mark.asyncio +async def test_execute_task_default_standalone_route_still_records_gate1_evidence(monkeypatch): + """No --session-id, no env session vars: execute_task must still record + and reconcile a real attempt under a generated trace identity, not skip + Gate 1 evidence for the default route.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + agent = _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", return_value=agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="do the thing", session_id=None) + + assert len(recorded) == 1 + assert recorded[0] # non-empty generated identity + assert reconciled == recorded # same trace_id reconciled as was recorded + + +@pytest.mark.asyncio +async def test_execute_task_two_default_calls_get_distinct_isolated_identities(monkeypatch): + """Two back-to-back default-route calls (no --session-id, no env session + vars) in the same process must each get their own generated trace id and + each record their own manifest -- a generated fallback identity must + never leak into the process env and get reused by a later default call, + which would silently merge two attempts under one identity.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + + def _make_agent(*args, **kwargs): + return _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", side_effect=_make_agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="first attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + await run_module.execute_task(goal="second attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + assert len(recorded) == 2 + assert recorded[0] and recorded[1] + assert recorded[0] != recorded[1] # distinct identities -> distinct manifests stored + assert reconciled == recorded + + +@pytest.mark.asyncio +async def test_execute_task_explicit_session_id_does_not_contaminate_later_default_call( + monkeypatch, +): + """An explicit --session-id call followed by a default-route call in the + same process must not collide: the first call's explicit identity must + not leak via ARTEMIS_SESSION_ID and get picked up as the second call's + "default" identity, which would merge two distinct attempts (and their + create-only manifests) under one trace id.""" + monkeypatch.delenv("ARTEMIS_SESSION_ID", raising=False) + monkeypatch.delenv("ARTEMIS_CLOUD_SESSION_ID", raising=False) + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + + recorded = [] + reconciled = [] + + def _make_agent(*args, **kwargs): + return _fake_agent() + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", side_effect=_make_agent), + patch.object( + run_module, + "record_attempt_manifest", + side_effect=lambda **kw: recorded.append(kw["trace_id"]), + ), + patch.object( + run_module, + "reconcile_and_store_verdict", + side_effect=lambda **kw: reconciled.append(kw["trace_id"]), + ), + ): + await run_module.execute_task(goal="explicit attempt", session_id="explicit-sid-123") + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + await run_module.execute_task(goal="default attempt", session_id=None) + assert os.environ.get("ARTEMIS_SESSION_ID") is None + + assert recorded == ["explicit-sid-123", recorded[1]] + assert recorded[1] != "explicit-sid-123" # default call got its own identity, not the leak + assert reconciled == recorded + + +@pytest.mark.asyncio +async def test_execute_task_reconciles_before_cleanup_and_survives_cleanup_failure(monkeypatch): + """A raising agent.clean() must not prevent reconcile_and_store_verdict + from running, and reconciliation must be observed to happen first.""" + monkeypatch.setattr(run_module.settings, "GOOGLE_API_KEY", "fake-test-key") + call_order = [] + agent = _fake_agent(clean_side_effect=RuntimeError("device disconnected")) + agent.clean.side_effect = None # set below after wrapping to record order + + async def clean_raises(): + call_order.append("clean") + raise RuntimeError("device disconnected") + + agent.clean = AsyncMock(side_effect=clean_raises) + + def reconcile_records(**kw): + call_order.append("reconcile") + + with ( + patch.object(run_module, "initialize_llm_config", return_value=_fake_llm_config()), + patch.object(run_module, "Agent", return_value=agent), + patch.object(run_module, "record_attempt_manifest"), + patch.object(run_module, "reconcile_and_store_verdict", side_effect=reconcile_records), + ): + # Must not raise: a cleanup failure must not propagate out of execute_task. + await run_module.execute_task(goal="do the thing", session_id="fixed-sid") + + assert call_order == ["reconcile", "clean"] diff --git a/tests/unit/config/test_attempt_manifest.py b/tests/unit/config/test_attempt_manifest.py new file mode 100644 index 00000000..99acd633 --- /dev/null +++ b/tests/unit/config/test_attempt_manifest.py @@ -0,0 +1,582 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate 1 mechanism tests: manifest determinism, storage, and the decoy-config +isolation control. + +All tests use ``tmp_path``/``monkeypatch`` exclusively. None of them touch the +real home directory, the real ``~/.artemis``/PAL config dir, or make any +provider/network call. +""" + +from __future__ import annotations + +import json +import threading + +import pytest + +from artemis.config.attempt_manifest import ( + TIER_MODELS, + ManifestAlreadyExistsError, + _node_tier, + build_attempt_manifest, + canonical_bytes, + digest_of, + read_stored_manifest, + store_attempt_manifest, +) +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.runtime import trace_store + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _uniform_config(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to one tier.""" + base = {node: _llm(provider, model) for node in _REQUIRED_NODES} + return LLMConfig( + utils=LLMConfigUtils(outputter=_llm(provider, model), hopper=_llm(provider, model)), + **base, + ) + + +def _sol_config(**node_overrides: LLMWithFallback) -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to 'sol'.""" + base = {node: _llm() for node in _REQUIRED_NODES} + base.update(node_overrides) + return LLMConfig(utils=LLMConfigUtils(outputter=_llm(), hopper=_llm()), **base) + + +@pytest.fixture(autouse=True) +def _isolate_traces_dir(tmp_path, monkeypatch): + """Point trace_store at an isolated tmp_path directory for every test here. + + This is the concrete guard against ever touching a real traces directory: + every test in this module writes manifests under ``tmp_path``, never under + the process's real TRACES_DIR. + """ + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + + +class TestCanonicalDigestStability: + """Requirement 2: same logical manifest -> byte-identical canonical form.""" + + def test_same_manifest_different_key_order_same_bytes_and_digest(self): + config = _sol_config() + manifest_a = build_attempt_manifest( + run_id="run-1", + attempt_id="attempt-1", + trace_id="trace-1", + tier="sol", + llm_config=config, + env={"ARTEMIS_CONFIG_DIR": "/tmp/does-not-matter"}, + checkpoint="launch", + now=1000.0, + ) + + # Same logical content, deliberately constructed via a different + # insertion order (dict equality ignores order, but a naive + # json.dumps without sort_keys would NOT produce identical bytes). + reordered = {k: manifest_a[k] for k in reversed(list(manifest_a.keys()))} + reordered["nodes"] = { + k: manifest_a["nodes"][k] for k in reversed(list(manifest_a["nodes"].keys())) + } + + assert reordered == manifest_a + assert canonical_bytes(reordered) == canonical_bytes(manifest_a) + assert digest_of(reordered) == digest_of(manifest_a) + + def test_canonical_bytes_have_no_whitespace_ambiguity(self): + manifest = build_attempt_manifest( + run_id="run-1", + attempt_id="attempt-1", + trace_id="trace-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1000.0, + ) + raw = canonical_bytes(manifest) + # No ambiguous JSON *structural* whitespace: a naive json.dumps(..., + # indent=2) would introduce ", " item separators and ": " key + # separators immediately followed by a structural character; our + # canonical form uses bare "," and ":" everywhere a separator is + # structurally required. Prose values (e.g. "no soft default: ...") + # legitimately contain ": " as English punctuation, so assert on the + # compact-separator round trip instead of a raw substring search. + assert ( + json.dumps(manifest, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode( + "utf-8" + ) + == raw + ) + assert b"\n" not in raw + assert b" " not in raw # no accidental double-space from indent=2-style output + # Round-trips to the same logical object. + assert json.loads(raw.decode("utf-8")) == manifest + + def test_digest_changes_when_a_non_tier_field_changes(self): + config_a = _sol_config() + # temperature is not part of the tier match (TIER_MODELS only looks + # at provider/model), so this stays within tier 'sol' while still + # changing a field that must affect the digest. + config_b = _sol_config( + planner=_llm(model="gpt-5.6-sol").model_copy(update={"temperature": 0.9}) + ) + manifest_a = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config_a, + env={}, + checkpoint="launch", + now=1.0, + ) + manifest_b = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config_b, + env={}, + checkpoint="launch", + now=1.0, + ) + assert digest_of(manifest_a) != digest_of(manifest_b) + + def test_mixed_tier_within_one_manifest_is_rejected_at_build_time(self): + config = _sol_config(planner=_llm(provider="google", model="gemini-3.5-flash-lite")) + with pytest.raises(ValueError, match="mixed_tier|other tier"): + build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + + +class TestAnthropicTierMatching: + """CHE-639: TIER_MODELS declares an Anthropic tier ('nova' -> claude-sonnet-5).""" + + def test_nova_tier_declared_with_expected_provider_and_model(self): + assert TIER_MODELS["nova"] == ("anthropic", "claude-sonnet-5") + + def test_anthropic_claude_sonnet_5_node_matches_nova_tier(self): + node = _llm(provider="anthropic", model="claude-sonnet-5") + assert _node_tier(node) == "nova" + + def test_undeclared_anthropic_model_does_not_match_any_tier(self): + node = _llm(provider="anthropic", model="claude-sonnet-4-5") + assert _node_tier(node) is None + + def test_build_attempt_manifest_accepts_nova_tier_end_to_end(self): + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="nova", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + assert manifest["tier"] == "nova" + assert manifest["nodes"]["planner"]["resolved_tier"] == "nova" + assert manifest["untiered_enabled_nodes"] == [] + + def test_build_attempt_manifest_rejects_unknown_tier(self): + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + with pytest.raises(ValueError, match="Unknown tier"): + build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="mercury", # type: ignore[arg-type] + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + + def test_enabled_utils_nodes_get_same_tier_matching_as_agent_nodes(self): + """outputter/hopper are the two required (non-nullable) utils nodes; + configuring them must give them the exact same enabled/tier-matched + treatment as a required agent node -- not a second-class entry. + """ + config = _uniform_config(provider="anthropic", model="claude-sonnet-5") + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="nova", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + for util in ("outputter", "hopper"): + entry = manifest["utils"][util] + assert entry["enabled"] is True + assert entry["resolved_tier"] == "nova" + assert entry["config"]["provider"] == "anthropic" + assert entry["config"]["model"] == "claude-sonnet-5" + assert manifest["untiered_enabled_nodes"] == [] + + +class TestNullableNodesRepresentedExplicitly: + def test_disabled_nodes_show_would_resolve_default_without_being_active(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + pixel_safety = manifest["nodes"]["validator_pixel_safety_net"] + assert pixel_safety["enabled"] is False + assert pixel_safety["config"] is None + assert pixel_safety["would_resolve_to"]["provider"] == "google" + assert pixel_safety["would_resolve_to"]["model"] == "gemini-3.5-flash-lite" + + history_analyzer = manifest["nodes"]["history_analyzer"] + assert history_analyzer["enabled"] is False + assert history_analyzer["would_resolve_provenance"] == "soft-default: inherits 'operator'" + + video_analyzer = manifest["utils"]["video_analyzer"] + assert video_analyzer["enabled"] is False + assert video_analyzer["would_resolve_to"] is None + assert "no soft default" in video_analyzer["would_resolve_provenance"] + + def test_fake_llm_flag_and_credentials_never_leak_secrets(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={"OPENAI_API_KEY": "sk-supersecretvalue1234", "ARTEMIS_FAKE_LLM": "0"}, + checkpoint="launch", + now=1.0, + ) + assert manifest["fake_llm_enabled"] is False + raw = canonical_bytes(manifest) + assert b"sk-supersecretvalue1234" not in raw + assert manifest["credentials"]["openai"]["set"] is True + assert manifest["credentials"]["openai"]["last4"] == "1234" + + def test_credential_shaped_artemis_env_vars_never_leak_raw_value(self): + """ARTEMIS_TENANT_TOKEN (a real secret-bearing var, see + artemis.config.constants.ENV_ARTEMIS_TENANT_TOKEN) must never appear + by raw value in env_overrides or the canonical bytes -- only a + set/last4 presence reference, exactly like _credential_reference. + """ + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={ + "ARTEMIS_TENANT_TOKEN": "super-secret-tenant-token-value", + "ARTEMIS_LIFECYCLE_TOKEN": "another-secret-lifecycle-value", + "ARTEMIS_FAKE_LLM": "0", + "ARTEMIS_CONFIG_DIR": "/tmp/does-not-matter", + }, + checkpoint="launch", + now=1.0, + ) + raw = canonical_bytes(manifest) + assert b"super-secret-tenant-token-value" not in raw + assert b"another-secret-lifecycle-value" not in raw + + tenant_ref = manifest["env_overrides"]["ARTEMIS_TENANT_TOKEN"] + assert tenant_ref == {"set": True, "last4": "alue"} + lifecycle_ref = manifest["env_overrides"]["ARTEMIS_LIFECYCLE_TOKEN"] + assert lifecycle_ref == {"set": True, "last4": "alue"} + + # Non-sensitive ARTEMIS_* resolution knobs still pass through as raw + # values -- this fix must not overcorrect into hashing everything. + assert manifest["env_overrides"]["ARTEMIS_CONFIG_DIR"] == "/tmp/does-not-matter" + assert manifest["env_overrides"]["ARTEMIS_FAKE_LLM"] == "0" + + def test_unset_credential_shaped_env_var_reports_set_false_without_last4(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="t", + tier="sol", + llm_config=_sol_config(), + env={"ARTEMIS_TENANT_TOKEN": ""}, + checkpoint="launch", + now=1.0, + ) + assert manifest["env_overrides"]["ARTEMIS_TENANT_TOKEN"] == {"set": False} + + +class TestStorageCreateOnly: + def test_store_and_read_back_bytes_match(self, tmp_path): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="trace-store-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + path, digest = store_attempt_manifest(manifest) + assert path.exists() + assert digest == digest_of(manifest) + + stored_bytes, stored_digest = read_stored_manifest("trace-store-1", "launch") + assert stored_bytes == canonical_bytes(manifest) + assert stored_digest == digest + + def test_second_write_for_same_trace_and_checkpoint_is_rejected(self): + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a", + trace_id="trace-store-2", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + store_attempt_manifest(manifest) + with pytest.raises(ManifestAlreadyExistsError): + store_attempt_manifest(manifest) + + def test_concurrent_writers_never_overwrite_each_others_bytes(self): + """Reproduces the TOCTOU this storage must not have. + + A prior implementation checked ``Path.exists()`` then wrote via + ``os.replace()`` — a second writer landing between those two steps + silently clobbered the first writer's already-stored bytes instead of + raising. This drives two logically different manifests (same + trace/checkpoint key, different ``run_id``) through + ``store_attempt_manifest`` back-to-back and asserts the file on disk + still holds exactly the first writer's bytes, with the second writer + observing ``ManifestAlreadyExistsError`` rather than winning a race. + """ + first = build_attempt_manifest( + run_id="first-writer", + attempt_id="a", + trace_id="trace-race-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + second = build_attempt_manifest( + run_id="second-writer", + attempt_id="a", + trace_id="trace-race-1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=2.0, + ) + assert canonical_bytes(first) != canonical_bytes(second) + + first_path, first_digest = store_attempt_manifest(first) + with pytest.raises(ManifestAlreadyExistsError): + store_attempt_manifest(second) + + stored_bytes, stored_digest = read_stored_manifest("trace-race-1", "launch") + assert stored_bytes == canonical_bytes(first) + assert stored_digest == first_digest + assert first_path.read_bytes() == canonical_bytes(first) + + def test_two_threads_racing_the_same_key_leave_exactly_one_winner(self): + """True concurrent reproduction: two threads, one (trace_id, checkpoint). + + Exactly one thread's bytes must land on disk and the other must + observe ``ManifestAlreadyExistsError`` — never a torn write and never + a silent overwrite of the winner by the loser. + """ + manifest_by_run_id = { + run_id: build_attempt_manifest( + run_id=run_id, + attempt_id="a", + trace_id="trace-race-2", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=float(i), + ) + for i, run_id in enumerate(("racer-a", "racer-b")) + } + results: dict[str, tuple[str, str] | Exception] = {} + start = threading.Barrier(2) + + def _race(run_id: str) -> None: + start.wait() + try: + results[run_id] = store_attempt_manifest(manifest_by_run_id[run_id]) + except ManifestAlreadyExistsError as exc: + results[run_id] = exc + + threads = [threading.Thread(target=_race, args=(rid,)) for rid in manifest_by_run_id] + for t in threads: + t.start() + for t in threads: + t.join() + + winners = [rid for rid, res in results.items() if not isinstance(res, Exception)] + losers = [rid for rid, res in results.items() if isinstance(res, Exception)] + assert len(winners) == 1 + assert len(losers) == 1 + assert isinstance(results[losers[0]], ManifestAlreadyExistsError) + + winning_manifest = manifest_by_run_id[winners[0]] + stored_bytes, stored_digest = read_stored_manifest("trace-race-2", "launch") + assert stored_bytes == canonical_bytes(winning_manifest) + assert stored_digest == digest_of(winning_manifest) + + def test_different_checkpoints_for_same_trace_coexist(self): + base_kwargs = dict( + run_id="r", + attempt_id="a", + trace_id="trace-store-3", + tier="sol", + llm_config=_sol_config(), + env={}, + ) + launch = build_attempt_manifest(**base_kwargs, checkpoint="launch", now=1.0) + worker_start = build_attempt_manifest(**base_kwargs, checkpoint="worker_start", now=2.0) + termination = build_attempt_manifest(**base_kwargs, checkpoint="termination", now=3.0) + + store_attempt_manifest(launch) + store_attempt_manifest(worker_start) + store_attempt_manifest(termination) + + for checkpoint in ("launch", "worker_start", "termination"): + assert read_stored_manifest("trace-store-3", checkpoint) is not None + + +class TestDecoyConfigIsolation: + """Requirement 3(b): decoy higher-priority config file must not leak + across independently-resolved attempts, and a stored manifest must never + change after the fact. + + This exercises the *env-var-driven config resolution layer in-process*: + ``build_attempt_manifest`` never reads ``os.environ`` itself, so the + "decoy" here is simulated purely via the explicit ``env`` dict and an + isolated ``tmp_path`` config directory -- the real ``get_config_path()`` + cascade, the real user home directory, and the real MCP/device process + boundary are NOT exercised by this test (see the PR description / final + report for the explicit isolation-boundary callout). + """ + + def test_attempt_a_manifest_and_digest_survive_later_decoy_mutation(self, tmp_path): + decoy_dir = tmp_path / "decoy-config-dir" + decoy_dir.mkdir() + decoy_override = decoy_dir / "llm-config.override.jsonc" + decoy_override.write_text('{"nodes": {}}', encoding="utf-8") + + # Attempt A resolves using an explicit env snapshot that "points at" + # the decoy dir (simulating ARTEMIS_CONFIG_DIR) plus a config object + # already fully resolved for tier 'sol'. Nothing here reads the decoy + # file's contents -- build_attempt_manifest only ever consumes + # explicit llm_config/env/tier, which is exactly the property under + # test: an env var *naming* a decoy path cannot retroactively alter a + # manifest that was built from a different, already-resolved config. + env_pointing_at_decoy = {"ARTEMIS_CONFIG_DIR": str(decoy_dir)} + manifest_a = build_attempt_manifest( + run_id="run-iso", + attempt_id="attempt-a", + trace_id="trace-iso-a", + tier="sol", + llm_config=_sol_config(), + env=env_pointing_at_decoy, + checkpoint="launch", + now=10.0, + ) + path_a, digest_a = store_attempt_manifest(manifest_a) + bytes_a_before = path_a.read_bytes() + + # Mutate the decoy file to something that would resolve to a + # different (mixed) tier if it were ever loaded and passed in. + decoy_override.write_text( + '{"nodes": {"planner": {"provider": "anthropic", "model": "claude-x"}}}', + encoding="utf-8", + ) + + # Attempt B resolves independently -- its own explicit llm_config, + # built fresh, deliberately picking a different tier to prove + # resolution is not sticky/global. + manifest_b = build_attempt_manifest( + run_id="run-iso", + attempt_id="attempt-b", + trace_id="trace-iso-b", + tier="luna", + llm_config=_uniform_config(provider="google", model="gemini-3.5-flash-lite"), + env={"ARTEMIS_CONFIG_DIR": str(decoy_dir)}, + checkpoint="launch", + now=20.0, + ) + store_attempt_manifest(manifest_b) + + # Attempt A's already-stored manifest is untouched, byte-for-byte. + bytes_a_after = path_a.read_bytes() + assert bytes_a_after == bytes_a_before + _, digest_a_reread = read_stored_manifest("trace-iso-a", "launch") + assert digest_a_reread == digest_a + + # Attempt B resolved to its own distinct tier/digest -- the decoy + # mutation influenced neither attempt (A never re-read it; B never + # read it at all, since resolution is a pure function of the + # explicit llm_config argument). + assert manifest_a["tier"] == "sol" + assert manifest_b["tier"] == "luna" + assert digest_of(manifest_a) != digest_of(manifest_b) + + # Never touched a real home directory or the real PAL config dir. + assert str(decoy_dir).startswith(str(tmp_path)) diff --git a/tests/unit/config/test_attempt_reconciliation.py b/tests/unit/config/test_attempt_reconciliation.py new file mode 100644 index 00000000..3385791c --- /dev/null +++ b/tests/unit/config/test_attempt_reconciliation.py @@ -0,0 +1,780 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate 1 mechanism tests: llm_usage <-> manifest reconciliation and the +seven-reason batch accept/reject rule. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from artemis.config.attempt_manifest import ( + build_attempt_manifest, + digest_of, + store_attempt_manifest, +) +from artemis.config.attempt_reconciliation import ( + AttemptRecord, + REJECT_REASONS, + reconcile_attempt, + reconcile_attempt_batch_by_run_id, + reconcile_finished_attempt, + validate_batch, +) +from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.data_engine.models import SessionMetadata, TraceRecord +from artemis.data_engine.storage import StorageManager +from artemis.runtime import trace_store + + +def _llm(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMWithFallback: + return LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + + +_REQUIRED_NODES = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", +) + + +def _uniform_config(provider: str = "openai", model: str = "gpt-5.6-sol") -> LLMConfig: + """A fully-populated LLMConfig where every required node resolves to one tier.""" + node = _llm(provider, model) + return LLMConfig( + **{n: node for n in _REQUIRED_NODES}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +def _sol_config() -> LLMConfig: + return _uniform_config() + + +def _manifest(**overrides): + kwargs = dict( + run_id="r", + attempt_id="a1", + trace_id="t1", + tier="sol", + llm_config=_sol_config(), + env={}, + checkpoint="launch", + now=1.0, + ) + kwargs.update(overrides) + return build_attempt_manifest(**kwargs) + + +def _usage(node: str, source: str | None = "openai:gpt-5.6-sol") -> dict: + event = {"node": node, "prompt_tokens": 10, "completion_tokens": 5} + if source is not None: + event["source"] = source + return event + + +class TestReconciliation: + def test_matching_source_is_match(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner")]) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "match" + assert not result.has_mismatch + + def test_mismatched_source_flags_mismatch(self): + manifest = _manifest() + result = reconcile_attempt( + "a1", manifest, [_usage("planner", source="google:gemini-3.5-flash-lite")] + ) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "mismatch" + assert result.has_mismatch + + def test_missing_source_is_unverified_identity_not_inferred(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner", source=None)]) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "unverified_identity" + assert result.has_unverified_identity + + def test_usage_for_node_absent_from_manifest_is_unmapped_call(self): + manifest = _manifest() + result = reconcile_attempt( + "a1", manifest, [_usage("some_new_node_not_in_manifest", source="openai:gpt-5.6-sol")] + ) + unmapped = next(n for n in result.nodes if n.node == "some_new_node_not_in_manifest") + assert unmapped.verdict == "unmapped_call" + assert result.has_unmapped_call + + def test_utils_node_usage_reconciles_as_match_not_unmapped_call(self): + """outputter/hopper are declared in manifest["utils"], not manifest["nodes"]. + + reconcile_attempt merges both dicts by node name before matching, so a + real llm_usage event recorded under node="outputter" (e.g. the Flash + runner's Outputter synthesis call, which fires on essentially every + real run) must reconcile as "match" against the manifest's utils + entry -- never as unmapped_call just because it lives under a + different top-level manifest key than the twelve agent nodes. + """ + manifest = _manifest() + result = reconcile_attempt( + "a1", + manifest, + [_usage("outputter"), _usage("hopper")], + ) + outputter = next(n for n in result.nodes if n.node == "outputter") + hopper = next(n for n in result.nodes if n.node == "hopper") + assert outputter.verdict == "match" + assert hopper.verdict == "match" + assert not result.has_unmapped_call + + def test_enabled_node_never_invoked_is_not_invoked_informational(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, []) + planner = next(n for n in result.nodes if n.node == "planner") + assert planner.verdict == "not_invoked" + + def test_disabled_node_never_invoked_is_disabled_not_invoked(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, []) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "disabled_not_invoked" + + def test_never_fabricates_zero_usage_for_non_firing_node(self): + manifest = _manifest() + result = reconcile_attempt("a1", manifest, [_usage("planner")]) + # Every other enabled node with no usage event is not_invoked, never + # synthesized as a zero-usage "match". + others = [n for n in result.nodes if n.node != "planner" and n.manifest_enabled] + assert others + assert all(n.verdict == "not_invoked" for n in others) + + def test_aliased_trace_node_name_reconciles_against_manifest_node(self): + """validator_pixel_safety_net is traced under "safety_net_pixel_validation" + (see artemis.agents.validator.validator._validate_action_precondition_pixel's + @trace name), so a usage event recorded with that trace-scope node + name must reconcile as a match against the manifest's + validator_pixel_safety_net entry, not surface as unmapped_call. + """ + # Enable validator_pixel_safety_net explicitly so it has a real + # manifest entry to reconcile against. + config = _uniform_config() + config = config.model_copy(update={"validator_pixel_safety_net": _llm()}) + manifest = build_attempt_manifest( + run_id="r", + attempt_id="a1", + trace_id="t1", + tier="sol", + llm_config=config, + env={}, + checkpoint="launch", + now=1.0, + ) + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="openai:gpt-5.6-sol")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "match" + assert not result.has_unmapped_call + # No stray "safety_net_pixel_validation" entry should remain unmapped. + assert not any(n.node == "safety_net_pixel_validation" for n in result.nodes) + + def test_soft_defaulted_node_with_would_resolve_to_match_reconciles_as_match(self): + """validator_pixel_safety_net left unset (the normal/default state) is + soft-defaulted by LLMConfig.get_agent() to lightweight_judge_default() + (google:gemini-3.5-flash-lite, a Luna-tier model) when actually + invoked. The manifest entry is `enabled=False` (correctly recording it + wasn't explicitly configured) but carries `would_resolve_to` for + exactly this case. This is the *legitimate* positive case: a manifest + whose own declared `tier` genuinely IS "luna" (i.e. its enabled nodes + already resolve to Luna-tier models), so the soft default's tier + agrees with the attempt's own tier. A receipt matching + `would_resolve_to` here is a legitimate, expected production + occurrence and must reconcile as "match", not "mismatch". + """ + luna_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + manifest = _manifest(tier="luna", llm_config=luna_config) + node_entry = manifest["nodes"]["validator_pixel_safety_net"] + assert node_entry["enabled"] is False + assert node_entry["would_resolve_to"] == { + "provider": "google", + "model": "gemini-3.5-flash-lite", + "api_base": None, + "temperature": 0.0, + "thinking_budget": None, + "thinking_level": None, + "reasoning_effort": None, + "include_thoughts": None, + "enable_grounding": None, + "fallback": { + "provider": "google", + "model": "gemini-3.1-flash-lite", + "api_base": None, + "temperature": 0.0, + "thinking_budget": None, + "thinking_level": None, + "reasoning_effort": None, + "include_thoughts": None, + "enable_grounding": None, + }, + "fix_model": None, + "timeout": None, + } + + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "match" + # The manifest's own `enabled` field must stay honest: this node was + # not explicitly configured, even though the receipt matched. + assert pixel_safety.manifest_enabled is False + + record = _record( + manifest, [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")] + ) + verdict = validate_batch([record]) + assert verdict.accepted is True + + def test_soft_defaulted_node_cross_tier_would_resolve_to_is_rejected_not_matched(self): + """Negative control for the cross-tier-mixing defect: a `tier="sol"` + manifest's soft-defaulted validator_pixel_safety_net still would + resolve to a Luna-tier model (google:gemini-3.5-flash-lite) if + invoked, since would_resolve_to is a fixed function of the node, not + of the attempt's own declared tier. A receipt matching that Luna-tier + would_resolve_to must NOT reconcile as "match" against a Sol-pinned + attempt -- that would let an attempt pinned to Sol tier silently + invoke a Luna-tier model via the soft-default path. It must reconcile + as "mismatch", and the batch must be rejected, not accepted. + """ + manifest = _manifest(tier="sol") # validator_pixel_safety_net left None (default). + node_entry = manifest["nodes"]["validator_pixel_safety_net"] + assert node_entry["enabled"] is False + assert node_entry["would_resolve_to"]["provider"] == "google" + assert node_entry["would_resolve_to"]["model"] == "gemini-3.5-flash-lite" + + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "mismatch" + assert result.has_mismatch + + record = _record( + manifest, [_usage("safety_net_pixel_validation", source="google:gemini-3.5-flash-lite")] + ) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_soft_defaulted_node_with_mismatched_source_still_reconciles_as_mismatch(self): + """Negative control: this fix must not turn off legitimate mismatch + detection for soft-defaulted nodes -- only make the *expected* source + honest (derived from would_resolve_to instead of always None).""" + manifest = _manifest() # validator_pixel_safety_net left None (default). + result = reconcile_attempt( + "a1", + manifest, + [_usage("safety_net_pixel_validation", source="openai:gpt-5.6-sol")], + ) + pixel_safety = next(n for n in result.nodes if n.node == "validator_pixel_safety_net") + assert pixel_safety.verdict == "mismatch" + assert result.has_mismatch + + +_UNSET = object() + + +def _record(manifest, usage_events, digest=_UNSET) -> AttemptRecord: + reconciliation = reconcile_attempt(manifest["attempt_id"], manifest, usage_events) + return AttemptRecord( + attempt_id=manifest["attempt_id"], + manifest=manifest, + reconciliation=reconciliation, + stored_digest=digest_of(manifest) if digest is _UNSET else digest, + ) + + +class TestBatchValidationAllSevenReasons: + def test_empty_batch_is_missing_snapshot(self): + verdict = validate_batch([]) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" + + def test_accept_when_batch_is_clean(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.accepted is True + assert verdict.reason is None + assert verdict.invalid_attempts == () + + def test_missing_identity(self): + manifest = _manifest() + broken = copy.deepcopy(manifest) + broken["source_sha"] = "" + record = _record(broken, [_usage("planner")], digest=digest_of(broken)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_missing_identity_when_source_sha_is_unverified_sentinel(self): + """A truthy but never-git-verified source_sha (the "0"*40 fallback + _canonical_repo_sha emits when the tree isn't a real git checkout) + must not silently pass as a real identity.""" + manifest = _manifest() + unverified = copy.deepcopy(manifest) + unverified["source_sha"] = "0" * 40 + unverified["source_sha_provenance"] = "unknown-not-a-git-checkout" + record = _record(unverified, [_usage("planner")], digest=digest_of(unverified)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_missing_identity_when_untiered_enabled_node_present(self): + """An enabled node with no declared tier (custom/unknown-model) must + fail batch validation, even though it built successfully and didn't + trip the mixed_tier build-time check.""" + manifest = _manifest() + untiered = copy.deepcopy(manifest) + untiered["untiered_enabled_nodes"] = ["some_custom_node"] + record = _record(untiered, [_usage("planner")], digest=digest_of(untiered)) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_identity" + assert record in verdict.invalid_attempts + + def test_fake_mode_enabled(self): + manifest = _manifest(env={"ARTEMIS_FAKE_LLM": "1"}) + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "fake_mode_enabled" + + def test_missing_snapshot_when_no_stored_digest(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner")], digest=None) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" + + def test_digest_drift_when_manifest_mutated_after_storing(self): + manifest = _manifest() + original_digest = digest_of(manifest) + tampered = copy.deepcopy(manifest) + tampered["nodes"]["planner"]["config"]["model"] = "some-other-model" + record = _record(tampered, [_usage("planner")], digest=original_digest) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "digest_drift" + # The original (tampered) record is preserved for audit, not discarded. + assert verdict.invalid_attempts[0].manifest["nodes"]["planner"]["config"]["model"] == ( + "some-other-model" + ) + + def test_mixed_tier_across_batch(self): + sol_manifest = _manifest(tier="sol") + luna_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + luna_manifest = build_attempt_manifest( + run_id="r", + attempt_id="a2", + trace_id="t2", + tier="luna", + llm_config=luna_config, + env={}, + checkpoint="launch", + now=2.0, + ) + records = [ + _record(sol_manifest, [_usage("planner")]), + _record(luna_manifest, [_usage("planner", source="google:gemini-3.5-flash-lite")]), + ] + verdict = validate_batch(records) + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + assert len(verdict.invalid_attempts) == 2 + + def test_unmapped_call(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner"), _usage("totally_unknown_node")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unmapped_call" + + def test_unexpected_model_or_endpoint_from_mismatch(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner", source="anthropic:claude-x")]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_unexpected_model_or_endpoint_from_unverified_identity(self): + manifest = _manifest() + record = _record(manifest, [_usage("planner", source=None)]) + verdict = validate_batch([record]) + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + + def test_all_seven_reject_reasons_are_reachable_and_documented(self): + # Sanity check that the reasons enumerated in the spec are exactly + # what REJECT_REASONS declares -- catches drift if a reason is + # renamed in one place but not the other. + assert set(REJECT_REASONS) == { + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", + } + + def test_rejection_preserves_original_attempt_data_unmodified(self): + manifest = _manifest(env={"ARTEMIS_FAKE_LLM": "1"}) + record = _record(manifest, [_usage("planner")]) + verdict = validate_batch([record]) + assert verdict.invalid_attempts[0].manifest is manifest + assert verdict.invalid_attempts[0].reconciliation is record.reconciliation + + +@pytest.fixture +def _isolate_traces_dir(tmp_path, monkeypatch): + """Point trace_store at an isolated tmp_path directory for this test only. + + Never touches the process's real TRACES_DIR or the real home directory. + """ + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + + +def _seeded_db(tmp_path, session_id: str): + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + storage.create_session(SessionMetadata(session_id=session_id, initial_goal="test goal")) + return storage, db_path, traces_dir + + +class TestReconcileFinishedAttempt: + """Post-run orchestration: stored manifest + native DB receipts -> verdict. + + Uses only tmp_path-backed trace storage and a tmp_path SQLite DB seeded + via a writable StorageManager -- never the real traces directory or the + real DataEngine database. + """ + + def test_no_stored_manifest_returns_none_not_a_verdict(self, tmp_path, _isolate_traces_dir): + _storage, db_path, traces_dir = _seeded_db(tmp_path, "trace-no-manifest") + result = reconcile_finished_attempt( + trace_id="trace-no-manifest", + session_id="trace-no-manifest", + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + assert result is None + + def test_matching_usage_accepts_the_attempt(self, tmp_path, _isolate_traces_dir): + trace_id = "trace-accept" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + + storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{node}", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is True + + def test_mismatched_usage_rejects_with_original_receipts_preserved( + self, tmp_path, _isolate_traces_dir + ): + trace_id = "trace-reject" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + + storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + # planner fires with a model that does not match the manifest's + # pinned 'sol' identity -- a leaked/decoy config would look like this. + storage.create_trace( + TraceRecord( + trace_id="usage-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is False + assert verdict.reason == "unexpected_model_or_endpoint" + # The original mismatched receipt is retained on the reconciliation + # result, not discarded because the batch was rejected. + invalid = verdict.invalid_attempts[0] + planner_node = next(n for n in invalid.reconciliation.nodes if n.node == "planner") + assert planner_node.usage_sources == ("google:gemini-3.5-flash-lite",) + + def test_no_usage_events_at_all_is_not_invoked_not_a_fabricated_match( + self, tmp_path, _isolate_traces_dir + ): + """No DB rows for this session (e.g. the task crashed before any LLM + call) must reconcile every node as not_invoked/disabled_not_invoked, + never a synthesized match.""" + trace_id = "trace-no-usage" + manifest = _manifest(attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + _storage, db_path, traces_dir = _seeded_db(tmp_path, trace_id) + + verdict = reconcile_finished_attempt( + trace_id=trace_id, + session_id=trace_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + ) + + assert verdict is not None + assert verdict.accepted is True + + +class TestReconcileAttemptBatchByRunId: + """Cross-attempt batch reconciliation grouped by a shared run_id. + + This is the production-reachable home for the mixed_tier rejection path + (reconcile_finished_attempt can never exercise it since it always + validates a batch of exactly one attempt). Uses only tmp_path-backed + trace storage and a tmp_path SQLite DB -- never the real traces + directory or the real DataEngine database. + """ + + def test_two_attempts_same_run_id_same_tier_matching_usage_is_accepted( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-accept" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + for trace_id in ("attempt-1", "attempt-2"): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + manifest = _manifest(run_id=run_id, attempt_id=trace_id, trace_id=trace_id) + store_attempt_manifest(manifest) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-{node}", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is True + assert verdict.invalid_attempts == () + + def test_mixed_tier_across_attempts_sharing_run_id_is_rejected( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-mixed-tier" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + sol_trace_id = "attempt-sol" + storage.create_session(SessionMetadata(session_id=sol_trace_id, initial_goal="test goal")) + sol_manifest = _manifest(run_id=run_id, attempt_id=sol_trace_id, trace_id=sol_trace_id) + store_attempt_manifest(sol_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{sol_trace_id}-planner", + session_id=sol_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol"}, + ) + ) + + terra_trace_id = "attempt-terra" + storage.create_session(SessionMetadata(session_id=terra_trace_id, initial_goal="test goal")) + terra_config = _uniform_config(provider="google", model="gemini-3.8-flash") + terra_manifest = build_attempt_manifest( + run_id=run_id, + attempt_id=terra_trace_id, + trace_id=terra_trace_id, + tier="terra", + llm_config=terra_config, + env={}, + checkpoint="launch", + now=3.0, + ) + store_attempt_manifest(terra_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{terra_trace_id}-planner", + session_id=terra_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.8-flash"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + assert len(verdict.invalid_attempts) == 2 + invalid_ids = {a.attempt_id for a in verdict.invalid_attempts} + assert invalid_ids == {sol_trace_id, terra_trace_id} + + def test_attempt_under_different_run_id_is_excluded_from_batch( + self, tmp_path, _isolate_traces_dir + ): + run_id = "run-target" + other_run_id = "run-other" + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, traces_dir) + + target_trace_id = "attempt-target" + storage.create_session( + SessionMetadata(session_id=target_trace_id, initial_goal="test goal") + ) + target_manifest = _manifest( + run_id=run_id, attempt_id=target_trace_id, trace_id=target_trace_id + ) + store_attempt_manifest(target_manifest) + for node in _REQUIRED_NODES: + storage.create_trace( + TraceRecord( + trace_id=f"usage-{target_trace_id}-{node}", + session_id=target_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": node, "source": "openai:gpt-5.6-sol"}, + ) + ) + + # A different run_id, deliberately a different tier -- if this leaked + # into the target batch it would falsely trip mixed_tier. + other_trace_id = "attempt-other-run" + storage.create_session(SessionMetadata(session_id=other_trace_id, initial_goal="test goal")) + other_config = _uniform_config(provider="google", model="gemini-3.5-flash-lite") + other_manifest = build_attempt_manifest( + run_id=other_run_id, + attempt_id=other_trace_id, + trace_id=other_trace_id, + tier="luna", + llm_config=other_config, + env={}, + checkpoint="launch", + now=4.0, + ) + store_attempt_manifest(other_manifest) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{other_trace_id}-planner", + session_id=other_trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is True + assert len(verdict.invalid_attempts) == 0 + + def test_zero_matching_attempts_returns_missing_snapshot(self, tmp_path, _isolate_traces_dir): + traces_root = Path(trace_store.TRACES_DIR) + db_path = tmp_path / "usage.db" + traces_dir = tmp_path / "usage_traces" + + verdict = reconcile_attempt_batch_by_run_id( + run_id="run-does-not-exist", + checkpoint="launch", + db_path=db_path, + traces_dir=traces_dir, + traces_root=traces_root, + ) + assert verdict.accepted is False + assert verdict.reason == "missing_snapshot" diff --git a/tests/unit/config/test_attempt_usage_reader.py b/tests/unit/config/test_attempt_usage_reader.py new file mode 100644 index 00000000..b7c62461 --- /dev/null +++ b/tests/unit/config/test_attempt_usage_reader.py @@ -0,0 +1,155 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only ``llm_usage`` receipt lookup, direct from the DataEngine DB. + +All tests write into a ``tmp_path`` SQLite file via a writable +``StorageManager`` (never the real traces directory), then read it back +through :func:`read_llm_usage_events` exactly as the read-only offline path +would. +""" + +from __future__ import annotations + +import json + +from artemis.config.attempt_usage_reader import read_llm_usage_events +from artemis.data_engine.models import SessionMetadata, TraceRecord +from artemis.data_engine.storage import StorageManager + + +def _seeded_storage(tmp_path, session_id: str) -> tuple[StorageManager, str, str]: + db_path = tmp_path / "data_engine.db" + traces_dir = tmp_path / "traces" + storage = StorageManager(db_path, traces_dir) + storage.create_session(SessionMetadata(session_id=session_id, initial_goal="test goal")) + return storage, str(db_path), str(traces_dir) + + +class TestReadLlmUsageEvents: + def test_returns_only_llm_usage_rows_for_the_given_session(self, tmp_path): + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol", "prompt_tokens": 10}, + ) + ) + storage.create_trace( + TraceRecord( + trace_id="trace-2", + session_id="session-a", + type="tool", + name="ask_explorer", + payload={"args": {}, "result": "irrelevant, not an llm_usage trace"}, + ) + ) + # A different session's llm_usage row must never leak into session-a's read. + storage.create_session(SessionMetadata(session_id="session-b", initial_goal="other")) + storage.create_trace( + TraceRecord( + trace_id="trace-3", + session_id="session-b", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "google:gemini-3.5-flash-lite"}, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "planner" + assert events[0]["source"] == "openai:gpt-5.6-sol" + + def test_events_with_no_step_id_are_still_found(self, tmp_path): + """The whole point of querying `traces` directly instead of walking + steps: a background/utility llm_usage call recorded with no step_id + must not be silently missed.""" + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + step_id=None, + type="llm_call", + name="llm_usage", + payload={"node": "hopper", "source": "openai:gpt-5.6-sol"}, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "hopper" + + def test_missing_database_returns_empty_list_not_an_error(self, tmp_path): + events = read_llm_usage_events( + tmp_path / "does-not-exist.db", tmp_path / "traces", "session-a" + ) + assert events == [] + + def test_corrupt_payload_row_is_skipped_not_raised(self, tmp_path): + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": "openai:gpt-5.6-sol"}, + ) + ) + # Directly corrupt one row's payload to simulate a partially-written + # or truncated record, bypassing the Pydantic-validated write path. + with storage._get_connection() as conn: + conn.execute( + "INSERT INTO traces (trace_id, session_id, type, name, timestamp, status, payload) " + "VALUES ('trace-2', 'session-a', 'llm_call', 'llm_usage', 2.0, 'success', ?)", + ("{not valid json",), + ) + conn.commit() + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0]["node"] == "planner" + + def test_returned_payload_matches_original_bytes_exactly(self, tmp_path): + """Reconciliation must see the original receipt, not a derived summary.""" + storage, db_path, traces_dir = _seeded_storage(tmp_path, "session-a") + original_payload = { + "node": "operator", + "source": "openai:gpt-5.6-sol", + "prompt_tokens": 123, + "completion_tokens": 45, + "session_cached_ratio": 0.5, + } + storage.create_trace( + TraceRecord( + trace_id="trace-1", + session_id="session-a", + type="llm_call", + name="llm_usage", + payload=original_payload, + ) + ) + + events = read_llm_usage_events(db_path, traces_dir, "session-a") + + assert len(events) == 1 + assert events[0] == json.loads(json.dumps(original_payload)) diff --git a/tests/unit/config/test_qualification_manifest_che537.py b/tests/unit/config/test_qualification_manifest_che537.py new file mode 100644 index 00000000..1ddffa10 --- /dev/null +++ b/tests/unit/config/test_qualification_manifest_che537.py @@ -0,0 +1,170 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Structural checks for the CHE-537 pinned qualification manifest. + +These are plain dict/field assertions, not JSON Schema validation (no +`jsonschema` dependency introduced for one manifest). The manifest and its +schema doc are authored, not executed, by this issue -- these tests only +guard against the manifest silently drifting out of the shape the +qualification execution stage (CHE-540+) and the Gate 1 tier vocabulary +(PR #1 / CHE-491) depend on. +""" + +import json +from pathlib import Path +import re + +REPO_ROOT = Path(__file__).resolve().parents[3] +MANIFEST_PATH = REPO_ROOT / "qualification/manifests/pocket_actual_save_relaunch.v1.json" +JOURNEY_PATH = REPO_ROOT / "qualification/journeys/pocket_actual_save_relaunch.md" +EVIDENCE_SCHEMA_PATH = REPO_ROOT / "qualification/evidence_schema.v1.json" + +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + +# Must stay in lockstep with artemis/config/attempt_manifest.py's TIER_MODELS +# (PR #1 / CHE-491). This test intentionally hardcodes the same tuple rather +# than importing that (unmerged, draft) module, so it fails loudly if this +# manifest's tiers/providers/models drift from what Gate 1 actually declares. +_KNOWN_TIER_MODELS = { + "luna": ("google", "gemini-3.5-flash-lite"), + "terra": ("google", "gemini-3.8-flash"), + "sol": ("openai", "gpt-5.6-sol"), +} + + +def _load_manifest() -> dict: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def test_manifest_and_companion_files_exist(): + assert MANIFEST_PATH.is_file() + assert JOURNEY_PATH.is_file() + assert EVIDENCE_SCHEMA_PATH.is_file() + + +def test_fork_sha_is_a_valid_git_sha_and_matches_source_baseline(): + manifest = _load_manifest() + fork_sha = manifest["fork_sha"]["value"] + assert _SHA_RE.match(fork_sha), f"fork_sha must be a 40-hex-char git SHA, got {fork_sha!r}" + assert fork_sha == "64e1b3226b9553bdf4e5a368f14b8a40c0af6111", ( + "fork_sha must match the CHE-537 observed source baseline; if the pin " + "legitimately moved, this test and FORK_MAINTENANCE.md's promotion " + "record must be updated together, not silently." + ) + + +def test_candidate_app_fields_are_pinned_by_che540(): + """CHE-537 left these fields null as a hold; CHE-540 (manifest revision v2) + pins the exact pocket-actual build under test. If the pin legitimately + moves, this test and the manifest's candidate_app.note must be updated + together (revision_policy: a changed app_sha/apk_digest_sha256 forces a + new manifest revision and invalidates prior evidence). + """ + manifest = _load_manifest() + candidate = manifest["candidate_app"] + assert _SHA_RE.match(candidate["app_sha"]), ( + f"app_sha must be a 40-hex-char git SHA, got {candidate['app_sha']!r}" + ) + assert candidate["app_sha"] == "2c42481778e6f9b88a9502f55decdbe4e1a76e83" + assert _SHA256_RE.match(candidate["apk_digest_sha256"]), ( + f"apk_digest_sha256 must be a 64-hex-char sha256 digest, got {candidate['apk_digest_sha256']!r}" + ) + assert candidate["apk_digest_sha256"] == ( + "04795d5f3995c8e895f6440a9763c8a003c8b6ff955541e3965f5e71564e2e12" + ) + assert candidate["package_name"] == "dev.cheese.pocketactual" + + +def test_dependency_lock_digest_is_pinned_by_che540(): + manifest = _load_manifest() + digest = manifest["dependency_lock"]["digest_sha256"] + assert _SHA256_RE.match(digest), ( + f"digest_sha256 must be a 64-hex-char sha256 digest, got {digest!r}" + ) + assert digest == "2c63863dd5827482d7e206a887219c6438ce61dc494d9cc596f862378decc19a" + + +def test_model_targets_match_gate1_tier_vocabulary(): + manifest = _load_manifest() + targets = manifest["model_targets"] + assert len(targets) >= 2, "CHE-388 requires baselining at least two model tiers" + for target in targets: + tier = target["tier"] + assert tier in _KNOWN_TIER_MODELS, f"unknown tier {tier!r} not in Gate 1 TIER_MODELS" + expected_provider, expected_model = _KNOWN_TIER_MODELS[tier] + assert target["provider"] == expected_provider + assert target["model"] == expected_model + assert target["provider_id"] == f"{expected_provider}:{expected_model}" + + +def test_model_targets_are_luna_and_terra_per_che388_plan_not_sol(): + manifest = _load_manifest() + tiers = {t["tier"] for t in manifest["model_targets"]} + assert tiers == {"luna", "terra"}, ( + "CHE-388's plan text says 'compare Luna and Terra model tiers'; " + "'sol' resolves to an OpenAI-backed model and the workspace announcement " + "bars all OpenAI routing indefinitely, so it must be excluded here, not pinned." + ) + excluded = manifest["sol_tier_excluded"] + assert excluded["tier"] == "sol" + assert "unblock_condition" in excluded + + +def test_fixture_counts_match_che388_plan_v2(): + manifest = _load_manifest() + fixture = manifest["fixture"] + assert fixture["positive_runs_required"] == 10 + assert fixture["negative_controls_required"] == 1 + + +def test_revision_policy_and_review_contract_present(): + manifest = _load_manifest() + assert "rule" in manifest["revision_policy"] + review = manifest["review_contract"] + assert review["no_self_review"] is True + assert "Opus AND Sol" in review["chain_as_originally_settled"] + assert "mbp-claude-sonnet-5" in review["candidate_reviewer_this_issue"] + + +def test_evidence_schema_declares_verdict_enum_covering_infra_and_device_failures(): + schema = json.loads(EVIDENCE_SCHEMA_PATH.read_text(encoding="utf-8")) + verdict_enum = set(schema["properties"]["verdict"]["enum"]) + assert verdict_enum == {"pass", "fail_assertion", "fail_infrastructure", "device_unavailable"} + + +def test_evidence_schema_reject_reasons_match_gate1_reconciliation_vocabulary(): + schema = json.loads(EVIDENCE_SCHEMA_PATH.read_text(encoding="utf-8")) + reject_reasons = set( + schema["properties"]["reconciliation"]["properties"]["reject_reasons"]["items"]["enum"] + ) + # Mirrors artemis/config/attempt_reconciliation.py's REJECT_REASONS (PR #1 / CHE-491). + assert reject_reasons == { + "missing_identity", + "unmapped_call", + "mixed_tier", + "unexpected_model_or_endpoint", + "fake_mode_enabled", + "digest_drift", + "missing_snapshot", + } + + +def test_journey_defines_fixture_reset_and_negative_control(): + text = JOURNEY_PATH.read_text(encoding="utf-8") + assert "pm clear" in text + assert "force-stop" in text + assert "Negative control" in text + assert "WRONG-SUFFIX" in text diff --git a/tests/unit/core/test_diagnostics.py b/tests/unit/core/test_diagnostics.py index 322c0f27..6d882616 100644 --- a/tests/unit/core/test_diagnostics.py +++ b/tests/unit/core/test_diagnostics.py @@ -408,6 +408,37 @@ async def test_credentials_probe_and_dynamic_update(): assert result.metadata["api_keys"]["google"] == "test_gemini_key_1234567890" +@pytest.mark.asyncio +async def test_credentials_probe_ignores_placeholder_openai_endpoint(monkeypatch): + from artemis.config import settings + + for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY"): + monkeypatch.setattr(settings, key, None) + monkeypatch.setenv("OPENAI_BASE_URL", "") + + result = await LLMCredentialsProbe().probe() + + assert not any( + entry["raw_key"] == "" for entry in result.metadata["providers"] + ) + + +@pytest.mark.asyncio +async def test_credentials_probe_reports_keyless_openai_endpoint(monkeypatch): + from artemis.config import settings + + for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY"): + monkeypatch.setattr(settings, key, None) + monkeypatch.setenv("OPENAI_BASE_URL", "https://openai-proxy.example/v1") + + result = await LLMCredentialsProbe().probe() + + assert any( + entry["provider"] == "custom" and entry["raw_key"] == "https://openai-proxy.example/v1" + for entry in result.metadata["providers"] + ) + + @pytest.mark.asyncio async def test_emulator_manager_lifecycle(): """Verify EmulatorManager status querying, validation, and dismissal.""" diff --git a/tests/unit/mcp/test_background_task_runner.py b/tests/unit/mcp/test_background_task_runner.py index c0fbf7c8..ec5156d4 100644 --- a/tests/unit/mcp/test_background_task_runner.py +++ b/tests/unit/mcp/test_background_task_runner.py @@ -1,9 +1,15 @@ import asyncio +import json from unittest.mock import AsyncMock, MagicMock import pytest -from mcp_server.background.task_runner import _initialize_agent +from mcp_server.background.task_runner import ( + _build_arg_parser, + _initialize_agent, + _reconcile_and_store_verdict, + _record_attempt_manifest, +) @pytest.mark.asyncio @@ -102,3 +108,277 @@ async def test_run_task_applies_pro_tuning_to_agent_config( fake_builder.with_explorer.assert_called_once_with(pro_mode=expect_mode) fake_agent.run_task.assert_awaited_once() assert trace_store.read_status(trace_id)["status"] == "completed" + + +def _uniform_llm_config(provider: str, model: str): + """A fully-populated LLMConfig where every required node resolves to one tier. + + Mirrors ``tests/unit/config/test_attempt_reconciliation.py``'s + ``_uniform_config`` helper -- kept local to this file since it exercises + the real ``LLMConfig``, not a mock, so ``_record_attempt_manifest`` can + genuinely infer a tier from ``llm_config.planner`` the same way the real + ``run_task`` path does. + """ + from artemis.config.llm import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback + + node = LLMWithFallback( + provider=provider, model=model, fallback=LLM(provider=provider, model=model) + ) + required_nodes = ( + "planner", + "summarizer", + "operator", + "operator_summarizer", + "log_reader_sub_agent", + "log_analyzer", + "diagnoser", + "checker", + "planner_avatar", + "history_analyzer_expert", + "diagnoser_expert", + "explorer", + ) + return LLMConfig( + **{n: node for n in required_nodes}, + utils=LLMConfigUtils(outputter=node, hopper=node), + ) + + +class TestRealWorkerMultiAttemptRunIdGrouping: + """Proves the real ``run_task``-facing ``run_id`` plumbing (not a + synthetic ``build_attempt_manifest`` call) actually produces manifests + that ``reconcile_attempt_batch_by_run_id`` groups together and rejects on + ``mixed_tier`` -- the "real worker output reaches validate_batch" gap + ``TestReconcileAttemptBatchByRunId`` (config-level, synthetic manifests + only) cannot close on its own. + + Calls the real ``_record_attempt_manifest`` (the exact function + ``run_task`` calls, with the same ``run_id`` parameter ``run_task`` now + threads through to it) directly, twice, with a shared ``run_id`` but + distinct ``trace_id``s and tiers -- mirroring this file's existing + tmp_path-isolated ``TRACES_DIR`` pattern rather than driving the full + (heavily mocked) SDK agent path a second time in one test. + """ + + def test_two_real_attempts_sharing_run_id_with_different_tiers_is_rejected_mixed_tier( + self, tmp_path, monkeypatch + ): + from pathlib import Path + + from artemis.config.attempt_reconciliation import reconcile_attempt_batch_by_run_id + from artemis.data_engine.models import SessionMetadata, TraceRecord + from artemis.data_engine.storage import StorageManager + from artemis.runtime import trace_store + + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + run_id = "shared-retry-run-id" + sol_trace_id = "real-attempt-sol" + terra_trace_id = "real-attempt-terra" + + sol_llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + terra_llm_config = _uniform_llm_config("google", "gemini-3.8-flash") + + # The exact call run_task makes at its "launch" checkpoint, invoked + # twice with the same run_id -- this is the real plumbing added to + # thread run_id through, not a direct build_attempt_manifest() call. + _record_attempt_manifest( + trace_id=sol_trace_id, checkpoint="launch", llm_config=sol_llm_config, run_id=run_id + ) + _record_attempt_manifest( + trace_id=terra_trace_id, + checkpoint="launch", + llm_config=terra_llm_config, + run_id=run_id, + ) + + db_path = tmp_path / "usage.db" + usage_traces_dir = tmp_path / "usage_traces" + storage = StorageManager(db_path, usage_traces_dir) + for trace_id, source in ( + (sol_trace_id, "openai:gpt-5.6-sol"), + (terra_trace_id, "google:gemini-3.8-flash"), + ): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": source}, + ) + ) + + verdict = reconcile_attempt_batch_by_run_id( + run_id=run_id, + checkpoint="launch", + db_path=db_path, + traces_dir=usage_traces_dir, + traces_root=Path(trace_store.TRACES_DIR), + ) + + assert verdict.accepted is False + assert verdict.reason == "mixed_tier" + invalid_ids = {a.attempt_id for a in verdict.invalid_attempts} + assert invalid_ids == {sol_trace_id, terra_trace_id} + + +def test_cli_arg_parser_accepts_run_id_flag(): + """The real `__main__` CLI surface must accept --run-id without exiting 2. + + Reproduces the exact repro from the wiring-gap review: the CLI parser + used to have no --run-id flag at all, so passing it would fail argparse + with "unrecognized arguments" (exit code 2) before run_task was ever + called. Exercises _build_arg_parser() directly -- the same + ArgumentParser the `if __name__ == "__main__":` block constructs -- so + this proves the CLI surface itself, not just the in-process run_task + Python parameter. + """ + parser = _build_arg_parser() + + args = parser.parse_args( + [ + "--trace-id", + "trace-abc", + "--task-desc", + "Do the thing", + "--model", + "Flash", + "--run-id", + "shared-run-xyz", + ] + ) + + assert args.run_id == "shared-run-xyz" + + # Omitting --run-id must still parse cleanly and default to None, so an + # unmodified CLI invocation retains today's exact 1:1 run_id<->trace_id + # behavior in run_task. + args_no_run_id = parser.parse_args( + ["--trace-id", "trace-abc", "--task-desc", "Do the thing", "--model", "Flash"] + ) + assert args_no_run_id.run_id is None + + +class TestTerminationTimeBatchReconciliation: + """Proves _reconcile_and_store_verdict -- the function the finally: block + in run_task actually calls at termination -- genuinely fires batch + reconciliation and rejects a real mixed-tier batch, not merely that + reconcile_attempt_batch_by_run_id exists and can be called directly (that + is already covered by TestRealWorkerMultiAttemptRunIdGrouping and the + config-level TestReconcileAttemptBatchByRunId tests). + + This closes the second half of the wiring gap: even with --run-id + plumbed through the CLI, nothing previously called the batch reconciler + at termination -- the finally: block always called single-attempt + reconciliation only. + """ + + def test_termination_reconciliation_writes_mixed_tier_batch_verdict( + self, tmp_path, monkeypatch + ): + from pathlib import Path + + from artemis.config.constants import ENV_ARTEMIS_TRACES_DIR + from artemis.data_engine.models import SessionMetadata, TraceRecord + from artemis.data_engine.storage import StorageManager + from artemis.runtime import trace_store + + traces_dir = tmp_path / "traces" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(traces_dir)) + # get_data_engine_db_path() / get_traces_dir() (called internally by + # _reconcile_and_store_verdict, not injectable) must resolve to the + # same tmp_path tree the manifests and llm_usage receipts below are + # written under. + monkeypatch.setenv(ENV_ARTEMIS_TRACES_DIR, str(traces_dir)) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + run_id = "shared-termination-run-id" + sol_trace_id = "termination-attempt-sol" + terra_trace_id = "termination-attempt-terra" + + sol_llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + terra_llm_config = _uniform_llm_config("google", "gemini-3.8-flash") + + # First attempt: same real _record_attempt_manifest call run_task + # makes at its "launch" checkpoint. + _record_attempt_manifest( + trace_id=sol_trace_id, checkpoint="launch", llm_config=sol_llm_config, run_id=run_id + ) + _record_attempt_manifest( + trace_id=terra_trace_id, + checkpoint="launch", + llm_config=terra_llm_config, + run_id=run_id, + ) + + from artemis.config.paths import get_data_engine_db_path + + db_path = get_data_engine_db_path() + storage = StorageManager(db_path, traces_dir) + for trace_id, source in ( + (sol_trace_id, "openai:gpt-5.6-sol"), + (terra_trace_id, "google:gemini-3.8-flash"), + ): + storage.create_session(SessionMetadata(session_id=trace_id, initial_goal="test goal")) + storage.create_trace( + TraceRecord( + trace_id=f"usage-{trace_id}-planner", + session_id=trace_id, + type="llm_call", + name="llm_usage", + payload={"node": "planner", "source": source}, + ) + ) + + # The exact call run_task's finally: block makes at termination for + # the second (terra) attempt, now with a real, shared run_id -- this + # is the wiring under test, not a direct + # reconcile_attempt_batch_by_run_id() call. + _reconcile_and_store_verdict(trace_id=terra_trace_id, checkpoint="launch", run_id=run_id) + + batch_verdict_path = ( + Path(trace_store.get_trace_dir(terra_trace_id)) + / "attempt_reconciliation_batch_verdict.json" + ) + assert batch_verdict_path.exists(), ( + "termination-time _reconcile_and_store_verdict did not write a batch verdict " + "file even though a real, non-default run_id was passed" + ) + + payload = json.loads(batch_verdict_path.read_text(encoding="utf-8")) + assert payload["accepted"] is False + assert payload["reason"] == "mixed_tier" + + def test_termination_reconciliation_skips_batch_when_run_id_matches_trace_id( + self, tmp_path, monkeypatch + ): + """Default single-attempt behavior (run_id omitted / equal to trace_id) + must not produce a batch verdict file at all -- proves the batch path + is genuinely additive and does not fire for today's unmodified callers. + """ + from pathlib import Path + + from artemis.config.constants import ENV_ARTEMIS_TRACES_DIR + from artemis.runtime import trace_store + + traces_dir = tmp_path / "traces" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(traces_dir)) + monkeypatch.setenv(ENV_ARTEMIS_TRACES_DIR, str(traces_dir)) + monkeypatch.setenv("ARTEMIS_FAKE_LLM", "0") + + trace_id = "solo-attempt" + llm_config = _uniform_llm_config("openai", "gpt-5.6-sol") + _record_attempt_manifest(trace_id=trace_id, checkpoint="launch", llm_config=llm_config) + + # No manifest for this trace_id sharing a run_id exists other than + # itself, and run_id defaults to trace_id inside + # _record_attempt_manifest -- mirrors an unmodified caller exactly. + _reconcile_and_store_verdict(trace_id=trace_id, checkpoint="launch", run_id=None) + + batch_verdict_path = ( + Path(trace_store.get_trace_dir(trace_id)) / "attempt_reconciliation_batch_verdict.json" + ) + assert not batch_verdict_path.exists() diff --git a/tests/unit/mcp/test_diagnose_tool.py b/tests/unit/mcp/test_diagnose_tool.py index cd4627f5..dde60bc6 100644 --- a/tests/unit/mcp/test_diagnose_tool.py +++ b/tests/unit/mcp/test_diagnose_tool.py @@ -857,6 +857,32 @@ def test_verify_credentials_calls_endpoint_providers_by_url(temp_trace_env): assert "base_url" not in calls["google"].kwargs +def test_verify_credentials_uses_provider_base_url(temp_trace_env): + validate = AsyncMock(return_value=(True, "verified")) + cred = _probe( + "gemini_api_key", + ProbeStatus.PASS, + category=ProbeCategory.CREDENTIALS, + metadata={ + "providers": [ + { + "provider": "anthropic", + "label": "Claude", + "raw_key": "SECRET-ANTHROPIC", + "base_url": "https://anthropic-proxy.example", + } + ], + "api_keys": {}, + }, + ) + probes = [p for p in _healthy_probes() if p.id != "gemini_api_key"] + [cred] + _run(probes, validate=validate, verify_credentials=True) + + call = validate.await_args + assert call.args == ("anthropic", "SECRET-ANTHROPIC") + assert call.kwargs["base_url"] == "https://anthropic-proxy.example" + + def test_invalid_primary_credential_blocks_and_redacts_message(temp_trace_env): async def _validate(provider, api_key, timeout=12.0): if provider == "google": diff --git a/tests/unit/memory/test_history_chunking.py b/tests/unit/memory/test_history_chunking.py index f24c6128..d24e9ff6 100644 --- a/tests/unit/memory/test_history_chunking.py +++ b/tests/unit/memory/test_history_chunking.py @@ -1005,7 +1005,7 @@ async def ainvoke(self, messages): assert service.get_summary("chunk:1-2") is None -def test_capsule_fallback_model_resolution_google_only_and_not_primary(): +def test_capsule_fallback_model_resolution_uses_configured_provider_and_not_primary(): mgr = HistoryChunkManager( capsule_service=StubCapsuleService(), chunking_config=SimpleNamespace( @@ -1027,8 +1027,7 @@ def ctx_with(provider, model): mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.6-flash")) == "gemini-3.6-flash" ) - # Non-google fallbacks cannot ride the raw google model path. - assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) is None + assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) == "gpt-4o-mini" # A fallback identical to the primary adds nothing. assert mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None diff --git a/tests/unit/runtime/test_daemon_client.py b/tests/unit/runtime/test_daemon_client.py index 67a90d3b..6f6750f9 100644 --- a/tests/unit/runtime/test_daemon_client.py +++ b/tests/unit/runtime/test_daemon_client.py @@ -163,6 +163,26 @@ def test_submit_task_to_daemon_pro_tuning_defaults_to_null(): assert data["explorer_mode"] is None +def test_submit_task_to_daemon_forwards_run_id(): + """A daemon-dispatched attempt must carry its Gate 1 run_id through /api/run, + the same way it already carries session_id -- otherwise the worker the + Daemon spawns has no batch-grouping key to record its attempt manifest + under (see artemis.config.attempt_lifecycle_hooks).""" + mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}') + with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen: + submit_task_to_daemon("goal", profile="flash", session_id="sess-1", run_id="batch-42") + data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8")) + assert data["run_id"] == "batch-42" + + +def test_submit_task_to_daemon_run_id_defaults_to_null(): + mock_resp = _daemon_ok_response(b'{"status": "queued", "tasks": [{"session_id": "s1"}]}') + with patch("urllib.request.urlopen", return_value=mock_resp) as mock_urlopen: + submit_task_to_daemon("goal", profile="flash", session_id="sess-1") + data = json.loads(mock_urlopen.call_args[0][0].data.decode("utf-8")) + assert data["run_id"] is None + + def test_submit_batch_to_daemon_forwards_pro_tuning_knobs(): from artemis.runtime.daemon_client import submit_batch_to_daemon diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 11e9077a..a50d5f74 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -15,11 +15,33 @@ """Unit tests for ARTEMIS Unified CLI application.""" from typer.testing import CliRunner +import pytest from artemis.interfaces.cli.main import app runner = CliRunner() +@pytest.mark.asyncio +async def test_execute_task_records_configuration_failure_in_existing_trace(tmp_path, monkeypatch): + import artemis.interfaces.cli.commands.run as run_module + from artemis.runtime import trace_store + + def fail_initialization(): + raise RuntimeError("Planner requires GOOGLE_API_KEY in .env") + + session_id = "configuration-failure" + monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path / "traces")) + trace_store.init_trace(session_id, "Open settings", "flash") + monkeypatch.setattr(run_module, "initialize_llm_config", fail_initialization) + + with pytest.raises(RuntimeError, match="GOOGLE_API_KEY"): + await run_module.execute_task("Open settings", session_id=session_id) + + status = trace_store.read_status(session_id) + assert status["status"] == "failed" + assert status["error"] == "Planner requires GOOGLE_API_KEY in .env" + + def test_cli_help(): """Verify top-level CLI help returns status 0 and lists core subcommands.""" result = runner.invoke(app, ["--help"]) diff --git a/tests/unit/test_credentials_validator.py b/tests/unit/test_credentials_validator.py index 8ae63d8a..86034496 100644 --- a/tests/unit/test_credentials_validator.py +++ b/tests/unit/test_credentials_validator.py @@ -78,3 +78,53 @@ async def test_validate_ocr_success(): is_valid, msg = await validate_api_key("ocr", "valid_ocr_key_12345") assert is_valid assert "verified successfully" in msg + + +@pytest.mark.parametrize( + ("provider", "setting", "configured_url", "base_url", "expected_url"), + [ + ( + "openai", + "OPENAI_BASE_URL", + "https://openai-config.example/v1", + None, + "https://openai-config.example/v1/models", + ), + ( + "openai", + "OPENAI_BASE_URL", + "https://openai-config.example/v1", + "https://openai-model.example/v1", + "https://openai-model.example/v1/models", + ), + ( + "anthropic", + "ANTHROPIC_BASE_URL", + "https://anthropic-config.example", + None, + "https://anthropic-config.example/v1/models", + ), + ( + "anthropic", + "ANTHROPIC_BASE_URL", + "https://anthropic-config.example", + "https://anthropic-model.example", + "https://anthropic-model.example/v1/models", + ), + ], +) +@pytest.mark.asyncio +async def test_validate_provider_url_precedence( + monkeypatch, provider, setting, configured_url, base_url, expected_url +): + from artemis.config import settings + + monkeypatch.setattr(settings, setting, configured_url) + with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get: + response = MagicMock(status_code=200) + mock_get.return_value = response + + valid, _ = await validate_api_key(provider, "test-key", base_url=base_url) + + assert valid + assert mock_get.await_args.args[0] == expected_url diff --git a/tests/unit/test_file_utils.py b/tests/unit/test_file_utils.py new file mode 100644 index 00000000..57573acd --- /dev/null +++ b/tests/unit/test_file_utils.py @@ -0,0 +1,20 @@ +from io import StringIO + +from artemis.utils.file import load_jsonc, strip_json_comments + + +def test_jsonc_comments_preserve_urls_and_comment_like_strings(): + text = r"""{ + // line comment + "api_base": "https://example.test/v1", + "literal": "// keep /* both */ and a quote: \\\"", + /* block comment */ + "enabled": true + }""" + + assert load_jsonc(StringIO(text)) == { + "api_base": "https://example.test/v1", + "literal": '// keep /* both */ and a quote: \\"', + "enabled": True, + } + assert "line comment" not in strip_json_comments(text) diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index c99b01d2..0ce9a916 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -14,7 +14,99 @@ """Unit tests for ModelFactory instantiation and grounding tool wiring.""" -from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider +from unittest.mock import patch + +import pytest + +from artemis.llm.router import ( + ModelEndpoint, + ModelFactory, + ModelProvider, + resolve_provider_base_url, +) + + +@pytest.mark.parametrize( + ("provider", "setting", "default_url"), + [ + (ModelProvider.GOOGLE, "GOOGLE_BASE_URL", None), + (ModelProvider.VERTEX_AI, "VERTEX_AI_BASE_URL", None), + (ModelProvider.OPENAI, "OPENAI_BASE_URL", None), + (ModelProvider.ANTHROPIC, "ANTHROPIC_BASE_URL", None), + (ModelProvider.OPENROUTER, "OPEN_ROUTER_BASE_URL", "https://openrouter.ai/api/v1"), + (ModelProvider.XAI, "XAI_BASE_URL", "https://api.x.ai/v1"), + (ModelProvider.OLLAMA, "OLLAMA_BASE_URL", "http://localhost:11434/v1"), + (ModelProvider.VLLM, "VLLM_BASE_URL", "http://localhost:8000/v1"), + (ModelProvider.CUSTOM, "CUSTOM_BASE_URL", "http://localhost:8000/v1"), + ], +) +def test_provider_base_url_precedence(monkeypatch, provider, setting, default_url): + """Every provider uses model endpoint, provider setting, then its native default.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example") + configured = ModelEndpoint(provider=provider, api_base="https://model-config.example") + assert resolve_provider_base_url(configured) == "https://model-config.example" + + assert ( + resolve_provider_base_url(ModelEndpoint(provider=provider)) + == "https://provider-config.example" + ) + + monkeypatch.setattr(settings, setting, " ") + assert resolve_provider_base_url(ModelEndpoint(provider=provider)) == default_url + + +@pytest.mark.parametrize( + ("provider", "constructor", "setting"), + [ + (ModelProvider.GOOGLE, "langchain_google_genai.ChatGoogleGenerativeAI", "GOOGLE_BASE_URL"), + (ModelProvider.VERTEX_AI, "langchain_google_vertexai.ChatVertexAI", "VERTEX_AI_BASE_URL"), + ], +) +def test_google_provider_constructors_receive_resolved_base_url( + monkeypatch, provider, constructor, setting +): + """Gemini and Vertex pass their selected custom service endpoint to LangChain.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example") + endpoint = ModelEndpoint( + provider=provider, + model_name="gemini-test", + api_key="test-key", + api_base="https://model-config.example", + ) + + with patch(constructor) as chat_model: + ModelFactory.create_model(endpoint) + + assert chat_model.call_args.kwargs["base_url"] == "https://model-config.example" + + +@pytest.mark.parametrize( + ("provider", "setting"), + [ + (ModelProvider.OPENROUTER, "OPEN_ROUTER_BASE_URL"), + (ModelProvider.XAI, "XAI_BASE_URL"), + (ModelProvider.OLLAMA, "OLLAMA_BASE_URL"), + (ModelProvider.VLLM, "VLLM_BASE_URL"), + (ModelProvider.CUSTOM, "CUSTOM_BASE_URL"), + ], +) +def test_openai_compatible_provider_constructors_receive_setting_url( + monkeypatch, provider, setting +): + """OpenAI-compatible providers pass their provider-specific configured URL.""" + from artemis.config import settings + + monkeypatch.setattr(settings, setting, "https://provider-config.example/v1") + endpoint = ModelEndpoint(provider=provider, model_name="test-model", api_key="test-key") + + with patch("langchain_openai.ChatOpenAI") as chat_model: + ModelFactory.create_model(endpoint) + + assert chat_model.call_args.kwargs["base_url"] == "https://provider-config.example/v1" def test_model_factory_anthropic_instantiation(): @@ -43,6 +135,116 @@ def test_model_factory_openai_instantiation(): assert getattr(model, "reasoning_effort", None) == "medium" +def test_model_factory_openai_endpoint_prefers_model_configuration(monkeypatch): + """A model-specific endpoint overrides the configured OpenAI default.""" + from artemis.config import settings + + monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://openai-default.example/v1") + endpoint = ModelEndpoint( + provider=ModelProvider.OPENAI, + model_name="gpt-test", + api_key="test-key", + api_base="https://openai-model.example/v1", + ) + + with patch("langchain_openai.ChatOpenAI") as chat_openai: + ModelFactory.create_model(endpoint) + + assert chat_openai.call_args.kwargs["base_url"] == "https://openai-model.example/v1" + + +@pytest.mark.parametrize( + "model_name", + ["claude-sonnet-5", "claude-opus-5", "claude-haiku-4-5", "claude-fable-5-1"], +) +def test_model_factory_anthropic_omits_temperature_for_rejecting_model(model_name): + """Models that 400 on `temperature` never receive the parameter, even with thinking.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name=model_name, + api_key="sk-ant-test-key", + reasoning_effort="high", + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert "temperature" not in chat_anthropic.call_args.kwargs + assert chat_anthropic.call_args.kwargs["thinking"] == { + "type": "enabled", + "budget_tokens": 32768, + } + + +def test_model_factory_anthropic_keeps_temperature_for_accepting_model(): + """Models that still accept `temperature` continue to receive the configured value.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-3-7-sonnet-20250219", + api_key="sk-ant-test-key", + temperature=0.3, + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert chat_anthropic.call_args.kwargs["temperature"] == 0.3 + + +def test_model_factory_anthropic_forces_thinking_temperature_when_accepted(): + """Thinking budget still forces temperature=1.0 on models that accept the parameter.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-3-7-sonnet-20250219", + api_key="sk-ant-test-key", + reasoning_effort="high", + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(endpoint) + + assert chat_anthropic.call_args.kwargs["temperature"] == 1.0 + + +def test_model_factory_anthropic_uses_configured_or_model_endpoint(monkeypatch): + """Anthropic receives the environment default unless the model specifies one.""" + from artemis.config import settings + + monkeypatch.setattr(settings, "ANTHROPIC_BASE_URL", "https://anthropic-default.example") + default_endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-test", + api_key="test-key", + ) + model_endpoint = default_endpoint.model_copy( + update={"api_base": "https://anthropic-model.example"} + ) + + with patch("langchain_anthropic.ChatAnthropic") as chat_anthropic: + ModelFactory.create_model(default_endpoint) + ModelFactory.create_model(model_endpoint) + + assert [call.kwargs["base_url"] for call in chat_anthropic.call_args_list] == [ + "https://anthropic-default.example", + "https://anthropic-model.example", + ] + + +def test_llm_config_api_base_reaches_model_endpoint(): + from artemis.config.llm import deep_merge_llm_config, get_default_llm_config + from artemis.services.llm import _resolve_endpoint + + endpoint_url = "https://openai-proxy.example/v1" + config = deep_merge_llm_config( + get_default_llm_config(), {"planner": {"api_base": endpoint_url}} + ) + + class Context: + llm_config = config + + assert _resolve_endpoint(Context(), "planner").api_base == endpoint_url + + def test_robust_chat_model_wrapper_grounding_google(): """Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini.""" from artemis.services.llm import RobustChatModelWrapper diff --git a/tests/unit/test_llm_router_anthropic_stream.py b/tests/unit/test_llm_router_anthropic_stream.py new file mode 100644 index 00000000..c94b604c --- /dev/null +++ b/tests/unit/test_llm_router_anthropic_stream.py @@ -0,0 +1,135 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression test for the Anthropic streaming `model_dump` crash (CHE-643). + +``langchain_anthropic``'s ``ChatAnthropic._make_message_chunk_from_anthropic_event`` +reads ``event.context_management`` off `message_delta` stream events and calls +``.model_dump()`` on it unconditionally. The Anthropic SDK's `RawMessageDeltaEvent` +has no typed field for `context_management` (that name is only declared on +`ChatAnthropic` itself, as a plain request-side dict), so when the API includes +that block in a response, Pydantic's `extra="allow"` stores it as a raw `dict` +instead of a submodel — and `.model_dump()` raises +`AttributeError: 'dict' object has no attribute 'model_dump'`, discarding the +whole in-flight response (see `artemis/services/llm.py::_complete_attempt`). +""" + +import inspect +from unittest.mock import patch + +from anthropic.types.raw_message_delta_event import Delta, RawMessageDeltaEvent +from anthropic.types.usage import Usage +import pytest + +from artemis.llm.router import _patch_anthropic_stream_context_management + + +def _unpatched_make_message_chunk(): + """The original langchain-anthropic method, regardless of test order. + + Other tests/modules construct real Anthropic models (e.g. + ``test_llm_grounding.py``), which applies this module's process-global + patch as a side effect. Walk the `__wrapped__`-style closure our patch + builds to recover the pristine method so this test proves the upstream + bug independent of suite ordering. + """ + from langchain_anthropic import ChatAnthropic + + current = ChatAnthropic._make_message_chunk_from_anthropic_event + closure = inspect.getclosurevars(current).nonlocals + return closure.get("original", current) + + +def _delta_event_with_raw_context_management() -> RawMessageDeltaEvent: + """Builds the exact malformed event shape the live Anthropic API sent.""" + delta = Delta.model_construct( + stop_reason="end_turn", + stop_sequence=None, + container=None, + ) + return RawMessageDeltaEvent.model_construct( + type="message_delta", + delta=delta, + usage=Usage(input_tokens=1, output_tokens=1), + context_management={"applied_edits": []}, + ) + + +def test_raw_dict_context_management_crashes_without_the_patch(): + """Reproduces the upstream bug so the workaround has something to guard against.""" + from langchain_anthropic import ChatAnthropic + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + event = _delta_event_with_raw_context_management() + unpatched = _unpatched_make_message_chunk() + + with patch.object(ChatAnthropic, "_make_message_chunk_from_anthropic_event", unpatched): + with pytest.raises(AttributeError, match="model_dump"): + model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + +def test_patch_normalizes_raw_dict_context_management(): + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + event = _delta_event_with_raw_context_management() + + chunk, _ = model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + assert chunk is not None + assert chunk.response_metadata["context_management"] == {"applied_edits": []} + + +def test_patch_normalizes_raw_dict_container(): + """The same unconditional `.model_dump()` pattern applies to `delta.container`.""" + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + + model = ChatAnthropic.model_construct(model="claude-sonnet-5") + delta = Delta.model_construct(stop_reason="end_turn", stop_sequence=None) + delta.container = {"id": "container_123"} + event = RawMessageDeltaEvent.model_construct( + type="message_delta", + delta=delta, + usage=Usage(input_tokens=1, output_tokens=1), + ) + + chunk, _ = model._make_message_chunk_from_anthropic_event( + event, stream_usage=True, coerce_content_to_string=True + ) + + assert chunk is not None + assert chunk.response_metadata["container"] == {"id": "container_123"} + + +def test_patch_is_idempotent_across_multiple_model_constructions(): + from langchain_anthropic import ChatAnthropic + + _patch_anthropic_stream_context_management() + patched_once = ChatAnthropic._make_message_chunk_from_anthropic_event + + # Re-applying (as every Anthropic model construction does) must not wrap + # the wrapper again — otherwise each new stream event pays for one more + # layer of indirection for the lifetime of the process. + _patch_anthropic_stream_context_management() + _patch_anthropic_stream_context_management() + + assert ChatAnthropic._make_message_chunk_from_anthropic_event is patched_once