From 27a5cd69cdc263bca4cdf1bc7667463966bd9deb Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 22 Aug 2026 11:01:50 +0530 Subject: [PATCH 01/41] feat(harness): generate a suite with one writer per use case, counted by use case and branch --- harness-ui/server.py | 14 ++ src/fi/alk/harness/persona_guides.py | 199 +++++++++++++++ src/fi/alk/harness/progress.py | 113 +++++++++ src/fi/alk/harness/scenario.py | 12 + src/fi/alk/harness/scenario_tools.py | 144 +++++++++-- src/fi/alk/harness/scenarios.py | 238 ++++++++++++++++++ .../harness/skills/write-scenarios/SKILL.md | 23 +- 7 files changed, 717 insertions(+), 26 deletions(-) create mode 100644 src/fi/alk/harness/persona_guides.py create mode 100644 src/fi/alk/harness/progress.py diff --git a/harness-ui/server.py b/harness-ui/server.py index 153fbe66..f084f1e9 100644 --- a/harness-ui/server.py +++ b/harness-ui/server.py @@ -460,6 +460,20 @@ async def world(): db.close() +@app.get("/api/generation") +async def generation(session: str = ""): + """What the suite generation is doing right now, so the page can draw it while it runs. + + Polled rather than streamed: the page may be opened halfway through a suite, refreshed, or + opened somewhere else entirely, and each of those has to show the same thing. Empty when + nothing has been generated here, which the page reads as "no fan-out to show". + """ + from fi.alk.harness import progress + + out = _folder(session) + return progress.read(out) if out else {} + + @app.get("/api/scenarios") async def scenarios(): """Every scenario, with its files and its three gates re-run. diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py new file mode 100644 index 00000000..01231a6d --- /dev/null +++ b/src/fi/alk/harness/persona_guides.py @@ -0,0 +1,199 @@ +"""The behaviour guidance the platform already uses for a simulated caller. + +A persona profile names what somebody is like: impatient and direct, cautious and skeptical. It +does not say how that should sound turn by turn, and a model handed only the label improvises +one, which is how "in a hurry" became a caller who says it every turn instead of a caller who +cuts in once and accepts the first workable answer. + +The platform solved that with lookup tables mapping each value to a sentence of guidance, and +voice simulation has run on them for months. They are read from there rather than restated here, +because two copies of the same wording drift and then a caller behaves one way on the platform +and another way through the harness, for reasons nobody can see. + +Read, not imported: the tables live inside a Django app this package cannot import, but they are +plain literals, so they are parsed out of the file. Absent, every lookup answers with nothing and +a persona still renders — one without guidance, never a crash. +""" + +from __future__ import annotations + +import ast +import os +from functools import lru_cache +from pathlib import Path + +# Where the platform's tables are mounted. Colon-separated so voice and chat guides can both be +# offered; the first file defining a table wins, so voice takes precedence when both are present. +GUIDES_ENV = "HARNESS_PERSONA_GUIDES" + +WANTED = ( + "VOICE_PERSONALITY_GUIDES", + "VOICE_COMMUNICATION_STYLE_GUIDES", + "CHAT_PERSONALITY_GUIDES", + "CHAT_COMMUNICATION_STYLE_GUIDES", + "CHAT_TONE_GUIDES", + "CHAT_VERBOSITY_GUIDES", +) + + +def _tables_in(path: Path) -> dict[str, dict[str, str]]: + """Every guidance table defined in one file, by name. + + Parsed rather than executed. The file sits in an app with imports this process cannot + satisfy, and running it to read a dictionary would fail for reasons that have nothing to do + with the dictionary. + """ + found: dict[str, dict[str, str]] = {} + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): + return found + for node in tree.body: + targets = ( + [node.target] if isinstance(node, ast.AnnAssign) else getattr(node, "targets", []) + ) + for target in targets: + name = getattr(target, "id", "") + if name not in WANTED or node.value is None: + continue + try: + value = ast.literal_eval(node.value) + except ValueError: + continue + if isinstance(value, dict) and value: + found[name] = {str(k).lower(): str(v) for k, v in value.items()} + return found + + +@lru_cache(maxsize=1) +def guides() -> dict[str, dict[str, str]]: + """Every table the platform offers this harness, merged.""" + merged: dict[str, dict[str, str]] = {} + for raw in (os.environ.get(GUIDES_ENV) or "").split(":"): + if not raw.strip(): + continue + for name, table in _tables_in(Path(raw.strip())).items(): + merged.setdefault(name, table) + return merged + + +def guidance_for(kind: str, value: str, *, voice: bool = True) -> str: + """The platform's sentence for one persona value, or nothing. + + ``kind`` is ``personality``, ``communication_style``, ``tone`` or ``verbosity``. Voice tables + are preferred for a spoken call and the chat table is the fallback, because the two describe + the same disposition and only one of them is written for speech. + """ + if not value.strip(): + return "" + tables = guides() + order = ("VOICE", "CHAT") if voice else ("CHAT", "VOICE") + for prefix in order: + table = tables.get(f"{prefix}_{kind.upper()}_GUIDES") or {} + found = table.get(value.strip().lower()) + if found: + return found + return "" + + +def available() -> bool: + """Whether any guidance was found, so a build can say so rather than silently omitting it.""" + return bool(guides()) + + +# Where the platform's persona model is mounted, for the values it accepts. +VOCABULARY_ENV = "HARNESS_PERSONA_VOCABULARY" + +# The persona fields worth constraining, and the choice class each is drawn from. Only the ones +# that change behaviour or routing: a free-text occupation harms nothing, an accent nobody +# recognises silently loses the voice it was supposed to select. +FIELDS = { + "gender": "GenderChoices", + "age_group": "AgeGroupChoices", + "occupation": "ProfessionChoices", + "location": "LocationChoices", + "personality": "PersonalityChoices", + "communication_style": "CommunicationStyleChoices", + "accent": "AccentChoices", + "languages": "LanguageChoices", +} + +# Constrained because something downstream reads them. The rest are offered as vocabulary but a +# writer who needs a value outside them is not stopped: an unknown occupation costs nothing, +# an unknown accent costs the voice. +ENFORCED = ("personality", "communication_style", "accent", "languages") + + +@lru_cache(maxsize=1) +def vocabulary() -> dict[str, list[str]]: + """What the platform accepts for each persona field. + + Parsed out of the model's ``TextChoices`` classes for the same reason the guidance is read + rather than restated: the platform is the one that has to understand these values, so it is + the one that decides what they are. A persona written in words of its own renders fine, gets + no behaviour guidance, and cannot be grouped with anything on the platform afterwards. + """ + path = os.environ.get(VOCABULARY_ENV) or "" + if not path or not Path(path).exists(): + return {} + try: + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + except (OSError, SyntaxError): + return {} + + by_class: dict[str, list[str]] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + values: list[str] = [] + for item in node.body: + if not isinstance(item, ast.Assign): + continue + try: + held = ast.literal_eval(item.value) + except ValueError: + continue + # ``NAME = "value", "Label"`` is the choices shape; a bare string is also accepted. + if isinstance(held, tuple) and held and isinstance(held[0], str): + values.append(held[0]) + elif isinstance(held, str): + values.append(held) + if values: + by_class[node.name] = values + + return { + field: by_class[cls] for field, cls in FIELDS.items() if by_class.get(cls) + } + + +def offered(field: str) -> list[str]: + """The values this field accepts, or nothing if the platform's model was not readable.""" + return list(vocabulary().get(field, [])) + + +def unrecognised(persona: dict[str, object]) -> list[str]: + """Persona values the platform would not recognise, as sentences saying what to use instead. + + Only the fields something downstream actually reads, and only when the vocabulary was found: + a harness that cannot see the platform's model must not start refusing personas over it. + """ + known = vocabulary() + if not known: + return [] + problems: list[str] = [] + for field in ENFORCED: + allowed = known.get(field) or [] + if not allowed: + continue + held = persona.get(field) + values = held if isinstance(held, list) else ([held] if held else []) + lowered = {str(one).strip().lower() for one in allowed} + for one in values: + text = str(one).strip() + if text and text.lower() not in lowered: + problems.append( + f"persona {field} {text!r} is not one the platform knows, so it will not " + f"reach the call. Use one of: {', '.join(allowed)}. Anything else this " + "person is like belongs in persona.metadata." + ) + return problems diff --git a/src/fi/alk/harness/progress.py b/src/fi/alk/harness/progress.py new file mode 100644 index 00000000..eb765917 --- /dev/null +++ b/src/fi/alk/harness/progress.py @@ -0,0 +1,113 @@ +"""What the fan-out is doing right now, written where a UI can read it. + +Generating a suite in parallel is the one thing this harness does where nothing appears for +several minutes and then everything appears at once. Told nothing, a person cannot tell a +working run from a hung one, and the honest answer to "is it stuck" is the only thing they want. + +So the fan-out writes its own state as it goes: which use cases it split the work into, which +are running, how many scenarios each has proved, and which have finished. A file rather than a +stream, because the reader is a page that may be opened halfway through, refreshed, or opened on +another machine, and each of those has to show the same thing. +""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +PROGRESS = "generation.json" + +WAITING = "waiting" +RUNNING = "running" +DONE = "done" +FAILED = "failed" + + +def _path(destination: Path) -> Path: + return Path(destination) / PROGRESS + + +def _write(destination: Path, state: dict[str, Any]) -> None: + """Replace the file atomically. + + A reader polling this will otherwise catch a half-written file and show nothing, which looks + exactly like the failure it is meant to rule out. + """ + path = _path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp") + try: + with os.fdopen(handle, "w", encoding="utf-8") as writing: + json.dump(state, writing, indent=2) + os.replace(temporary, path) + except BaseException: + Path(temporary).unlink(missing_ok=True) + raise + + +def read(destination: Path) -> dict[str, Any]: + """The current state, or nothing if no suite has been generated here.""" + path = _path(destination) + if not path.exists(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + + +def planned( + destination: Path, allocation: list[tuple[str, int]], *, at_once: int, asked: int +) -> None: + """The split, before any of it starts. Written first so the tree appears immediately.""" + _write( + destination, + { + "state": RUNNING, + "asked": asked, + "at_once": at_once, + "kept": 0, + "slices": [ + {"use_case": case, "wanted": count, "kept": 0, "state": WAITING} + for case, count in allocation + ], + }, + ) + + +def _change(destination: Path, use_case: str, **fields: Any) -> None: + state = read(destination) + for slice_ in state.get("slices", []): + if slice_.get("use_case") == use_case: + slice_.update(fields) + break + state["kept"] = sum(one.get("kept", 0) for one in state.get("slices", [])) + _write(destination, state) + + +def started(destination: Path, use_case: str) -> None: + _change(destination, use_case, state=RUNNING) + + +def kept(destination: Path, use_case: str, count: int) -> None: + """How many this slice has proved so far. Called as they land, not at the end.""" + _change(destination, use_case, kept=count) + + +def finished(destination: Path, use_case: str, count: int) -> None: + _change(destination, use_case, state=DONE, kept=count) + + +def failed(destination: Path, use_case: str, why: str) -> None: + _change(destination, use_case, state=FAILED, why=why[:300]) + + +def settled(destination: Path, *, kept_total: int) -> None: + """The whole fan-out is over and the suite is written.""" + state = read(destination) + state["state"] = DONE + state["kept"] = kept_total + _write(destination, state) diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 90428c6c..1c62c5f1 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -154,6 +154,11 @@ class Scenario(BaseModel): name: str use_case: str = "" + # Which branch of that use case this is: the condition that makes this row different from + # its siblings. A use case fans out into several — the ordinary path, the one that cannot be + # completed, the rule under pressure — and each is its own test. Coverage is counted on the + # pair, so a use case can carry many scenarios without any of them reading as a duplicate. + branch: str = "" tests: str = "" # What this scenario changes about the world after it is reset, as code: a file defining @@ -232,6 +237,13 @@ def validate_scenario( missing := scenario.persona.missing_profile_fields() ): problems.append("persona is incomplete: " + ", ".join(missing)) + elif scenario.persona is not None: + # A persona written in words of its own renders fine and then does nothing: no behaviour + # guidance attaches to it, and the accent it names selects no voice. Caught here, where + # the writer is still holding the scenario and can fix it in one turn. + from .persona_guides import unrecognised + + problems.extend(unrecognised(scenario.persona.model_dump())) if not scenario.sub_goals: problems.append( "no sub_goals: nothing would be graded. Name the entries of the catalogue this " diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index e6249837..6bc78c10 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -50,6 +50,31 @@ def _err(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}], "is_error": True} +def persona_field(name: str) -> dict[str, Any]: + """The schema for one persona field, carrying the platform's own values where it has them. + + Offered as an enum so the values arrive right the first time. Without the platform's model + to read, it stays a plain string rather than an enum of nothing. + """ + from .persona_guides import offered + + allowed = offered(name) + return {"type": "string", "enum": allowed} if allowed else {"type": "string"} + + +def persona_vocabulary_note() -> str: + """A sentence about why the persona fields are constrained, when they are.""" + from .persona_guides import vocabulary + + if not vocabulary(): + return "" + return ( + " The listed values are the ones the platform understands: they carry behaviour " + "guidance into the call and select the caller's voice. Anything else about this person " + "goes in metadata, where it is free text." + ) + + def write_scenarios( scenarios: list[Scenario], destination: Path, catalogue: Catalogue | None = None ) -> Path: @@ -168,17 +193,22 @@ def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[s # "cancel a pending order", which is neither what it tests nor distinguishable afterwards # from the scenario that really does test that. A use case is how coverage is counted, so a # duplicate quietly overstates it. - claimed: dict[str, list[str]] = {} + # Keyed on the pair, not the use case alone. A use case fans out into several branches and + # each is a separate test, so keying on the use case alone caps a suite at one scenario per + # use case — which is how a request for forty against fourteen use cases became unsaveable. + claimed: dict[tuple[str, str], list[str]] = {} for one in kept: case = (one.use_case or "").strip().lower() + branch = (one.branch or "").strip().lower() if case: - claimed.setdefault(case, []).append(one.name) - for case, names in claimed.items(): + claimed.setdefault((case, branch), []).append(one.name) + for (case, branch), names in claimed.items(): if len(names) > 1: + where = f"{case!r}" if not branch else f"{case!r} / {branch!r}" problems.append( - f"{' and '.join(names)} both claim the use case {case!r}. Give each the use case " - "it actually exercises, or drop the one that duplicates the other. Coverage is " - "counted by use case, so two scenarios sharing one hides a gap." + f"{' and '.join(names)} both claim {where}. Give each the branch it actually " + "exercises, or drop the one that duplicates the other. Coverage is counted by " + "use case and branch, so two scenarios sharing both hides a gap." ) # Sub-goals are shared so results roll up. A suite where every scenario invents its own is a @@ -193,10 +223,27 @@ def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[s def scenario_tools( - contract: AgentContract, world_root: Path, destination: Path, *, wanted: int + contract: AgentContract, + world_root: Path, + destination: Path, + *, + wanted: int, + can_save: bool = True, + start_from: list[Scenario] | None = None, ) -> tuple[Any, list[Scenario]]: - """A server for writing scenarios against one built environment.""" - kept: list[Scenario] = load_scenarios(destination) + """A server for writing scenarios against one built environment. + + ``can_save`` is what makes several writers safe at once. Saving rewrites the index and + removes any folder not in the saver's own list, so two writers saving concurrently delete + each other's work. A writer that only submits keeps its scenarios in ``kept``, and whoever + spawned it merges the lists and writes once. + + ``start_from`` seeds that list. A parallel writer starts empty rather than from disk, so it + is never counted as already having what a sibling wrote. + """ + kept: list[Scenario] = ( + list(start_from) if start_from is not None else load_scenarios(destination) + ) catalogue = load_catalogue(destination) simulator_prompt = load_simulator_prompt(destination) target = {"count": wanted} @@ -373,6 +420,12 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "string", "description": "Which of the agent's use cases this belongs to.", }, + "branch": { + "type": "string", + "description": "The condition that makes this scenario different from the " + "others in the same use case, in one line: what is true here that is not " + "true of its siblings.", + }, "tests": { "type": "string", "description": "One line: what this scenario is trying to find out.", @@ -386,23 +439,27 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "object", "description": "Who the simulated person is, separate from the task. Use " "the established voice-scenario shape and only grounded, test-relevant " - "details. This fills the simulator prompt's persona slot.", + "details. This fills the simulator prompt's persona slot." + + persona_vocabulary_note(), "properties": { "name": {"type": "string"}, - "gender": {"type": "string"}, - "age_group": {"type": "string"}, - "occupation": {"type": "string"}, - "location": {"type": "string"}, - "personality": {"type": "string"}, - "communication_style": {"type": "string"}, + "gender": persona_field("gender"), + "age_group": persona_field("age_group"), + "occupation": persona_field("occupation"), + "location": persona_field("location"), + "personality": persona_field("personality"), + "communication_style": persona_field("communication_style"), "initial_message": { "type": "string", "description": "The caller's natural opening request, specific to " "this scenario. Do not use a generic greeting.", }, "keywords": {"type": "array", "items": {"type": "string"}}, - "languages": {"type": "array", "items": {"type": "string"}}, - "accent": {"type": "string"}, + "languages": { + "type": "array", + "items": persona_field("languages"), + }, + "accent": persona_field("accent"), "multilingual": {"type": "boolean"}, "metadata": {"type": "object"}, }, @@ -612,6 +669,50 @@ async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: write_scenarios(kept, destination, catalogue) return _ok(f"{name} dropped. {len(kept)} left") + @tool( + "generate_suite", + "Write a whole suite at once by splitting it across the agent's use cases, one writer " + "per use case, several running at the same time. Use this when somebody asks for a " + "suite rather than a particular scenario: writing twenty or fifty one at a time runs " + "out of turns long before it finishes. Everything it produces has cleared the same " + "three gates. Saved when it completes.", + schema({"count": int, "at_once": int}, ["count"]), + ) + async def generate_suite(args: dict[str, Any]) -> dict[str, Any]: + from .scenarios import write_in_parallel + + count = int(args.get("count") or 0) + if count < 1: + return _err("say how many scenarios the suite should have") + at_once = int(args.get("at_once") or 0) or 4 + cases = [one for one in contract.real_use_cases if one.strip()] + if not cases: + return _err( + "this contract names no use cases, so there is nothing to split the work " + "across. Write them one at a time with submit_scenario, or fix the contract." + ) + produced = await write_in_parallel( + contract, + out=destination, + wanted=count, + use_cases=cases, + at_once=at_once, + ) + # The suite is already on disk. The open session's own list has to be brought level with + # it, or a later save_scenarios here would write out the stale list and delete every + # folder the fan-out just produced. + kept[:] = produced + target["count"] = len(produced) + by_case: dict[str, int] = {} + for one in produced: + name = one.use_case or "unassigned" + by_case[name] = by_case.get(name, 0) + 1 + lines = "\n".join(f" {n} x {case[:70]}" for case, n in sorted(by_case.items())) + return _ok( + f"{len(produced)} scenarios across {len(by_case)} use cases, {at_once} writers at a " + f"time. Each cleared all three gates and the suite is saved.\n{lines}" + ) + @tool( "save_scenarios", "Write the kept scenarios out. Every one has already been proved by submit_scenario, so " @@ -670,8 +771,10 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: fix_tool_tool, aim_for, drop_scenario, - save_scenarios, - ], + ] + # Only the session a person is talking to may fan out. A writer that is itself one slice + # of a fan-out calling this would split its own slice again, and so on. + + ([generate_suite, save_scenarios] if can_save else []), ) return server, kept @@ -688,6 +791,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: "fix_tool", "aim_for", "drop_scenario", + "generate_suite", "save_scenarios", ) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 57a4a581..dc32223a 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -10,6 +10,7 @@ from __future__ import annotations +import asyncio from collections.abc import Callable from pathlib import Path from typing import Any @@ -25,6 +26,8 @@ permission_gate, provider_env, ) +from . import progress +from .catalogue import load_catalogue from .contract import AgentContract from .scenario import Scenario from .scenario_tools import ( @@ -33,6 +36,7 @@ load_scenarios, scenario_tools, world_summary, + write_scenarios, ) from .session import Stage from .tools import qualified @@ -131,6 +135,10 @@ def opening(contract: AgentContract, wanted: int = 10, existing: int = 0) -> str "across several turns. If a proof says an intended check is vacuous or broken, repair " "that named sub-goal with add_sub_goal and resubmit. Never evade a gate by deleting a " "check for behavior the scenario still claims to test. Then save_scenarios." + "\n\nFor a suite rather than one scenario, say briefly how you are splitting it " + "across the agent's use cases and then write it with generate_suite in the same turn: " + "it runs a writer per use case at the same time and saves what they prove, where " + "writing this many one at a time would run out of turns before finishing." ) @@ -139,6 +147,236 @@ def load(destination: Path) -> list[Scenario]: return load_scenarios(Path(destination)) +# How many writers run at once. Each is a model session with its own subprocess, and each gate +# restores its own copy of the world, so this is bounded by the machine rather than by the API. +AT_ONCE = 4 + + +def shares(wanted: int, use_cases: list[str]) -> list[tuple[str, int]]: + """How many scenarios each use case is asked to produce. + + Evenly, with the remainder going to the ones named first, because a contract lists its + primary use cases before its marginal ones. A use case that turns out to have less in it + than its share says returns fewer; nothing forces it to pad. + """ + if not use_cases: + return [] + if wanted <= len(use_cases): + return [(case, 1) for case in use_cases[:wanted]] + each, extra = divmod(wanted, len(use_cases)) + return [(case, each + (1 if i < extra else 0)) for i, case in enumerate(use_cases)] + + +def callers_for(index: int, wanted: int) -> str: + """Which callers this slice should write, so the suite varies across slices as well as within. + + Instruction alone cannot do this. Each writer is blind to the others, so each independently + picks the safest value and the suite converges on it: measured across three suites, more + than half the callers came out "Professional and formal" and over three quarters American, + with nobody doing anything wrong. Worse, a slice writing a single scenario has nothing to + vary at all. + + So the spread is dealt out here, the same way the work is. Each slice is handed a different + starting point in the platform's own vocabularies and told to begin there. It is a + suggestion rather than a rule, because the caller still has to suit the scenario: a stolen + phone is not a cheerful call whatever this hands out. + """ + from .persona_guides import offered + + people = offered("personality") + accents = offered("accent") + if not people: + return "" + picks = [people[(index + step) % len(people)] for step in range(max(1, wanted))] + accent = accents[index % len(accents)] if accents else "" + said = ( + "\n\nStart from these callers, and move off them only where the scenario calls for " + f"somebody else: {', '.join(picks)}." + ) + if accent: + said += ( + f" At least one of your callers has a {accent} accent. Other writers are covering " + "other use cases with other callers, so a suite where everyone sounds the same is " + "what happens when each of us picks the safest option." + ) + return said + + +def branch_opening( + contract: AgentContract, use_case: str, wanted: int, callers: str = "" +) -> str: + return ( + f"Write {wanted} scenarios for {contract.agent!r}, all of them within this one " + "use case:\n\n" + f" {use_case}\n\n" + "Write nothing outside it. Somebody else is covering the other use cases at the same " + "time, so a scenario that strays is either a duplicate of theirs or a gap in yours.\n\n" + "Every scenario carries this use case verbatim in `use_case`, and its own one-line " + "`branch` saying what makes it different from the others you write here. Branches are " + "where the variety lives: the ordinary path, the branch that cannot be completed, the " + "rule under pressure, state that has to carry across turns, the same request against a " + "differently seeded world.\n\n" + "Look at the world first with inspect_world so every scenario names real records, and " + "read the sub-goals already defined. Work out each solution with try_calls before you " + "submit it. Submit each one with submit_scenario and then stop: do not save, and do not " + "ask what to do next. Whoever asked for this collects the suite and writes it.\n\n" + "Vary the caller across the scenarios you write. Everyone else is writing their own use " + "case and cannot see yours, so a suite where every caller is professional and formal is " + "what happens when each writer picks the safest value. Give different scenarios " + "different personalities, communication styles and accents from the values offered, and " + "let the caller suit the situation: somebody whose card was declined is not in the same " + "mood as somebody booking a routine morning ride." + callers + ) + + +async def _write_one_use_case( + contract: AgentContract, + use_case: str, + count: int, + *, + index: int = 0, + destination: Path, + on_event: Callable[..., Any] | None, + ask: Callable[..., Any] | None, +) -> list[Scenario]: + """One use case's share, written by its own session. Returns what it proved, unsaved.""" + server, kept = scenario_tools( + contract, + destination, + destination, + wanted=count, + can_save=False, + start_from=[], + ) + progress.started(destination, use_case) + + def watch(event: Any) -> None: + # Report as they land rather than at the end. A slice that proves its first scenario + # four minutes in is the difference between a run that looks alive and one that does not. + progress.kept(destination, use_case, len(kept)) + if on_event: + on_event(event) + allowed = [ + qualified(SCENARIO_SERVER, name) for name in TOOL_NAMES if name != "save_scenarios" + ] + options = ClaudeAgentOptions( + system_prompt=( + f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" + f"\n\n## Its world\n\n{world_summary(destination)}" + f"\n\n## Your slice\n\nYou are writing only the scenarios for: {use_case}" + ), + allowed_tools=allowed, + mcp_servers={SCENARIO_SERVER: server}, + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=turns_for(count), + model=chosen_model(), + env=provider_env(), + ) + options.disallowed_tools = list(UNWANTED) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + stage = Stage(options, name=f"{SKILL}:{use_case[:40]}") + try: + async with stage: + await stage.say( + branch_opening(contract, use_case, count, callers_for(index, count)), + on_event=watch, + ) + except Exception as broke: # noqa: BLE001 - one slice failing must not lose the others + progress.failed(destination, use_case, str(broke)) + if on_event: + on_event({"type": "slice_failed", "use_case": use_case, "why": str(broke)[:300]}) + return list(kept) + progress.finished(destination, use_case, len(kept)) + return list(kept) + + +def merged(written: list[list[Scenario]]) -> list[Scenario]: + """One suite out of several writers, with the collisions they could not see removed. + + The writers run blind to each other, so two can land on the same folder name or on the same + use case and branch. Both are dropped here rather than at save time, where the loser would + silently overwrite the winner's folder. + """ + suite: list[Scenario] = [] + names: set[str] = set() + pairs: set[tuple[str, str]] = set() + for batch in written: + for one in batch: + pair = ((one.use_case or "").strip().lower(), (one.branch or "").strip().lower()) + if one.name in names or (pair[0] and pair in pairs): + continue + names.add(one.name) + if pair[0]: + pairs.add(pair) + suite.append(one) + return suite + + +async def write_in_parallel( + contract: AgentContract, + *, + out: Path | None = None, + wanted: int = 10, + use_cases: list[str] | None = None, + at_once: int = AT_ONCE, + on_event: Callable[..., Any] | None = None, + ask: Callable[..., Any] | None = None, +) -> list[Scenario]: + """Write a suite with one session per use case, then save it once. + + Sequentially, a suite costs roughly three turns a scenario against one budget, which is why + asking for forty stopped around twenty-five. Here each use case is written by its own + session, so the wall clock is the slowest use case rather than the sum of all of them, and + the turn budget is per slice rather than shared. + + Saving stays here, once, for a reason: ``save_scenarios`` regenerates the index and deletes + any folder it does not know about, so letting the writers save would have each of them + remove the others' work. + """ + destination = out or artifact_dir(contract.agent) + cases = [case for case in (use_cases or contract.real_use_cases) if case.strip()] + if not cases: + # Nothing to partition on. One writer, the ordinary path, rather than no scenarios. + return await write(contract, out=destination, wanted=wanted, on_event=on_event, ask=ask) + + allocation = shares(wanted, cases) + progress.planned(destination, allocation, at_once=at_once, asked=wanted) + if on_event: + on_event({"type": "planned", "slices": allocation, "at_once": at_once}) + + limit = asyncio.Semaphore(max(1, at_once)) + + async def guarded(use_case: str, count: int, index: int) -> list[Scenario]: + async with limit: + return await _write_one_use_case( + contract, + use_case, + count, + index=index, + destination=destination, + on_event=on_event, + ask=ask, + ) + + written = await asyncio.gather( + *( + guarded(case, count, index) + for index, (case, count) in enumerate(allocation) + ), + return_exceptions=False, + ) + + suite = merged([load_scenarios(destination), *written]) + write_scenarios(suite, destination, load_catalogue(destination)) + progress.settled(destination, kept_total=len(suite)) + if on_event: + on_event({"type": "saved", "kept": len(suite), "asked": wanted}) + return load(destination) + + async def write( contract: AgentContract, *, diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 86ac6c86..474f8ce7 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -20,6 +20,7 @@ afterwards. ``` name short identifier; it becomes this scenario's folder use_case which of the agent's use cases this belongs to +branch what makes this one different from its siblings in that use case tests one line: what this scenario is trying to find out instruction the task, written to the person the agent is serving persona who that person is: identity, communication style, languages/accent and characteristics @@ -157,6 +158,16 @@ rule under pressure, the state that has to carry, the same request against a dif world. Keep that plan concise and continue immediately unless the person explicitly asked to review it. + +**Then write the suite with `generate_suite`, not one scenario at a time.** It splits the work +across the agent's use cases and runs several writers at once, each proving its own scenarios +through the same three gates. Writing a suite yourself with `submit_scenario` costs about three +turns per scenario against one budget, so a request for twenty or fifty runs out long before it +finishes, and what does get written is lost because nothing was saved. + +Use `submit_scenario` for what it is good at: one scenario somebody asked for by name, a +replacement for one that came back wrong, or filling a specific gap in a suite that already +exists. Anything described as a number of scenarios is a suite. After inspecting the world, submit the first scenario in the same response. Then prove and save one scenario at a time. Never silently compose the whole suite before the next tool call: the UI must show progress, and already-proved work must survive a stopped or timed-out model turn. @@ -212,9 +223,7 @@ Every stance still obeys the bar above: a real person could bring it, a competen fail it, and the values are real. A stance chooses *what to look at*, never whether the scenario has to be honest. -Two rules keep this from turning into noise. **Each scenario carries one use case, and no two -scenarios carry the same one** — a duplicate is either the same test twice or one of them is -mislabelled, and it hides a gap while appearing to fill it. And a stance that produces nothing new +Two rules keep this from turning into noise. **Each scenario carries one use case and one branch, and no two scenarios carry the same pair** — a duplicate is either the same test twice or one of them is mislabelled, and it hides a gap while appearing to fill it. Several scenarios sharing a use case is normal and expected; that is what branches are for. What is not allowed is two rows that agree on both. And a stance that produces nothing new for a given agent produces nothing: an agent with no rules to bend does not need an adversarial scenario invented for it. @@ -376,10 +385,12 @@ hides the problem and everything built afterwards inherits it. 1. `inspect_world` with no table, then look at the ones that matter. Read the sub-goals already defined. 2. Read the agent's hard rules. Each one is a branch waiting to be written. -3. For each scenario: work out the solution, `try_calls` it with your `setup_code`, then +3. For a suite, say how you are splitting it and then `generate_suite` with the count. It + writes the whole thing and saves it, and you report what came back. +4. For a single scenario: work out the solution, `try_calls` it with your `setup_code`, then `submit_scenario`. -4. Read what comes back. A refusal names which gate failed and why. -5. `save_scenarios` when you have the number that was asked for. +5. Read what comes back. A refusal names which gate failed and why. +6. `save_scenarios` when you have the number that was asked for. ## Finishing From efdb0253e2c8302cf03665408e427291b24b88ed Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 22 Aug 2026 12:10:19 +0530 Subject: [PATCH 02/41] feat(harness): write a suite to a plan, review what came back, and cap a large ask --- src/fi/alk/harness/scenario_tools.py | 81 +++- src/fi/alk/harness/scenarios.py | 393 +++++++++++++++--- .../harness/skills/write-scenarios/SKILL.md | 11 + 3 files changed, 413 insertions(+), 72 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 6bc78c10..fe373417 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -672,30 +672,79 @@ async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: @tool( "generate_suite", "Write a whole suite at once by splitting it across the agent's use cases, one writer " - "per use case, several running at the same time. Use this when somebody asks for a " - "suite rather than a particular scenario: writing twenty or fifty one at a time runs " - "out of turns long before it finishes. Everything it produces has cleared the same " - "three gates. Saved when it completes.", - schema({"count": int, "at_once": int}, ["count"]), + "per slice, several running at the same time, then reviewing what came back and " + "filling what it missed. Use this whenever somebody asks for a number of scenarios " + "rather than one in particular: writing twenty or fifty one at a time runs out of " + "turns long before it finishes.\n\n" + "Pass `slices` when you know how the suite should be divided, which you do once you " + "have looked at the world: give each use case a share in proportion to how much can " + "genuinely go wrong in it, and name the angle each slice should take. Without it the " + "work is divided evenly, which pads the thin use cases and under-covers the rich ones. " + "Everything produced clears the same three gates, and the suite is saved.", + schema( + { + "count": int, + "at_once": int, + "slices": { + "type": ["array", "null"], + "description": "How to divide the suite. One entry per writer.", + "items": { + "type": "object", + "properties": { + "use_case": { + "type": "string", + "description": "One of the agent's use cases, worded as the " + "contract words it.", + }, + "angle": { + "type": "string", + "description": "What this slice should look for: the ordinary " + "path, the branch that cannot be completed, the rule under " + "pressure, state that has to carry.", + }, + "count": { + "type": "integer", + "description": "How many scenarios this slice is worth, in " + "proportion to how much can genuinely go wrong in it.", + }, + "why": { + "type": "string", + "description": "Why it earns that share.", + }, + }, + "required": ["use_case", "count"], + }, + }, + }, + ["count"], + ), ) async def generate_suite(args: dict[str, Any]) -> dict[str, Any]: - from .scenarios import write_in_parallel + from .scenarios import MOST_AT_ONCE, MOST_IN_ONE_GO, write_in_parallel - count = int(args.get("count") or 0) - if count < 1: + asked = int(args.get("count") or 0) + if asked < 1: return _err("say how many scenarios the suite should have") - at_once = int(args.get("at_once") or 0) or 4 cases = [one for one in contract.real_use_cases if one.strip()] - if not cases: + given = args.get("slices") or None + if not cases and not given: return _err( "this contract names no use cases, so there is nothing to split the work " "across. Write them one at a time with submit_scenario, or fix the contract." ) + + # A large ask is served a batch at a time. Spinning up a writer per scenario would put + # hundreds of model sessions on one machine, and the person waiting would see nothing + # for an hour. A batch they can read, and an offer of the rest, is the better trade. + count = min(asked, MOST_IN_ONE_GO) + at_once = max(1, min(int(args.get("at_once") or 0) or 4, MOST_AT_ONCE)) + produced = await write_in_parallel( contract, out=destination, wanted=count, use_cases=cases, + slices=given, at_once=at_once, ) # The suite is already on disk. The open session's own list has to be brought level with @@ -703,15 +752,25 @@ async def generate_suite(args: dict[str, Any]) -> dict[str, Any]: # folder the fan-out just produced. kept[:] = produced target["count"] = len(produced) + by_case: dict[str, int] = {} for one in produced: name = one.use_case or "unassigned" by_case[name] = by_case.get(name, 0) + 1 lines = "\n".join(f" {n} x {case[:70]}" for case, n in sorted(by_case.items())) - return _ok( + said = ( f"{len(produced)} scenarios across {len(by_case)} use cases, {at_once} writers at a " f"time. Each cleared all three gates and the suite is saved.\n{lines}" ) + if asked > count: + said += ( + f"\n\n{asked - count} of the {asked} asked for are still to write. " + f"{MOST_IN_ONE_GO} is as many as one pass does, so that the suite can be looked " + "at before more is spent on it. Show what came back, then ask whether to carry " + "on with the rest, change direction first, or stop here. Call generate_suite " + "again for the next batch once they have said." + ) + return _ok(said) @tool( "save_scenarios", diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index dc32223a..e877ce36 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -11,11 +11,13 @@ from __future__ import annotations import asyncio +import os +from dataclasses import dataclass from collections.abc import Callable from pathlib import Path from typing import Any -from claude_agent_sdk import ClaudeAgentOptions +from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool from .config import ( UNWANTED, @@ -39,7 +41,7 @@ write_scenarios, ) from .session import Stage -from .tools import qualified +from .tools import qualified, schema SKILL = "write-scenarios" @@ -147,24 +149,107 @@ def load(destination: Path) -> list[Scenario]: return load_scenarios(Path(destination)) -# How many writers run at once. Each is a model session with its own subprocess, and each gate -# restores its own copy of the world, so this is bounded by the machine rather than by the API. +# What a suite costs, and what it is allowed to cost. +# +# Writers run as separate model sessions, so wall clock is roughly the number of scenarios +# divided by how many run at once. The two ceilings below exist for different reasons: one +# protects the machine, the other protects the person waiting. Asking for a thousand scenarios +# is a reasonable thing to want and an unreasonable thing to do in one go, so a large ask is +# served a batch at a time with the rest offered back. AT_ONCE = 4 +MOST_AT_ONCE = int(os.environ.get("HARNESS_WRITERS_AT_ONCE") or 8) +MOST_IN_ONE_GO = int(os.environ.get("HARNESS_SUITE_BATCH") or 50) +# How many times the suite is reviewed and topped up after the first pass. One is enough to +# catch a slice that came back short or a use case nobody covered; more turns it into a loop +# that keeps finding smaller things to say. +TOP_UP_ROUNDS = 1 -def shares(wanted: int, use_cases: list[str]) -> list[tuple[str, int]]: - """How many scenarios each use case is asked to produce. + +@dataclass(frozen=True) +class Slice: + """One writer's share of a suite: what to write, how much, and why it is worth writing.""" + + use_case: str + angle: str = "" + count: int = 1 + why: str = "" + + def named(self) -> str: + return f"{self.use_case} — {self.angle}" if self.angle else self.use_case + + +def even_slices(wanted: int, use_cases: list[str]) -> list[Slice]: + """The fallback split, when nobody said how the work should be divided. Evenly, with the remainder going to the ones named first, because a contract lists its - primary use cases before its marginal ones. A use case that turns out to have less in it - than its share says returns fewer; nothing forces it to pad. + primary use cases before its marginal ones. It is a poor plan and it is meant to be: a use + case with one real branch gets the same share as one with six, so the first pads and the + second under-covers. It exists so a caller that supplies no plan still gets a suite. """ if not use_cases: return [] if wanted <= len(use_cases): - return [(case, 1) for case in use_cases[:wanted]] + return [Slice(use_case=case, count=1) for case in use_cases[:wanted]] each, extra = divmod(wanted, len(use_cases)) - return [(case, each + (1 if i < extra else 0)) for i, case in enumerate(use_cases)] + return [ + Slice(use_case=case, count=each + (1 if i < extra else 0)) + for i, case in enumerate(use_cases) + ] + + +def planned(wanted: int, use_cases: list[str], given: list[dict] | None) -> list[Slice]: + """The split this suite will actually be written to. + + A plan supplied by the caller wins, because whoever is talking to the person has just read + the contract and the world and knows which use cases have something in them. Sizing every + use case identically is the thing that made suites pad in one place and under-cover in + another, and the plan is the only part of the process that knows the difference. + + Anything the plan leaves out is filled in evenly, and anything it over-asks for is trimmed, + so a plan can be rough without producing a suite nobody asked for. + """ + if not given: + return even_slices(wanted, use_cases) + + known = {case.strip().lower(): case for case in use_cases} + slices: list[Slice] = [] + for one in given: + if not isinstance(one, dict): + continue + case = str(one.get("use_case") or "").strip() + if not case: + continue + # Match the contract's own wording where the plan paraphrased it, so a slice is filed + # under a use case the coverage count recognises rather than a near-miss of one. + case = known.get(case.lower(), case) + try: + count = max(1, int(one.get("count") or 1)) + except (TypeError, ValueError): + count = 1 + slices.append( + Slice( + use_case=case, + angle=str(one.get("angle") or "").strip(), + count=count, + why=str(one.get("why") or "").strip(), + ) + ) + if not slices: + return even_slices(wanted, use_cases) + + # Trim from the end rather than scaling everything down: the plan put its most valuable + # slices first, and shaving one scenario off each is how a deliberate plan becomes an even + # one again. + total = sum(one.count for one in slices) + while total > wanted and slices: + last = slices[-1] + if last.count > 1: + slices[-1] = Slice(last.use_case, last.angle, last.count - 1, last.why) + else: + slices.pop() + total = sum(one.count for one in slices) + return slices def callers_for(index: int, wanted: int) -> str: @@ -202,60 +287,84 @@ def callers_for(index: int, wanted: int) -> str: return said -def branch_opening( - contract: AgentContract, use_case: str, wanted: int, callers: str = "" +def brief_for( + contract: AgentContract, mine: Slice, siblings: list[Slice], callers: str ) -> str: + """What one writer is told: its share, what everyone else holds, and the bar. + + Written as a brief rather than a template because a writer that cannot see its siblings + will otherwise write what they are writing. Naming their angles is cheaper than discovering + the overlap at the merge and throwing the loser away. + """ + others = "\n".join(f" - {one.named()}" for one in siblings if one is not mine) + aim = f" {mine.use_case}" + if mine.angle: + aim += f"\n Angle: {mine.angle}" + if mine.why: + aim += f"\n Worth testing because: {mine.why}" + return ( - f"Write {wanted} scenarios for {contract.agent!r}, all of them within this one " - "use case:\n\n" - f" {use_case}\n\n" - "Write nothing outside it. Somebody else is covering the other use cases at the same " - "time, so a scenario that strays is either a duplicate of theirs or a gap in yours.\n\n" - "Every scenario carries this use case verbatim in `use_case`, and its own one-line " + f"Write {mine.count} scenario{'s' if mine.count != 1 else ''} for {contract.agent!r}, " + "all of them within this one slice:\n\n" + f"{aim}\n\n" + + ( + "The rest of the suite is being written at the same time by others, covering:\n" + f"{others}\n\nStay out of theirs. A scenario that strays is either a duplicate of " + "somebody else's or a gap in yours.\n\n" + if others + else "" + ) + + "Every scenario carries this use case verbatim in `use_case`, and its own one-line " "`branch` saying what makes it different from the others you write here. Branches are " "where the variety lives: the ordinary path, the branch that cannot be completed, the " "rule under pressure, state that has to carry across turns, the same request against a " "differently seeded world.\n\n" - "Look at the world first with inspect_world so every scenario names real records, and " - "read the sub-goals already defined. Work out each solution with try_calls before you " - "submit it. Submit each one with submit_scenario and then stop: do not save, and do not " - "ask what to do next. Whoever asked for this collects the suite and writes it.\n\n" - "Vary the caller across the scenarios you write. Everyone else is writing their own use " - "case and cannot see yours, so a suite where every caller is professional and formal is " - "what happens when each writer picks the safest value. Give different scenarios " - "different personalities, communication styles and accents from the values offered, and " - "let the caller suit the situation: somebody whose card was declined is not in the same " - "mood as somebody booking a routine morning ride." + callers + "What each one has to be, before you submit it:\n" + " - every value real, read out of the world with inspect_world, never invented\n" + " - an instruction that is a circumstance the person is living through, not a script " + "of lines to say\n" + " - a setup that makes true whatever the instruction presumes, and a ready check that " + "proves it\n" + " - a solution worked out with try_calls first, so the gates are not where you find " + "out it cannot be passed\n" + " - sub-goals named from the shared catalogue, and checks that assert the right call " + "with the right arguments or the right end state, never that something merely happened\n" + " - a scenario a competent agent could plausibly fail. If any correct implementation " + "passes it for free, it teaches nothing and is not worth the run\n\n" + "Look at the world first, and read the sub-goals already defined. Submit each scenario " + "with submit_scenario and then stop: do not save, and do not ask what to do next. " + "Whoever asked for this collects the suite and writes it." + callers ) -async def _write_one_use_case( +async def _write_slice( contract: AgentContract, - use_case: str, - count: int, + mine: Slice, + siblings: list[Slice], *, - index: int = 0, + index: int, destination: Path, on_event: Callable[..., Any] | None, ask: Callable[..., Any] | None, ) -> list[Scenario]: - """One use case's share, written by its own session. Returns what it proved, unsaved.""" + """One slice, written by its own session. Returns what it proved, unsaved.""" server, kept = scenario_tools( contract, destination, destination, - wanted=count, + wanted=mine.count, can_save=False, start_from=[], ) - progress.started(destination, use_case) + progress.started(destination, mine.named()) def watch(event: Any) -> None: # Report as they land rather than at the end. A slice that proves its first scenario # four minutes in is the difference between a run that looks alive and one that does not. - progress.kept(destination, use_case, len(kept)) + progress.kept(destination, mine.named(), len(kept)) if on_event: on_event(event) + allowed = [ qualified(SCENARIO_SERVER, name) for name in TOOL_NAMES if name != "save_scenarios" ] @@ -263,33 +372,33 @@ def watch(event: Any) -> None: system_prompt=( f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" f"\n\n## Its world\n\n{world_summary(destination)}" - f"\n\n## Your slice\n\nYou are writing only the scenarios for: {use_case}" + f"\n\n## Your slice\n\nYou are writing only: {mine.named()}" ), allowed_tools=allowed, mcp_servers={SCENARIO_SERVER: server}, permission_mode="default", cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), setting_sources=[], - max_turns=turns_for(count), + max_turns=turns_for(mine.count), model=chosen_model(), env=provider_env(), ) options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) - stage = Stage(options, name=f"{SKILL}:{use_case[:40]}") + stage = Stage(options, name=f"{SKILL}:{mine.named()[:40]}") try: async with stage: await stage.say( - branch_opening(contract, use_case, count, callers_for(index, count)), + brief_for(contract, mine, siblings, callers_for(index, mine.count)), on_event=watch, ) except Exception as broke: # noqa: BLE001 - one slice failing must not lose the others - progress.failed(destination, use_case, str(broke)) + progress.failed(destination, mine.named(), str(broke)) if on_event: - on_event({"type": "slice_failed", "use_case": use_case, "why": str(broke)[:300]}) + on_event({"type": "slice_failed", "slice": mine.named(), "why": str(broke)[:300]}) return list(kept) - progress.finished(destination, use_case, len(kept)) + progress.finished(destination, mine.named(), len(kept)) return list(kept) @@ -315,22 +424,140 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: return suite +def _suite_summary(suite: list[Scenario]) -> str: + """The whole suite as a reviewer needs to see it: what each row claims to test.""" + return "\n".join( + f" {one.name} | use case: {one.use_case} | branch: {one.branch} | tests: {one.tests}" + for one in suite + ) + + +async def gaps_in( + contract: AgentContract, + suite: list[Scenario], + *, + destination: Path, + wanted: int, + ask: Callable[..., Any] | None = None, +) -> list[Slice]: + """What the finished suite is missing, as slices that would fill it. + + Nobody looks at a suite written in parallel. Each writer sees its own slice and the merge + only removes collisions, so a use case that came back one short, or an obvious branch that + every writer assumed somebody else had, survives to the end and nobody notices. This is the + one pass that reads the suite as a whole. + """ + if not suite: + return [] + found: list[Slice] = [] + + @tool( + "submit_gaps", + "The gaps worth filling in this suite, as the slices that would fill them. Return " + "nothing when the suite covers what it should: a suite that is finished is a real " + "answer, and inventing work to report is worse than saying so.", + schema( + { + "gaps": { + "type": "array", + "description": "One entry per gap. Empty when the suite is covering what " + "it should.", + "items": { + "type": "object", + "properties": { + "use_case": {"type": "string"}, + "angle": { + "type": "string", + "description": "The scenario that is missing, in one line.", + }, + "why": {"type": "string"}, + }, + "required": ["use_case", "angle"], + }, + } + }, + ["gaps"], + ), + ) + async def submit_gaps(args: dict[str, Any]) -> dict[str, Any]: + for one in args.get("gaps") or []: + if not isinstance(one, dict): + continue + case = str(one.get("use_case") or "").strip() + if case: + found.append( + Slice( + use_case=case, + angle=str(one.get("angle") or "").strip(), + count=1, + why=str(one.get("why") or "").strip(), + ) + ) + return { + "content": [ + {"type": "text", "text": f"{len(found)} gap(s) recorded. Nothing else to do."} + ] + } + + server = create_sdk_mcp_server(name=REVIEW_SERVER, version="0.1.0", tools=[submit_gaps]) + allowed = [qualified(REVIEW_SERVER, "submit_gaps")] + options = ClaudeAgentOptions( + system_prompt=( + "You are reviewing a suite of tests somebody else wrote for an AI agent, in " + "parallel, each writer blind to the others. Your only job is to say what is " + "missing.\n\n" + "Look for: a use case of this agent that nothing covers; a use case covered only " + "on its ordinary path, where the branch that cannot be completed or the rule under " + "pressure is the interesting one; two rows that are the same test under different " + "names, leaving the branch one of them claimed uncovered.\n\n" + "Judge coverage of the agent, not of the plan. Do not ask for more of what is " + "already well covered, and do not report a gap you cannot name a scenario for. " + "A suite of the right size that covers what matters is finished, and saying so is " + f"the useful answer.\n\n## This agent\n\n{contract.brief()}" + ), + allowed_tools=allowed, + mcp_servers={REVIEW_SERVER: server}, + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=8, + model=chosen_model(), + env=provider_env(), + ) + options.disallowed_tools = list(UNWANTED) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + stage = Stage(options, name=f"{SKILL}:review") + try: + async with stage: + await stage.say( + f"This suite has {len(suite)} scenarios against a target of {wanted}:\n\n" + f"{_suite_summary(suite)}\n\n" + "Say what it is missing, then submit_gaps. Submit an empty list if it is " + "covering what it should." + ) + except Exception: # noqa: BLE001 - a review that fails leaves the suite as written + return [] + return found + + async def write_in_parallel( contract: AgentContract, *, out: Path | None = None, wanted: int = 10, use_cases: list[str] | None = None, + slices: list[dict] | None = None, at_once: int = AT_ONCE, + rounds: int = TOP_UP_ROUNDS, on_event: Callable[..., Any] | None = None, ask: Callable[..., Any] | None = None, ) -> list[Scenario]: - """Write a suite with one session per use case, then save it once. + """Write a suite with one session per slice, review it, fill what it missed, and save once. Sequentially, a suite costs roughly three turns a scenario against one budget, which is why - asking for forty stopped around twenty-five. Here each use case is written by its own - session, so the wall clock is the slowest use case rather than the sum of all of them, and - the turn budget is per slice rather than shared. + asking for forty stopped around twenty-five. Here the work is split into slices that run at + the same time, so the wall clock is the slowest slice rather than the sum of all of them. Saving stays here, once, for a reason: ``save_scenarios`` regenerates the index and deletes any folder it does not know about, so letting the writers save would have each of them @@ -338,23 +565,35 @@ async def write_in_parallel( """ destination = out or artifact_dir(contract.agent) cases = [case for case in (use_cases or contract.real_use_cases) if case.strip()] - if not cases: + if not cases and not slices: # Nothing to partition on. One writer, the ordinary path, rather than no scenarios. return await write(contract, out=destination, wanted=wanted, on_event=on_event, ask=ask) - allocation = shares(wanted, cases) - progress.planned(destination, allocation, at_once=at_once, asked=wanted) + at_once = max(1, min(at_once or AT_ONCE, MOST_AT_ONCE)) + allocation = planned(wanted, cases, slices) + progress.planned( + destination, + [(one.named(), one.count) for one in allocation], + at_once=at_once, + asked=wanted, + ) if on_event: - on_event({"type": "planned", "slices": allocation, "at_once": at_once}) + on_event( + { + "type": "planned", + "slices": [(one.named(), one.count) for one in allocation], + "at_once": at_once, + } + ) - limit = asyncio.Semaphore(max(1, at_once)) + limit = asyncio.Semaphore(at_once) - async def guarded(use_case: str, count: int, index: int) -> list[Scenario]: + async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenario]: async with limit: - return await _write_one_use_case( + return await _write_slice( contract, - use_case, - count, + mine, + siblings, index=index, destination=destination, on_event=on_event, @@ -362,14 +601,46 @@ async def guarded(use_case: str, count: int, index: int) -> list[Scenario]: ) written = await asyncio.gather( - *( - guarded(case, count, index) - for index, (case, count) in enumerate(allocation) - ), + *(guarded(one, allocation, index) for index, one in enumerate(allocation)), return_exceptions=False, ) - suite = merged([load_scenarios(destination), *written]) + + # Read the whole thing and fill what nobody covered. Bounded, because a reviewer asked + # twice will always find something smaller to say. + for _ in range(max(0, rounds)): + if len(suite) >= wanted: + break + missing = await gaps_in( + contract, suite, destination=destination, wanted=wanted, ask=ask + ) + missing = missing[: max(0, wanted - len(suite))] + if not missing: + break + if on_event: + on_event({"type": "topping_up", "slices": [one.named() for one in missing]}) + progress.planned( + destination, + [(one.named(), one.count) for one in [*allocation, *missing]], + at_once=at_once, + asked=wanted, + ) + for one in [*allocation, *missing]: + if one in allocation: + progress.finished(destination, one.named(), one.count) + more = await asyncio.gather( + *( + guarded(one, missing, len(allocation) + index) + for index, one in enumerate(missing) + ), + return_exceptions=False, + ) + before = len(suite) + suite = merged([suite, *more]) + allocation = [*allocation, *missing] + if len(suite) == before: + break + write_scenarios(suite, destination, load_catalogue(destination)) progress.settled(destination, kept_total=len(suite)) if on_event: diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 474f8ce7..67405d97 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -165,6 +165,17 @@ through the same three gates. Writing a suite yourself with `submit_scenario` co turns per scenario against one budget, so a request for twenty or fifty runs out long before it finishes, and what does get written is lost because nothing was saved. +**Pass your plan to it.** The tool takes the split as an argument, and you have just read the +world and know which use cases have +something in them; it is the part of this only you can do. Each slice names its use case, +the angle it should take, how many scenarios it is worth, and why. Left to itself the work is +divided evenly, which is how a use case with one real branch pads to three and one with six gets +three. + +A large request comes back a batch at a time rather than all at once, with the rest offered. When +that happens, show what came back and ask whether to carry on, change direction first, or stop. +Do not silently loop until the number is reached. + Use `submit_scenario` for what it is good at: one scenario somebody asked for by name, a replacement for one that came back wrong, or filling a specific gap in a suite that already exists. Anything described as a number of scenarios is a suite. From 4e69fe37e955e6b84a5fac8ec403969b98f5bf7c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 00:27:58 +0530 Subject: [PATCH 03/41] perf(harness): make model thinking env-configurable via ALK_HARNESS_THINKING, default off for stage speed --- src/fi/alk/harness/build.py | 2 ++ src/fi/alk/harness/config.py | 19 +++++++++++++++++++ src/fi/alk/harness/reception.py | 2 ++ src/fi/alk/harness/scenarios.py | 3 +++ 4 files changed, 26 insertions(+) diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index a6341e49..7a0db07b 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -26,6 +26,7 @@ permission_gate, provider_env, provisioning, + thinking_config, ) from .contract import AgentContract from .session import Stage @@ -202,6 +203,7 @@ def open_stage( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 44e30634..9cbb35ba 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -52,6 +52,24 @@ def chosen_model(model: str | None = None) -> str: return model or os.environ.get("ALK_HARNESS_MODEL", DEFAULT_MODEL) +def thinking_config() -> dict[str, Any]: + """How much the model may think, from ALK_HARNESS_THINKING. + + The Claude Code CLI defaults to adaptive thinking. In this harness the correctness of what a + stage produces is re-checked by code gates (a scenario is proved against the real world, a + contract is validated), so the model's private reasoning is spent on decisions the gates make + again anyway. Left unset, that reasoning was the majority of generated tokens and the majority + of wall time. Default to disabled for speed; ``adaptive`` restores the old behaviour, and an + integer sets an explicit budget for models that still honour one. + """ + setting = os.environ.get("ALK_HARNESS_THINKING", "disabled").strip().lower() + if setting in {"adaptive", "on", "auto"}: + return {"type": "adaptive", "display": "omitted"} + if setting.isdigit() and int(setting) > 0: + return {"type": "enabled", "budget_tokens": int(setting), "display": "omitted"} + return {"type": "disabled"} + + def provisioning(enabled: bool | None = None) -> bool: """Compatibility switch for callers selecting the legacy provisioning surface. @@ -123,6 +141,7 @@ def read_only_session( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(granted=allowed) + options.thinking = thinking_config() return options diff --git a/src/fi/alk/harness/reception.py b/src/fi/alk/harness/reception.py index 266f9252..7e858380 100644 --- a/src/fi/alk/harness/reception.py +++ b/src/fi/alk/harness/reception.py @@ -25,6 +25,7 @@ gate_hooks, permission_gate, provider_env, + thinking_config, ) from .session import Stage from .sources import AgentSource, clone_github_repository, resolve, supported @@ -150,6 +151,7 @@ async def point_at_agent(args: dict[str, Any]) -> dict[str, Any]: options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name="reception"), found diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index e877ce36..a0c68af8 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -27,6 +27,7 @@ load_skill, permission_gate, provider_env, + thinking_config, ) from . import progress from .catalogue import load_catalogue @@ -106,6 +107,7 @@ def open_stage( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name=SKILL), destination @@ -386,6 +388,7 @@ def watch(event: Any) -> None: options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() stage = Stage(options, name=f"{SKILL}:{mine.named()[:40]}") try: async with stage: From fa287cd93f9c59305b0ec0c4e813eaf351dd9079 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 00:27:58 +0530 Subject: [PATCH 04/41] fix(harness): unblock hosted voice runs (agent Vertex creds mount, provider passthrough, disable unlicensed ai_coustics, accent-driven simulator voice, attached-store schema restore, worker env allowlist) --- src/fi/alk/harness/provision.py | 15 +++++++ src/fi/alk/harness/run/live.py | 31 ++++++++++++++ src/fi/alk/harness/run/sdk_voice.py | 47 ++++++++++++++++++++- src/fi/alk/harness/secrets.py | 32 ++++++++++++++ src/fi/alk/harness/world/stores/postgres.py | 11 +++-- 5 files changed, 131 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index e01452a0..4c858713 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -1738,6 +1738,17 @@ def provision( else: compose = None managed = False + # The harness's fidelity order is provisioned > adopted > generated: run the agent's real + # services whenever it ships them. _managed_compose only models "agent + datastore" and + # silently drops any other service the agent's tools are actually served by -- an HTTP + # tools-api, a queue, a mock upstream -- which then leaves the world with no endpoint to + # forward to, so every tool call comes back "no such tool". So prefer the agent's own shipped + # Compose whenever it ships one (its real tool services come up and the world forwards to + # them), and fall back to the generated adapter only for agents that ship no usable Compose. + if compose is None: + shipped = compose_file(source_root) + if shipped is not None: + compose = shipped if compose is None and contract is not None: if not packaging.candidates and not (source_root / "Dockerfile").is_file(): try: @@ -2216,6 +2227,10 @@ def start_runtime( arguments.extend(("--volume", f"{source}:{target}:ro")) injected[name] = target mounted_credentials.add(name) + # A submitted service that declares its own credential volume has already been mounted (the + # host secret now lives at the container target). Re-validating that container path as a host + # file below would always fail, so only resolve GOOGLE_APPLICATION_CREDENTIALS here when it + # arrived as an injected host path (the generated-runtime path, which mounts no credentials). google_path = injected.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() if google_path and "GOOGLE_APPLICATION_CREDENTIALS" not in mounted_credentials: google_source = Path(google_path).expanduser() diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py index 90b4978e..e6873e64 100644 --- a/src/fi/alk/harness/run/live.py +++ b/src/fi/alk/harness/run/live.py @@ -278,6 +278,37 @@ def wire( "TOOLS_API_URL": url, "LIVEKIT_AGENT_NAME": agent_name, } + # The harness-generated agent-runtime does not carry the agent's own .env.local, and + # runtime_configuration_names only covers datastore config -- not the provider + # credentials the worker needs to actually run: LiveKit to register and place the + # call, Deepgram for STT/TTS, Vertex for the LLM. Pass them through from this + # process's environment (the sandbox has placed them here). + for _cred in ( + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + ): + _value = os.environ.get(_cred, "").strip() + if _value: + runtime_overrides.setdefault(_cred, _value) + # An agent's Compose commonly mounts its Vertex credential from an env-var source, e.g. + # ${VERTEX_CREDENTIALS:-/dev/null}:/etc/vertex/creds.json. With that variable unset the + # placeholder is mounted and the agent's own LLM cannot authenticate, so point it at the + # same resolved Google credential the harness already holds for the run. + _google_creds = runtime_overrides.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() + if _google_creds: + runtime_overrides.setdefault("VERTEX_CREDENTIALS", _google_creds) + # Voice agents often gate an audio-enhancement plugin on a license the harness cannot + # supply (e.g. ai_coustics noise cancellation). Unauthorized, it raises on the first + # inbound audio frame and the worker drops the call right after its greeting. Agents + # that read this flag skip that plugin; agents that do not simply ignore it. A run may + # override it to keep enhancement on when a license is present. + runtime_overrides.setdefault("DISABLE_AI_COUSTICS", "1") os.environ["LIVEKIT_TARGET_AGENT_NAME"] = agent_name caller_phone = fixture_phone(scenario) if caller_phone: diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 30c0b004..a612f78c 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -81,18 +81,61 @@ def model(kind: str, provider: str) -> str: "verification code should be sent, and disclose the actual code only after the " "agent says it was sent and explicitly asks you to read it. Answer repair questions " "with the missing fact, not by restarting the request. Never repeat the same answer " - "more than twice. When the requested outcome is complete, thank the agent and end " - "the call." + "more than twice. Do not end the call while the agent still has a step to finish: when " + "it asks to proceed, say yes and wait for it to actually complete the task and confirm " + "it is done. Only once the outcome is actually done and confirmed, thank the agent and " + "end the call." ), allow_interruptions=True, ) +# Deepgram aura encodes the speaker in the model name, so a persona's accent selects a voice by +# choosing the aura model. Only English accents aura actually ships are mapped; anything else keeps +# the default so a caller never loses a voice to an accent the provider cannot render. +_AURA_BY_ACCENT: dict[str, dict[str, list[str]]] = { + "american": { + "female": ["aura-asteria-en", "aura-luna-en", "aura-hera-en", "aura-stella-en"], + "male": ["aura-orion-en", "aura-arcas-en", "aura-perseus-en", "aura-zeus-en"], + }, + "british": {"female": ["aura-athena-en"], "male": ["aura-helios-en"]}, + "irish": {"female": ["aura-athena-en"], "male": ["aura-angus-en"]}, + "australian": {"female": ["aura-athena-en"], "male": ["aura-helios-en"]}, +} + + +def _aura_voice_for(persona: dict) -> str: + """A stable aura voice for one caller, chosen by accent and gender. + + Callers who share an accent still differ: the voice within the accent's set is picked by the + persona name, so a suite varies without being random between runs of the same scenario. + """ + accent = str(persona.get("accent") or "").strip().lower() + gender = str(persona.get("gender") or "").strip().lower() + if gender not in ("male", "female"): + gender = "female" + bucket = next( + (voices for key, voices in _AURA_BY_ACCENT.items() if key in accent), + _AURA_BY_ACCENT["american"], + ) + voices = bucket.get(gender) or next(iter(bucket.values())) + index = sum(ord(character) for character in str(persona.get("name") or "")) % len(voices) + return voices[index] + + def _scenario() -> simulate.Scenario: fixture = _json_env("HARNESS_FIXTURE", {}) persona = _json_env("HARNESS_PERSONA", {"name": "customer"}) persona = dict(persona) if isinstance(persona, dict) else {"name": "customer"} persona["role"] = "customer" + # Give the caller a voice from its accent when the suite runs on aura and none was set, so + # different callers sound different and match the accent the scenario wrote. + if ( + not persona.get("voice") + and not persona.get("voice_id") + and os.environ.get("SIMULATOR_TTS_PROVIDER", "deepgram").lower() == "deepgram" + ): + persona["voice"] = _aura_voice_for(persona) metadata = dict(persona.get("metadata") or {}) if isinstance(fixture, dict) and fixture.get("phone"): # LiveKit exposes this as participant metadata/attributes. A target can diff --git a/src/fi/alk/harness/secrets.py b/src/fi/alk/harness/secrets.py index 018ecdbf..35067b74 100644 --- a/src/fi/alk/harness/secrets.py +++ b/src/fi/alk/harness/secrets.py @@ -108,6 +108,38 @@ def worker_environment( "FI_BASE_URL", "FI_API_KEY", "FI_SECRET_KEY", + # Runner-owned Docker runtime + voice configuration. The harness starts store and agent + # containers on the host daemon and must reach them: ALK_DOCKER_NETWORK lets the store be + # reached by container name on a shared network, ALK_DOCKER_PUBLISHED_HOST/BIND_HOST give + # the host published services are on. Without these the child defaults to 127.0.0.1 -- + # its own loopback inside the sandbox -- and every store probe is refused. + "ALK_DOCKER_NETWORK", + "ALK_DOCKER_PUBLISHED_HOST", + "ALK_DOCKER_BIND_HOST", + "ALK_RUNNER_CONTAINER", + "ALK_HARNESS_MODEL", + "ALK_AGENT_MODEL", + "ALK_JUDGE_MODEL", + "ALK_USER_MODEL", + "CLOUD_ML_REGION", + "HARNESS_WEBHOOK_HOST", + "HARNESS_WEBHOOK_PORT", + "HARNESS_WEBHOOK_URL", + "HARNESS_RUNTIME_WEBHOOK_URL", + "HARNESS_VOICE_CASE", + "HARNESS_VOICE_INFRA_RETRIES", + "LIVEKIT_TARGET_AGENT_NAME", + # Local-dev convenience: let the developer's provider creds from their local environment + # reach the worker directly. A hosted provider supplies these through secret_refs instead. + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "ACCEPTANCE_LIVEKIT_URL", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", } child = {name: value for name, value in host.items() if name in allowed} reserved = { diff --git a/src/fi/alk/harness/world/stores/postgres.py b/src/fi/alk/harness/world/stores/postgres.py index 0a89b444..8f9f83b6 100644 --- a/src/fi/alk/harness/world/stores/postgres.py +++ b/src/fi/alk/harness/world/stores/postgres.py @@ -237,9 +237,14 @@ def save_to(self, path: str | Path) -> None: def load_from(self, path: str | Path) -> None: root = Path(path) schema = root / SCHEMA - if not schema.exists(): - raise StoreError(f"no saved Postgres schema at {schema}") - self.apply(schema.read_text(encoding="utf-8")) + # A standalone scenario store starts empty and needs the DDL; a compose-provisioned + # (Attached) store saves no schema.sql because store.json already carries the applied + # CREATE scripts, which Held.load_from replays. Apply schema.sql only when it exists and + # the tables are not already there, so restore works either way and never double-applies. + with self._connect() as connection: + has_schema = bool(self._tables(connection)) + if not has_schema and schema.exists(): + self.apply(schema.read_text(encoding="utf-8")) Held.load_from(self, root) # -- what a scenario changes ----------------------------------------------------- From 170fe05e58d00a6494e02348d754e60574151d0d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 03:31:23 +0530 Subject: [PATCH 05/41] fix(harness): restore local-env GOOGLE credential fallback and raise stage idle timeout for parallel suite generation --- src/fi/alk/harness/provision.py | 9 +++++++++ src/fi/alk/harness/session.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index 4c858713..f05810f2 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -2104,6 +2104,15 @@ def _runtime_credential_mounts( # the worker to see the placeholder. Platform credentials always get an # ALK-owned destination that cannot collide with submitted mounts. target = f"/run/harness-secrets/{runtime.name}" + if source is None or not _valid_google_credentials(source): + # Local development supplies the credential through the sandbox's own environment + # rather than an uploaded runtime configuration; fall back to it, at an ALK-owned + # destination so a placeholder Compose mount cannot shadow it. + env_value = os.environ.get(name, "").strip() + env_path = Path(env_value).expanduser() if env_value else None + if env_path is not None and _valid_google_credentials(env_path): + source = env_path + target = f"/run/harness-secrets/{env_path.name}" if source is None or not _valid_google_credentials(source): raise ProvisionError( "the submitted runtime needs GOOGLE_APPLICATION_CREDENTIALS, but neither its " diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index af652abd..4f2db62a 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -38,7 +38,7 @@ # remain alive forever after a dropped upstream stream, though, which previously left a hosted # job looking healthy while making no progress. Bound *inactivity*, not total stage duration: # long scenario suites remain valid as long as they keep producing observable work. -STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "180")) +STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "600")) STAGE_IDLE_RETRIES = int(os.getenv("ALK_STAGE_IDLE_RETRIES", "1")) From a9db3600c2281c9198a179d1f0cc024469f7eba3 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 03:31:23 +0530 Subject: [PATCH 06/41] feat(harness): drive simulator voice from persona accent, STT language from persona, and inject scenario background noise --- src/fi/alk/harness/background_noise.py | 56 +++++++++++++++++ src/fi/alk/harness/run/sdk_voice.py | 36 ++++++++++- src/fi/alk/harness/run/simulation.py | 18 ++++++ src/fi/simulate/simulation/engines/livekit.py | 60 ++++++++++++++++++- 4 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 src/fi/alk/harness/background_noise.py diff --git a/src/fi/alk/harness/background_noise.py b/src/fi/alk/harness/background_noise.py new file mode 100644 index 00000000..b6130245 --- /dev/null +++ b/src/fi/alk/harness/background_noise.py @@ -0,0 +1,56 @@ +"""Choose the caller-side ambient noise a scenario should be heard through. + +A scenario that sets ``background_noise`` wants the agent to handle a caller phoning from somewhere +real: a car, a street, an office. The clip is chosen here and handed to the voice engine, which +mixes it under the simulated caller's audio. + +Two sources, in order. A run may point ``ALK_BACKGROUND_NOISE_CATALOG`` at a JSON file of clips +(each with an ``environment`` tag and a ``url`` or ``path``); the catalog stays a local file so its +asset locations are never committed here. When no catalog matches, a LiveKit builtin clip is used, +which needs no external asset and always works. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +# LiveKit ships these; they are the reliable default when no custom catalog is configured. +_BUILTIN_BY_ENVIRONMENT: dict[str, str] = { + "street": "CITY_AMBIENCE", + "transit": "CITY_AMBIENCE", + "vehicle": "CITY_AMBIENCE", + "outdoors": "FOREST_AMBIENCE", + "retail": "CROWDED_ROOM", + "office": "OFFICE_AMBIENCE", + "home": "OFFICE_AMBIENCE", +} +_DEFAULT_BUILTIN = "OFFICE_AMBIENCE" + + +def source_for(environment: str = "", seed: str = "") -> str: + """A background-noise source for a scenario. + + Returns a ``url``/``path`` from the configured catalog when one matches the environment, else the + name of a LiveKit builtin clip. The choice is deterministic in ``seed`` so the same scenario + hears the same place across runs. + """ + env = (environment or "").strip().lower() + catalog = os.environ.get("ALK_BACKGROUND_NOISE_CATALOG", "").strip() + if catalog and Path(catalog).is_file(): + try: + entries = json.loads(Path(catalog).read_text(encoding="utf-8")) + except (OSError, ValueError): + entries = [] + if isinstance(entries, list) and entries: + pool = [ + entry + for entry in entries + if str(entry.get("environment", "")).strip().lower() == env + ] or entries + chosen = pool[sum(ord(character) for character in (seed or env or "x")) % len(pool)] + located = str(chosen.get("url") or chosen.get("path") or "").strip() + if located: + return located + return _BUILTIN_BY_ENVIRONMENT.get(env, _DEFAULT_BUILTIN) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index a612f78c..5597ac46 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -41,6 +41,40 @@ def _json_env(name: str, default): return parsed +# Language names a persona may carry, to the codes Deepgram STT expects. Unrecognised values that +# already look like a code are passed through; everything else falls back to English. +_LANGUAGE_CODES: dict[str, str] = { + "english": "en", "hindi": "hi", "hinglish": "hi", "spanish": "es", "french": "fr", + "german": "de", "portuguese": "pt", "italian": "it", "dutch": "nl", "japanese": "ja", + "korean": "ko", "mandarin": "zh", "chinese": "zh", "arabic": "ar", "russian": "ru", + "tamil": "ta", "telugu": "te", "bengali": "bn", +} + + +def _persona_stt_language() -> str: + """The STT language for this call's caller, from the persona's languages. + + An explicit SIMULATOR_STT_LANGUAGE always wins. Otherwise the persona's first language is used, + so a caller who speaks Hindi is transcribed as Hindi rather than forced to English. + """ + override = os.environ.get("SIMULATOR_STT_LANGUAGE", "").strip() + if override: + return override + raw = os.environ.get("HARNESS_PERSONA", "").strip() + if raw: + try: + languages = (json.loads(raw) or {}).get("languages") or [] + except ValueError: + languages = [] + if isinstance(languages, list) and languages: + first = str(languages[0]).strip().lower() + if first in _LANGUAGE_CODES: + return _LANGUAGE_CODES[first] + if 2 <= len(first) <= 5 and first.replace("-", "").isalpha(): + return first + return "en" + + def _simulator() -> simulate.SimulatorAgentDefinition: llm_provider = os.environ.get("SIMULATOR_LLM_PROVIDER", "google") stt_provider = os.environ.get("SIMULATOR_STT_PROVIDER", "deepgram") @@ -67,7 +101,7 @@ def model(kind: str, provider: str) -> str: stt={ "provider": stt_provider, "model": model("stt", stt_provider), - "language": os.environ.get("SIMULATOR_STT_LANGUAGE", "en"), + "language": _persona_stt_language(), }, tts={ "provider": tts_provider, diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index 85ae82f0..b6ed1792 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -586,6 +586,24 @@ def placed_once() -> tuple[int, dict[str, Any]]: os.environ["HARNESS_FIXTURE"] = json.dumps( scenario.fixture, ensure_ascii=False, default=str ) + # A scenario that asks to be heard through background noise selects a clip for the + # caller's environment; the voice engine mixes it under the caller. Cleared otherwise so + # a previous call's noise never leaks into a quiet one. + if getattr(scenario, "background_noise", False): + from ..background_noise import source_for + + environment = "" + if isinstance(scenario.fixture, dict): + environment = str( + scenario.fixture.get("environment") + or scenario.fixture.get("location") + or "" + ) + os.environ["HARNESS_BACKGROUND_NOISE"] = source_for( + environment, seed=scenario.name + ) + else: + os.environ.pop("HARNESS_BACKGROUND_NOISE", None) code = place_the_call( os.environ.get("HARNESS_VOICE_CASE", "2.1.2"), on_exchange=live_exchange if on_exchange else None, diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index ff10928c..cabd48a0 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -16,7 +16,16 @@ try: from livekit import api, rtc - from livekit.agents import Agent, AgentSession, RunContext, function_tool, metrics + from livekit.agents import ( + Agent, + AgentSession, + AudioConfig, + BackgroundAudioPlayer, + RunContext, + function_tool, + metrics, + ) + from livekit.agents.voice.background_audio import BuiltinAudioClip from livekit.agents.types import ( ATTRIBUTE_TRANSCRIPTION_TRACK_ID, TOPIC_TRANSCRIPTION, @@ -320,8 +329,57 @@ async def start_session( room=room, room_options=RoomOptions(**room_kwargs), ) + await self._maybe_start_background_audio(room, session) return session + async def _maybe_start_background_audio( + self, room: "rtc.Room", session: "AgentSession" + ) -> None: + """Mix caller-side ambient noise under the simulated caller, if the run asked for it. + + Off unless HARNESS_BACKGROUND_NOISE names a source: a LiveKit builtin clip name, or an + http(s) URL to an ambient file. Any failure is swallowed, because a call without ambience is + preferable to a dropped one. + """ + source = os.environ.get("HARNESS_BACKGROUND_NOISE", "").strip() + if not source: + return + + def _download() -> str | None: + import tempfile + import urllib.request + + try: + suffix = ( + ".mp3" if ".mp3" in source else ".ogg" if ".ogg" in source else ".wav" + ) + with urllib.request.urlopen(source, timeout=15) as response: + data = response.read() + handle = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) + handle.write(data) + handle.close() + return handle.name + except Exception: + return None + + try: + volume = float(os.environ.get("HARNESS_BACKGROUND_NOISE_VOLUME", "0.3")) + if source.startswith(("http://", "https://")): + clip_source: Any = await asyncio.to_thread(_download) + if not clip_source: + return + else: + clip_source = getattr(BuiltinAudioClip, source, None) + if clip_source is None: + return + player = BackgroundAudioPlayer( + ambient_sound=AudioConfig(clip_source, volume=volume) + ) + await player.start(room=room, agent_session=session) + self._background_player = player + except Exception: + logger.warning("background audio not started", exc_info=True) + def open_conversation(self) -> None: if self._session is None: raise RuntimeError("simulator_session_not_started") From 4e9111f238aea2122520e14e0e7e5092cffdb5ba Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 10:14:16 +0530 Subject: [PATCH 07/41] feat(harness): vary caller accents and end calls when the agent loops --- src/fi/alk/harness/run/sdk_voice.py | 10 ++++++---- src/fi/alk/harness/scenarios.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 5597ac46..7bcacbcb 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -115,10 +115,12 @@ def model(kind: str, provider: str) -> str: "verification code should be sent, and disclose the actual code only after the " "agent says it was sent and explicitly asks you to read it. Answer repair questions " "with the missing fact, not by restarting the request. Never repeat the same answer " - "more than twice. Do not end the call while the agent still has a step to finish: when " - "it asks to proceed, say yes and wait for it to actually complete the task and confirm " - "it is done. Only once the outcome is actually done and confirmed, thank the agent and " - "end the call." + "more than twice. Wait for the agent to finish the task rather than ending as soon as " + "it asks to proceed: say yes and let it complete and confirm the outcome. But if the " + "agent gives essentially the same response two or three times without making progress, " + "do not keep looping: say once that it is not working and that you will try again " + "later, then end the call. Once the outcome is actually completed and confirmed, thank " + "the agent and end the call." ), allow_interruptions=True, ) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index a0c68af8..85c23930 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -275,16 +275,23 @@ def callers_for(index: int, wanted: int) -> str: if not people: return "" picks = [people[(index + step) % len(people)] for step in range(max(1, wanted))] - accent = accents[index % len(accents)] if accents else "" said = ( "\n\nStart from these callers, and move off them only where the scenario calls for " f"somebody else: {', '.join(picks)}." ) - if accent: + if accents: + # Spread several offered accents across this writer's callers rather than naming just one, + # so the suite does not collapse to a single default accent and the agent's speech handling + # is genuinely varied. + spread = [ + accents[(index + step) % len(accents)] + for step in range(min(len(accents), max(2, wanted))) + ] said += ( - f" At least one of your callers has a {accent} accent. Other writers are covering " - "other use cases with other callers, so a suite where everyone sounds the same is " - "what happens when each of us picks the safest option." + " Give your callers varied accents from the offered set, a different one per caller " + f"where it fits rather than defaulting everyone to the same accent: {', '.join(spread)}. " + "A suite where every caller sounds the same is a missed test of the agent's speech " + "handling, so do not make them all American unless a scenario truly requires it." ) return said From a2d4a1e3115e9e432edc8694e3fcc1ce9f25990e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 10:14:45 +0530 Subject: [PATCH 08/41] feat(simulator): pick Cartesia voice and language by persona accent and gender --- .../data/voices_by_language_and_gender.json | 751 ++++++++++++++++++ src/fi/alk/harness/run/sdk_voice.py | 140 +++- 2 files changed, 878 insertions(+), 13 deletions(-) create mode 100644 src/fi/alk/harness/run/data/voices_by_language_and_gender.json diff --git a/src/fi/alk/harness/run/data/voices_by_language_and_gender.json b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json new file mode 100644 index 00000000..13e3028c --- /dev/null +++ b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json @@ -0,0 +1,751 @@ +{ + "en": { + "female": [ + "6ccbfb76-1fc6-48f7-b71d-91ac6298247b", + "e07c00bc-4134-4eae-9ea4-1a55fb45746b", + "f786b574-daa5-4673-aa0c-cbe3e8534c02", + "9626c31c-bec5-4cca-baa8-f8ba9e84c8bc", + "f9836c6e-a0bd-460e-9d3c-f7299fa60f94", + "829ccd10-f8b3-43cd-b8a0-4aeaa81f3b30", + "ec1e269e-9ca0-402f-8a18-58e0e022355a", + "66c6b81c-ddb7-4892-bdd5-19b5a7be38e7", + "a7b8d8fa-f6e5-4908-900e-0c11d1d82519", + "999df508-4de5-40a7-8bd3-8c12f678c284", + "26403c37-80c1-4a1a-8692-540551ca2ae5", + "694f9389-aac1-45b6-b726-9d9369183238", + "248be419-c632-4f23-adf1-5324ed7dbf1d", + "bf0a246a-8642-498a-9950-80c35e9276b5", + "57dcab65-68ac-45a6-8480-6c4c52ec1cd1", + "78ab82d5-25be-4f7d-82b3-7ad64e5b85b2", + "794f9389-aac1-45b6-b726-9d9369183238", + "03496517-369a-4db1-8236-3d3ae459ddf7", + "e8e5fffb-252c-436d-b842-8879b84445b6", + "b7d50908-b17c-442d-ad8d-810c63997ed9", + "32b3f3c5-7171-46aa-abe7-b598964aa793", + "00a77add-48d5-4ef6-8157-71e5437b282d", + "4af7c703-f2a9-45dd-a7fd-724cf7efc371", + "156fb8d2-335b-4950-9cb3-a2d33befec77", + "8d8ce8c9-44a4-46c4-b10f-9a927b99a853", + "c2ac25f9-ecc4-4f56-9095-651354df60c0", + "5c42302c-194b-4d0c-ba1a-8cb485c84ab9", + "3b554273-4299-48b9-9aaf-eefd438e3941", + "71a7ad14-091c-4e8e-a314-022ece01c121", + "e3827ec5-697a-4b7c-9704-1a23041bbc51", + "8f091740-3df1-4795-8bd9-dc62d88e5131", + "5abd2130-146a-41b1-bcdb-974ea8e19f56", + "91b4cf29-5166-44eb-8054-30d40ecc8081", + "f6ff7c0c-e396-40a9-a70b-f7607edb6937", + "11af83e2-23eb-452f-956e-7fee218ccb5c", + "e13cae5c-ec59-4f71-b0a6-266df3c9bb8e", + "6adbb439-0865-468c-9e68-adbb0eb2e71c", + "a01c369f-6d2d-4185-bc20-b32c225eab70", + "7ea5e9c2-b719-4dc3-b870-5ba5f14d31d8", + "f8f5f1b2-f02d-4d8e-a40d-fd850a487b3d", + "d7e54830-4754-4b17-952c-bcdb7e80a2fb", + "a38e4e85-e815-43ab-acf1-907c4688dd6c", + "f31cc6a7-c1e8-4764-980c-60a361443dd1", + "21b81c14-f85b-436d-aff5-43f2e788ecf8", + "f6141af3-5f94-418c-80ed-a45d450e7e2e", + "8985388c-1332-4ce7-8d55-789628aa3df4", + "043cfc81-d69f-4bee-ae1e-7862cb358650", + "1d3ba41a-96e6-44ad-aabb-9817c56caa68", + "c8605446-247c-4d39-acd4-8f4c28aa363c", + "607167f6-9bf2-473c-accc-ac7b3b66b30b", + "cccc21e8-5bcf-4ff0-bc7f-be4e40afc544", + "55deba52-bc73-4481-ab69-9c8831c8a7c3", + "996a8b96-4804-46f0-8e05-3fd4ef1a87cd", + "bf991597-6c13-47e4-8411-91ec2de5c466", + "daf747c6-6bc2-4083-bd59-aa94dce23f5d", + "c9440d34-5641-427b-bbb7-80ef7462576d", + "5c9e800f-2a92-4720-969b-99c4ab8fbc87", + "6d287143-8db3-434a-959c-df147192da27", + "56b87df1-594d-4135-992c-1112bb504c59", + "0c8ed86e-6c64-40f0-b252-b773911de6bb", + "573e3144-a684-4e72-ac2b-9b2063a50b53", + "15a9cd88-84b0-4a8b-95f2-5d583b54c72e", + "f4e8781b-a420-4080-81cf-576331238efa", + "a8136a0c-9642-497a-882d-8d591bdcb2fa", + "57b6bf63-c7a1-4ffc-8e10-23bf45152dd6", + "5e10a334-7fa5-46d4-a64b-5ae6185da3fd", + "761afc95-bef5-44dd-aa07-d3c678912e43", + "f9fc912e-52f0-448a-8bfa-47e9ca75f25a", + "2747b6cf-fa34-460c-97db-267566918881", + "af346552-54bf-4c2b-a4d4-9d2820f51b6c", + "d3e03deb-5439-4203-add1-ca9a7501eaa7", + "04bfd756-4fd4-42c2-9ccf-37f647c5bf54", + "ca566b43-944e-4474-b494-7d9f0695f307", + "4d3d2e9c-14e4-4802-a8d8-bd5268a73fde", + "8634bd27-0acf-4056-b014-4fea0385ed9e", + "b56c6aac-f35f-46f7-9361-e8f078cec72e", + "f0377496-2708-4cc9-b2f8-1b7fdb5e1a2a", + "0a9a5903-0a30-4d2e-b6b6-891f73d4b4e0", + "f6ce3444-478b-4ce4-982e-bcb72dffe7aa", + "0d2162c2-2fe9-40a7-b3c1-43eab576a64b", + "cb6a8744-41b0-4cdc-b643-fabeb545c6a9", + "e4d5f4c4-6601-4779-bee1-b3c14d629dc6", + "3d9b50f9-10c5-4026-9ae1-c4a698f67fc5", + "eb649460-7e23-43bc-ad20-0a7a2749b938", + "1f575487-6f3d-40e0-862a-814f55b5fb15", + "050f5a7a-9d2b-4b76-84e3-2d056a0a3eb0", + "6fbca103-0f7f-4e49-97ed-49a53b4f3534", + "87041166-c212-4838-9028-05d7437df750", + "9329fbdb-e285-4fba-95ec-592e15f14476", + "eef47c0d-cb49-4160-a4a0-6b97ed4c81e6", + "69092565-1c93-4a88-9f2c-ac8cddaf9f65", + "d6b0c62a-c7ff-477c-9a1f-eadd64b94360", + "80c81aee-b6ad-4d12-9af8-a9c79c2e141d", + "ca31ce53-ebf6-4e51-b87d-2f65d5d1f7f8", + "aef96ff9-4578-4b5d-9744-7fb347cbe4d4", + "643f5eee-459d-4b41-b4fc-0b8407139be6", + "dcc82bcd-647e-4478-955f-8232d5122f8b", + "045f0292-0731-4a4c-971d-64594fc2c35a", + "86600680-b836-41e1-9916-8475728dcc14", + "b5c1bab5-f036-481f-9295-4db6f06f6443", + "4b1e0bf9-53a0-4e9e-8664-ba1314dbcb38", + "e5a6cd18-d552-4192-9533-82a08cac8f23", + "ea93f57f-7c71-4d79-aeaa-0a39b150f6ca", + "63927f41-9616-4ac2-89cf-f3afa346e0ef", + "3308b492-50cc-417e-89dd-1f446c574546", + "320f7211-3dc3-4292-89b1-3661e8cac27c", + "a2364c9d-1fe3-4553-9eff-100c4fe5ffc8", + "48369ca9-0645-40de-9821-0d55e18a03c2", + "d6905573-8e91-4e32-b103-fd4d1205cd87", + "1ac31ebd-9113-405b-9d80-4a4bbbeea91c", + "3f38cbe2-ce6a-4051-b5dc-2b2ee20b9bc1", + "083de431-6b5c-4b18-a2dc-264eafa205f2", + "cec7cae1-ac8b-4a59-9eac-ec48366f37ae", + "8a1b8af0-c4f6-423f-a268-5507fd4aefdf", + "19e399df-5b30-4fba-9d1d-99434f993614", + "efc5488b-5429-4e72-aaa2-570981cf47d9", + "cc00e582-ed66-4004-8336-0175b85c85f6", + "3af40927-948e-429b-b92d-e2158f79fb9f", + "64b2a604-f0de-449f-9d90-255602357c05", + "c7c790c5-2bf4-47e4-bc83-5f43e61f3803", + "f4c1a0b2-669d-403f-b440-4b34b34856aa", + "cbaf8084-f009-4838-a096-07ee2e6612b1", + "c1b9a03e-747f-40ad-8e7b-18caf8aaac0b", + "e2d08065-b658-466b-ad52-cef8ee21d307", + "f762e181-ddc7-486e-9a48-636bd7e229d4", + "3ef78ba6-9aaa-46a2-b5b5-f9ded76a2370", + "a7a59115-2425-4192-844c-1e98ec7d6877", + "f39d8500-0d9b-4b8b-a080-38f5188f5892", + "1b4ea5fb-b1c0-43ee-a7be-4e315878c2b1", + "01eaafa9-308a-4276-a017-6ab0cf061b1f", + "03b1c65d-4b7f-4c09-91a8-e2f6f78cb2c9", + "4e41a434-85fc-4614-b203-af79ba44d473", + "09ed0318-2f4a-41b1-abe5-d11da7537c31", + "8918ddfe-2ad4-4cc8-a573-e020ca13f3f5", + "46788d8e-cdf9-4d5c-9125-094eb2e4d44c", + "f80e7298-93f5-46d0-86f2-b8f29cfc88bd", + "1242fb95-7ddd-44ac-8a05-9e8a22a6137d", + "02fe5732-a072-4767-83e3-a91d41d274ca", + "fb78f09f-f998-4061-ad51-d71f90388f0e", + "c2da2a3e-b0d6-46bf-a09a-68562617a50a", + "ba0add52-783c-4ec0-8b9c-7a6b60f99d1c", + "8843adfb-77d3-455a-86f9-de0651555ec6", + "5cc54223-ec0c-4c50-87e9-b9947264e1f4", + "57c63422-d911-4666-815b-0c332e4d7d6a", + "414da90b-16b3-4e88-86f5-3c3945e8fa4b", + "2d01710c-7c77-4cf1-b0d0-5902a25f6e17", + "a5def41e-2e73-433f-92f7-5f1d99fef05d", + "98c87826-dba2-44f4-b123-4c7e3c8a2647", + "62305e79-9d39-4643-b003-5e0b096fe4f4", + "5993c2c9-5d59-403e-b459-946c8b302086", + "30236d07-62d0-4c63-abf7-df46aa45e473", + "27c12970-3efb-4f39-a78a-2fbb7bddc941", + "134838f5-ce7e-4876-ac32-6367b99daf83" + ], + "male": [ + "228fca29-3a0a-435c-8728-5cb483251068", + "5ee9feff-1265-424a-9d7f-8e4d431a12c7", + "5cad89c9-d88a-4832-89fb-55f2f16d13d3", + "41468051-3a85-4b68-92ad-64add250d369", + "c961b81c-a935-4c17-bfb3-ba2239de8c2f", + "a167e0f3-df7e-4d52-a9c3-f949145efdab", + "79f8b5fb-2cc8-479a-80df-29f7a7cf1a3e", + "146485fd-8736-41c7-88a8-7cdd0da34d84", + "565510e8-6b45-45de-8758-13588fbaec73", + "98a34ef2-2140-4c28-9c71-663dc4dd7022", + "1463a4e1-56a1-4b41-b257-728d56e93605", + "ed81fd13-2016-4a49-8fe3-c0d2761695fc", + "34575e71-908f-4ab6-ab54-b08c95d6597d", + "00967b2f-88a6-4a31-8153-110a92134b9f", + "729651dc-c6c3-4ee5-97fa-350da1f88600", + "820a3788-2b37-4d21-847a-b65d8a68c99a", + "a0e99841-438c-4a64-b679-ae501e7d6091", + "c99d36f3-5ffd-4253-803a-535c1bc9c306", + "9fa83ce3-c3a8-4523-accc-173904582ced", + "d46abd1d-2d02-43e8-819f-51fb652c1c61", + "638efaaa-4d0c-442e-b701-3fae16aad012", + "e00d0e4c-a5c8-443f-a8a3-473eb9a62355", + "42b39f37-515f-4eee-8546-73e841679c1d", + "41534e16-2966-4c6b-9670-111411def906", + "1259b7e3-cb8a-43df-9446-30971a46b8b0", + "4df027cb-2920-4a1f-8c34-f21529d5c3fe", + "1fc31370-81b1-4588-9c1a-f93793c6e01d", + "87bc56aa-ab01-4baa-9071-77d497064686", + "f114a467-c40a-4db8-964d-aaba89cd08fa", + "bd9120b6-7761-47a6-a446-77ca49132781", + "701a96e1-7fdd-4a6c-a81e-a4a450403599", + "3e1ed423-17e5-4773-b87c-25b031106e41", + "da4a4eff-3b7e-4846-8f70-f075ff61222c", + "23e9e50a-4ea2-447b-b589-df90dbb848a2", + "ee7ea9f8-c0c1-498c-9279-764d6b56d189", + "97f4b8fb-f2fe-444b-bb9a-c109783a857a", + "4f7f1324-1853-48a6-b294-4e78e8036a83", + "7cf0e2b1-8daf-4fe4-89ad-f6039398f359", + "87748186-23bb-4158-a1eb-332911b0b708", + "13524ffb-a918-499a-ae97-c98c7c4408c4", + "7e19344f-9f17-47d7-a13a-4366ad06ebf3", + "3246e36c-ac8c-418d-83cd-4eaad5a3b887", + "2a4d065a-ac91-4203-a015-eb3fc3ee3365", + "40104aff-a015-4da1-9912-af950fbec99e", + "86e30c1d-714b-4074-a1f2-1cb6b552fb49", + "50d6beb4-80ea-4802-8387-6c948fe84208", + "63ff761f-c1e8-414b-b969-d1833d1c870c", + "ab109683-f31f-40d7-b264-9ec3e26fb85e", + "41f3c367-e0a8-4a85-89e0-c27bae9c9b6d", + "c45bc5ec-dc68-4feb-8829-6e6b2748095d", + "7fe6faca-172f-4fd9-a193-25642b8fdb07", + "ec58877e-44ae-4581-9078-a04225d42bd4", + "3dcaa773-fb1a-47f7-82a4-1bf756c4e1fb", + "726d5ae5-055f-4c3d-8355-d9677de68937", + "96c64eb5-a945-448f-9710-980abe7a514c", + "39b376fc-488e-4d0c-8b37-e00b72059fdd", + "7360f116-6306-4e9a-b487-1235f35a0f21", + "bbee10a8-4f08-4c5c-8282-e69299115055", + "0b32066b-2bcc-44b9-89ab-0223a09d1606", + "bfd3644b-d561-4b1c-a01f-d9af98cb67c0", + "8d110413-2f14-44a2-8203-2104db4340e9", + "5619d38c-cf51-4d8e-9575-48f61a280413", + "34d923aa-c3b5-4f21-aac7-2c1f12730d4b", + "64462aed-aafc-45d4-84cd-ecb4b3763a0a", + "5c43e078-5ba4-4e1f-9639-8d85a403f76a", + "36b42fcb-60c5-4bec-b077-cb1a00a92ec6", + "d7862948-75c3-4c7c-ae28-2959fe166f49", + "6a176356-ada1-4b48-b2ae-3a3fdd485680", + "586b6832-1ca1-43ad-b974-527dc13c2532", + "66f5935b-af2e-4ec9-bb3e-59112e9ddc93", + "236bb1fb-dc41-4a2b-84d6-d22d2a2aaae1", + "ee8b13e7-98af-4b15-89d1-8d402be10c94", + "1cb5b8bc-77c9-4e7c-a251-da02348e2727", + "f24ae0b7-a3d2-4dd1-89df-959bdc4ab179", + "db69127a-dbaf-4fa9-b425-2fe67680c348", + "5fb68a42-0ed7-46fa-8a8f-ad4b332fbf6f", + "b134c304-d095-4d2b-a77a-914f5e8e84e7", + "74f42072-6245-4fe2-b5dc-3dc9b56fdbd0", + "373e661a-f0ef-4e34-a09e-183184a443e6", + "9301949d-b7cd-40d9-a246-5a4430992d6b", + "e39b9fc0-23f5-4616-962a-da99c8ccb1dc", + "01fd7d67-d2a0-4e4e-8c48-42611c71a926", + "df872fcd-da17-4b01-a49f-a80d7aaee95e", + "6cb8801d-259a-4bdc-978f-b45808d58cd3", + "efa653e5-314d-46ca-9f90-70ac7d6ca71e", + "afb19d1b-4044-4f34-a962-f4aef640a002", + "c58bda25-abd5-4c72-97a2-4dbe049b368d", + "f688c0a6-dddd-48ba-8246-c099d494a162", + "a924b0e6-9253-4711-8fc3-5cb8e0188c94", + "6fccb471-26f7-4f7a-93dd-542935db6c20", + "17488b72-f815-44d8-bdd9-869971c3ec06", + "59697755-8cfb-4ccf-9da4-f2201d06b067", + "b58b6b46-1a27-46ba-8648-bc203a5d394e", + "3bf35adc-bcc4-464b-b834-c90c88cf6492", + "9c8880b2-ccf9-4730-b805-cea23df247d7", + "5cf0e4d9-ca2b-4fd5-81fa-89db3b645539", + "c0f43c66-9f21-4034-b485-8f1d3340d759", + "2948c301-9211-4112-8f36-4c3fc836ef12", + "49808e4c-998a-40a8-b2ea-8ac8e8ce779e", + "7a8ae0b6-504a-49af-92d3-4e7e2eb84ca1", + "cd6256ef-2b2a-41f6-a8d8-c1307af5061f", + "3ccc4544-84f7-45e3-ae57-5c52b5a1fac6", + "18f8d87b-0da9-4efa-b504-4580e303f7db", + "fdf6303b-4cfa-4f8e-b7ae-acb398984cf9", + "ea7c252f-6cb1-45f5-8be9-b4f6ac282242", + "2d5b8c3a-116c-4741-acaf-ba4fa289eba2", + "356f4a89-d056-4e2e-8c73-865fa4d3af0a", + "23112795-d54e-4560-9568-791a87c30201", + "1628cfcd-a161-4e47-98ff-46bffa4ab290", + "a892d232-f705-40d7-bc8d-e368b295ec2a", + "3d83e30f-c31b-4f26-b442-7075feafa53a", + "87a983d8-3471-4c4b-9ade-f1d10a4110ac", + "b2222537-1561-4425-8c3c-e1aca96ad853", + "8cbfe3ab-8364-4e72-b606-93f749519c66", + "d2c66146-c1c8-4c3a-9870-38e5a6b72442", + "5319c0b1-3dd1-4c00-b721-bfd2ec88ef56", + "f4a3a8e4-694c-4c45-9ca0-27caf97901b5", + "ed82c17b-4704-4d34-be43-5d19065acdf1", + "bbc5d060-50e1-45a3-87ff-191b8cea3092", + "b9cf5ec3-eaa4-46a5-a5b2-b0d0f22395a2", + "4c2dcd38-5608-45ca-8f11-51c88208d01c", + "90c896fa-aaa1-41af-a612-5267636440a3", + "d709a7e8-9495-4247-aef0-01b3207d11bf", + "1ce291a1-0771-4732-a3f7-8cca29bf055f", + "dbfa416f-d5c3-4006-854b-235ef6bdf4fd", + "6776173b-fd72-460d-89b3-d85812ee518d", + "921034a2-aace-4ef7-87b1-b9bc455c9a15", + "c78dd7ae-6692-4c44-a2a2-834e365afe60", + "0834f3df-e650-4766-a20c-5a93a43aa6e3", + "4cf80313-54dc-4ca9-a17c-3e5b8f68a78c", + "8d7d11ff-d985-48a2-a737-1da0b6fedc8b", + "3f04e815-3260-4f50-8fd9-af9c657be4c2", + "9a0894a9-28f0-436e-9a1d-e92bccbce4dd", + "710feaa3-b550-42f3-b3eb-6f37f2a7cc0a", + "0d42f0f6-c019-4082-b250-1c16133d1c82", + "efd255c7-f030-43d3-b5d8-c7b72063be70", + "7edf9efb-58fc-46ba-a648-3a00a86b111b", + "92c41dd4-04aa-45de-8504-a92b40cb8818", + "2f22b9bc-b0eb-4cb6-b5ae-0c099a0fdfad", + "79bfcec0-720c-41f2-a33a-f12383e9627f", + "e2d48e7b-cd73-4c4c-bc1e-f232580e8709", + "39c3388d-6b3f-4cec-88d7-900bd0899e00", + "c63361f8-d142-4c62-8da7-8f8149d973d6", + "9287676d-f0cc-423f-ac03-3b3c7242f091", + "da69d796-4603-4419-8a95-293bfc5679eb", + "f96dc0b1-7900-4894-a339-81fb46d515a7", + "c1c65fc2-528a-4dde-a2c4-f822785c2704", + "b1ce5126-4d08-42c3-adef-d3eb39e90c7a", + "adde00e9-c98f-42ae-a94d-fc9f92f11c76", + "9fb269e7-70fe-4cbe-aa3f-28bdb67e3e84", + "80713a53-e484-4f69-9852-7891096016ac", + "7c8ba972-4960-4c43-bea0-8178e2205696", + "6fd4f468-0345-4f41-81d0-3f48ebc295e0", + "fd098a10-ba9e-445e-b144-be2a9f3dac02", + "c4e848dc-d4fd-4bc8-90ea-8525563ec0e5", + "b08c966e-2146-4592-99eb-3171a714a43c", + "a3a4fe2a-d402-41d1-be7d-28f71eda755f", + "9d2b4a7f-7ced-4fb8-b570-9ce21fb931c8", + "6b622a1d-906f-44af-b60c-7bef365bf124", + "10d17ae0-8f64-472a-be00-f00a98c729e0", + "8e14933d-ecd7-402b-9505-795130d69b35", + "7b2c0a2e-3dd3-4a44-b16b-26ecd8134279", + "79b8126f-c5d9-4a73-8585-ba5e1a077ed6", + "725d43d6-1196-480e-bd87-728ae5eff9e1", + "63426c82-a0c9-4f23-a175-50eb64c95ec1", + "61001bc6-9064-40a4-b8b2-29178e0fa558", + "5c7b66c2-3b58-464d-8a12-093410a269c5", + "3d79b1fd-daaa-439c-bff3-903dc18e7684", + "cf14fdcd-24a0-4d63-958a-c784f33d8e7c", + "cb605424-d682-48e9-94db-34cc567cf1c6", + "abe7dee1-6051-43d3-9a9f-1ac1312497a7", + "aa086107-101b-4182-a628-c51186d74166", + "911b8b22-887f-4caf-bf87-85d834c08708", + "876c39e1-9ecd-42cd-b0c1-8b3906f0be19", + "83e45f18-fac4-40db-a43b-03257883b437", + "64875a07-f57e-4a70-b702-4e3fb25efeda" + ] + }, + "es": { + "female": [ + "5c5ad5e7-1020-476b-8b91-fdcbe9cc313c", + "cefcb124-080b-4655-b31f-932f3ee743de", + "c0c374aa-09be-42d9-9828-4d2d7df86962", + "d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", + "e9f0368b-3662-4a01-b037-e13ca5203c74", + "727f663b-0e90-4031-90f2-558b7334425b" + ], + "male": [ + "15d0c2e2-8d29-44c3-be23-d585d5f154a1", + "79743797-2087-422f-8dc7-86f9efca85f1", + "2695b6b5-5543-4be1-96d9-3967fb5e7fec", + "b5aa8098-49ef-475d-89b0-c9262ecf33fd", + "846fa30b-6e1a-49b9-b7df-6be47092a09a", + "b042270c-d46f-4d4f-8fb0-7dd7c5fe5615", + "5ef98b2a-68d2-4a35-ac52-632a2d288ea6" + ] + }, + "hi": { + "female": [ + "faf0731e-dfb9-4cfc-8119-259a79b27e12", + "95d51f79-c397-46f9-b49a-23763d3eaa2d", + "28ca2041-5dda-42df-8123-f58ea9c3da00", + "9cebb910-d4b7-4a4a-85a4-12c79137724c", + "bec003e2-3cb3-429c-8468-206a393c67ad", + "f91ab3e6-5071-4e15-b016-cde6f2bcd222", + "209d9a43-03eb-40d8-a7b7-51a6d54c052f", + "56e35e2d-6eb6-4226-ab8b-9776515a7094" + ], + "male": [ + "fd2ada67-c2d9-4afe-b474-6386b87d8fc3", + "be79f378-47fe-4f9c-b92b-f02cefa62ccf", + "9b953e7b-86a8-42f0-b625-1434fb15392b", + "bdab08ad-4137-4548-b9db-6142854c7525", + "7f423809-0011-4658-ba48-a411f5e516ba", + "393dd459-f8d8-4c3e-a86b-ec43a1113d0b", + "791d5162-d5eb-40f0-8189-f19db44611d8" + ] + }, + "de": { + "female": [ + "b9de4a89-2257-424b-94c2-db18ba68c81a", + "4ab1ff51-476d-42bb-8019-4d315f7c0c05", + "38aabb6a-f52b-4fb0-a3d1-988518f4dc06", + "3f4ade23-6eb4-4279-ab05-6a144947c4d5", + "11c61307-4f9e-4db8-ac3b-bfa5f2a731ce", + "1ade29fc-6b82-4607-9e70-361720139b12", + "6d4b1416-8d54-4d94-a788-8a802c086544" + ], + "male": [ + "384b625b-da5d-49e8-a76d-a2855d4f31eb", + "e00dd3df-19e7-4cd4-827a-7ff6687b6954", + "afa425cf-5489-4a09-8a3f-d3cb1f82150d", + "db229dfe-f5de-4be4-91fd-7b077c158578", + "b7187e84-fe22-4344-ba4a-bc013fcb533e", + "2be00b67-d53f-4eb5-89e7-96c224d56fbc" + ] + }, + "fr": { + "female": [ + "a8a1eb38-5f15-4c1d-8722-7ac0f329727d", + "65b25c5d-ff07-4687-a04c-da2f43ef6fa9", + "8832a0b5-47b2-4751-bb22-6a8e2149303d", + "6c64b57a-bc65-48e4-bff4-12dbe85606cd" + ], + "male": [ + "0418348a-0ca2-4e90-9986-800fb8b3bbc0", + "5c3c89e5-535f-43ef-b14d-f8ffe148c1f0", + "ab7c61f5-3daa-47dd-a23b-4ac0aac5f5c3", + "56df0456-8f47-4f7a-ac26-40c2f9797104" + ] + }, + "it": { + "female": [ + "d718e944-b313-4998-b011-d1cc078d4ef3", + "d609f27f-f1a4-410f-85bb-10037b4fba99", + "0e21713a-5e9a-428a-bed4-90d410b87f13", + "36d94908-c5b9-4014-b521-e69aee5bead0" + ], + "male": [ + "e5923af7-a329-4e9b-b95a-5ace4a083535", + "408daed0-c597-4c27-aae8-fa0497d644bf", + "e019ed7e-6079-4467-bc7f-b599a5dccf6f", + "79693aee-1207-4771-a01e-20c393c89e6f", + "029c3c7a-b6d9-44f0-814b-200d849830ff", + "88b329db-85d7-47cc-a5c5-98225a756721" + ] + }, + "pl": { + "male": [ + "82a7fc13-2927-4e42-9b8a-bb1f9e506521", + "4ef93bb3-682a-46e6-b881-8e157b6b4388", + "3d335974-4c4a-400a-84dc-ebf4b73aada6", + "2a3503b2-b6b6-4534-a224-e8c0679cec4a", + "887149a8-4616-42ad-b2ce-c3819176f45d" + ], + "female": [ + "dcf62f33-7cff-4f20-85b2-2efaa68cbc32", + "575a5d29-1fdc-4d4e-9afa-5a9a71759864", + "ea7b5eee-39d9-40b0-b241-1910cbca9c62" + ] + }, + "ru": { + "female": [ + "064b17af-d36b-4bfb-b003-be07dba1b649", + "642014de-c0e3-4133-adc0-36b5309c23e6", + "779673f3-895f-4935-b6b5-b031dc78b319", + "9ed9f7e7-3ef6-4773-9dd3-ffcb479ca1f0" + ], + "male": [ + "888b7df4-e165-4852-bfec-0ab2b96aaa46" + ] + }, + "pt": { + "female": [ + "1cf751f6-8749-43ab-98bd-230dd633abdb", + "700d1ee3-a641-4018-ba6e-899dcadc9e2b", + "d4b44b9a-82bc-4b65-b456-763fce4c52f9", + "f39bf583-3b3d-402f-9ffb-6179d9ec3e35" + ], + "male": [ + "5063f45b-d9e0-4095-b056-8f3ee055d411", + "a37639f0-2f0a-4de4-9942-875a187af878", + "6a16c1f4-462b-44de-998d-ccdaa4125a0a", + "6a360542-a117-4ed5-9e09-e8bf9b05eabb", + "fbee0e7d-a83a-4082-bad1-13c70f86da4e" + ] + }, + "ja": { + "female": [ + "0cd0cde2-3b93-42b5-bcb9-f214a591aa29", + "c7eafe22-8b71-40cd-850b-c5a3bbd8f8d2", + "59d4fd2f-f5eb-4410-8105-58db7661144f", + "2b568345-1d48-4047-b25f-7baccf842eb0", + "44863732-e415-4084-8ba1-deabe34ce3d2", + "31c55968-a9f4-4115-8831-3a16952179c8" + ], + "male": [ + "e8a863c6-22c7-4671-86ca-91cacffc038d", + "6b92f628-be90-497c-8f4c-3b035002df71", + "06950fa3-534d-46b3-93bb-f852770ea0b5", + "446f922f-c43a-4aad-9a8b-ad2af568e882", + "9e7ef2cf-b69c-46ac-9e35-bbfd73ba82af", + "a759ecc5-ac21-487e-88c7-288bdfe76999", + "97e7d7a9-dfaa-4758-a936-f5f844ac34cc", + "b8e1169c-f16a-4064-a6e0-95054169e553" + ] + }, + "ko": { + "female": [ + "304fdbd8-65e6-40d6-ab78-f9d18b9efdf9", + "29e5f8b4-b953-4160-848f-40fae182235b", + "663afeec-d082-4ab5-827e-2e41bf73a25b", + "15628352-2ede-4f1b-89e6-ceda0c983fbc" + ], + "male": [ + "af6beeea-d732-40b6-8292-73af0035b740" + ] + }, + "zh": { + "female": [ + "7a5d4663-88ae-47b7-808e-8f9b9ee4127b", + "bf32f849-7bc9-4b91-8c62-954588efcc30", + "e90c6678-f0d3-4767-9883-5d0ecf5894a8", + "f9a4b3a6-b44b-469f-90e3-c8e19bd30e99", + "0b904166-a29f-4d2e-bb20-41ca302f98e9", + "a53c3509-ec3f-425c-a223-977f5f7424dd" + ], + "male": [ + "eda5bbff-1ff1-4886-8ef1-4e69a77640a0", + "c59c247b-6aa9-4ab6-91f9-9eabea7dc69e", + "653b9445-ae0c-4312-a3ce-375504cff31e", + "16212f18-4955-4be9-a6cd-2196ce2c11d1" + ] + }, + "tr": { + "female": [ + "fa7bfcdc-603c-4bf1-a600-a371400d2f8c", + "bb2347fe-69e9-4810-873f-ffd759fe8420", + "0f95596c-09c4-4418-99fe-5c107e0713c0" + ], + "male": [ + "39f753ef-b0eb-41cd-aa53-2f3c284f948f", + "c1cfee3d-532d-47f8-8dd2-8e5b2b66bf1d", + "5a31e4fb-f823-4359-aa91-82c0ae9a991c", + "91e91d74-8eb4-43cd-97d3-7466c21db00d" + ] + }, + "sv": { + "female": [ + "f852eb8d-a177-48cd-bf63-7e4dcab61a36", + "6c6b05bf-ae5f-4013-82ab-7348e99ffdb2", + "00510a15-4216-4fdc-a0ab-05d74cd9f795" + ], + "male": [ + "38a146c3-69d7-40ad-aada-76d5a2621758", + "0caedb75-417f-4e36-9b64-c21354cb94c8", + "32a806e8-894e-41ad-a4d5-6d9154d7b1e6" + ] + }, + "nl": { + "male": [ + "9e8db62d-056f-47f3-b3b6-1b05767f9176", + "4aa74047-d005-4463-ba2e-a0d9b261fb87", + "af482421-80f4-4379-b00c-a118def29cde", + "4b250449-c635-4b63-bd1d-b654b12ffcd4" + ], + "female": [ + "0eb213fe-4658-45bc-9442-33a48b24b133", + "ac317dac-1b8f-434f-b198-a490e2a4914d" + ] + }, + "no": { + "male": [ + "d6dca1b6-cdd8-4e9c-823c-e03979261740" + ] + }, + "te": { + "male": [ + "38bded0a-3ab4-42d1-8e47-2e0b6b10ced9" + ], + "female": [ + "07bc462a-c644-49f1-baf7-82d5599131be" + ] + }, + "kn": { + "female": [ + "7c6219d2-e8d2-462c-89d8-7ecba7c75d65" + ], + "male": [ + "6baae46d-1226-45b5-a976-c7f9b797aae2" + ] + }, + "fi": { + "male": [ + "ae1a833b-0d95-4b7f-8d05-d6418c6f8049" + ], + "female": [ + "65c34eec-42c9-4a75-a8bd-b676fb847b72" + ] + }, + "mr": { + "male": [ + "f227bc18-3704-47fe-b759-8c78a450fdfa" + ], + "female": [ + "5c32dce6-936a-4892-b131-bafe474afe5f" + ] + }, + "da": { + "female": [ + "c323c793-41f9-47b8-99dc-9b44b0440b84" + ] + }, + "bn": { + "female": [ + "59ba7dee-8f9a-432f-a6c0-ffb33666b654" + ], + "male": [ + "2ba861ea-7cdc-43d1-8608-4045b5a41de5" + ] + }, + "sk": { + "male": [ + "ca590fdc-df56-4d2e-94a4-ef5b423c7ddf" + ], + "female": [ + "abf68668-6549-462c-8426-1fa7b466b91d" + ] + }, + "uk": { + "male": [ + "05ffab9c-d380-4909-8375-cd12f59238c3" + ] + }, + "el": { + "female": [ + "50849023-76e9-46c7-af52-9ec39888a165" + ], + "male": [ + "b45eba5b-2215-4da7-9c7c-121c95ed7b81" + ] + }, + "ta": { + "male": [ + "d2870b91-1b4c-47ab-81a8-3718d8e9c222" + ], + "female": [ + "7f98e662-142d-41ba-89a2-12452640ce6d" + ] + }, + "vi": { + "male": [ + "0e58d60a-2f1a-4252-81bd-3db6af45fb41" + ], + "female": [ + "b8cd71e3-bc14-4538-a530-d6314731c036" + ] + }, + "id": { + "male": [ + "a053f6bc-7df4-40de-96d4-de026bc47ce8" + ], + "female": [ + "b441c4fd-4910-4c55-ae56-f0291057e2cc" + ] + }, + "ro": { + "female": [ + "34acfaee-c556-41ee-a5f6-c687fb20357c" + ], + "male": [ + "3f64ef99-d87b-4b51-b217-df7351f7886a" + ] + }, + "ka": { + "male": [ + "dbebd077-80cb-4bcf-b43b-4552f96341bb" + ], + "female": [ + "0bfbea6c-2f8f-4f86-b411-aa2316561e36" + ] + }, + "ml": { + "female": [ + "b426013c-002b-4e89-8874-8cd20b68373a" + ] + }, + "ms": { + "male": [ + "8281db18-6ac5-47bb-91a8-ce23a1f1d951" + ], + "female": [ + "83604597-55fa-4ccc-8357-730b313f353f" + ] + }, + "he": { + "male": [ + "3e32f3c5-9ac0-4192-9994-87fdb277120f" + ] + }, + "bg": { + "female": [ + "fcbecbcc-0cef-4615-8b5a-712fe1b39dd0" + ], + "male": [ + "d132064c-b931-4a80-bf0d-02a331ec4572" + ] + }, + "th": { + "male": [ + "5de076e9-7b28-4442-b279-e7d80d573505" + ], + "female": [ + "ccc7bb22-dcd0-42e4-822e-0731b950972f" + ] + }, + "hu": { + "female": [ + "e97c3b37-1aa5-46af-afb7-9545086aaa92" + ], + "male": [ + "36e0c00b-1bfd-4ad7-a0e8-928d4cadca00" + ] + }, + "pa": { + "female": [ + "991c62ce-631f-48b0-8060-2a0ebecbd15b" + ], + "male": [ + "8bacd442-a107-4ec1-b6f1-2fcb3f6f4d56" + ] + }, + "cs": { + "female": [ + "bdc4a3ce-2e22-4398-8cd6-76b7160d2298" + ], + "male": [ + "89266bab-6e15-455d-8654-e18c440b0656" + ] + }, + "tl": { + "male": [ + "c4cbcb7d-d9fa-4eac-b547-46831718ef58" + ], + "female": [ + "9261664a-c3d0-4200-9038-5466bcf3a09c" + ] + }, + "ar": { + "female": [ + "6304c635-6681-4f9e-85b6-a97f4d26461a" + ], + "male": [ + "e3087ad8-7018-4154-9a87-11577f916cd4" + ] + }, + "gu": { + "female": [ + "4590a461-bc68-4a50-8d14-ac04f5923d22" + ], + "male": [ + "91925fe5-42ee-4ebe-96c1-c84b12a85a32" + ] + }, + "hr": { + "male": [ + "a1a16724-b1f3-4b27-9e47-8a175115e93c" + ], + "female": [ + "2a2624ad-bd06-4563-81fd-0519742e25d2" + ] + } +} diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 7bcacbcb..8779a15e 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -11,6 +11,7 @@ import asyncio import json import os +from functools import lru_cache from pathlib import Path from fi import simulate @@ -75,14 +76,126 @@ def _persona_stt_language() -> str: return "en" +# Cartesia voice selection: a persona's accent (or, failing that, language) chooses a catalog +# language bucket, and gender chooses within it, so a caller sounds like the accent the scenario +# wrote across dozens of languages rather than the handful of English voices Deepgram aura ships. +# Accent wins over language; both accept ISO codes and demonyms. Runs only when a Cartesia key is +# present; otherwise the Deepgram aura path below is used unchanged. +_CARTESIA_SUPPORTED_LANGS = frozenset( + { + "en", "es", "hi", "de", "fr", "it", "pl", "ru", "pt", "ja", "ko", "zh", "tr", "sv", + "nl", "no", "te", "kn", "fi", "mr", "da", "bn", "sk", "uk", "el", "ta", "vi", "id", + "ro", "ka", "ml", "ms", "he", "bg", "th", "hu", "pa", "cs", "tl", "ar", "gu", "hr", + } +) +_CARTESIA_ACCENT_TO_LANG: dict[str, str] = { + "spanish": "es", "south american": "es", "indian": "hi", "german": "de", "french": "fr", + "italian": "it", "polish": "pl", "russian": "ru", "portuguese": "pt", "brazilian": "pt", + "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", "turkish": "tr", + "swedish": "sv", "dutch": "nl", "norwegian": "no", "finnish": "fi", "danish": "da", + "slovak": "sk", "ukrainian": "uk", "greek": "el", "romanian": "ro", "georgian": "ka", + "bulgarian": "bg", "thai": "th", "hungarian": "hu", "czech": "cs", "croatian": "hr", + "vietnamese": "vi", "indonesian": "id", "malay": "ms", "malaysian": "ms", "tagalog": "tl", + "filipino": "tl", "arabic": "ar", "hebrew": "he", "israeli": "he", "telugu": "te", + "kannada": "kn", "marathi": "mr", "bengali": "bn", "tamil": "ta", "malayalam": "ml", + "punjabi": "pa", "gujarati": "gu", +} +_CARTESIA_LANGUAGE_TO_LANG: dict[str, str] = { + "english": "en", "hinglish": "hi", "spanish": "es", "hindi": "hi", "german": "de", + "french": "fr", "italian": "it", "polish": "pl", "russian": "ru", "portuguese": "pt", + "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", "turkish": "tr", + "swedish": "sv", "dutch": "nl", "norwegian": "no", "telugu": "te", "kannada": "kn", + "finnish": "fi", "marathi": "mr", "danish": "da", "bengali": "bn", "slovak": "sk", + "ukrainian": "uk", "greek": "el", "tamil": "ta", "vietnamese": "vi", "indonesian": "id", + "romanian": "ro", "georgian": "ka", "malayalam": "ml", "malay": "ms", "hebrew": "he", + "bulgarian": "bg", "thai": "th", "hungarian": "hu", "punjabi": "pa", "czech": "cs", + "tagalog": "tl", "filipino": "tl", "arabic": "ar", "gujarati": "gu", "croatian": "hr", +} +_CARTESIA_DEFAULT_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" + + +def _norm(value) -> str: + return str(value or "").strip().lower().replace("-", " ") + + +@lru_cache(maxsize=1) +def _cartesia_catalog() -> dict: + path = Path(__file__).parent / "data" / "voices_by_language_and_gender.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def _persona_language_name(persona: dict) -> str: + languages = persona.get("languages") + if isinstance(languages, list) and languages: + return _norm(languages[0]) + return _norm(persona.get("language")) + + +def _cartesia_lang_key(persona: dict) -> str: + """The catalog language bucket for a persona: accent wins, then language, else English.""" + accent = _norm(persona.get("accent")) + key = _CARTESIA_ACCENT_TO_LANG.get(accent) + if key in _CARTESIA_SUPPORTED_LANGS: + return key + language = _persona_language_name(persona) + key = _CARTESIA_LANGUAGE_TO_LANG.get(language) + if key in _CARTESIA_SUPPORTED_LANGS: + return key + if language in _CARTESIA_SUPPORTED_LANGS: + return language + return "en" + + +def _cartesia_voice_for(persona: dict) -> str: + """A stable Cartesia voice id for one caller, chosen by accent/language and gender. + + Deterministic by persona name so a caller keeps its voice across runs while a suite still + spreads voices. Falls back across gender and to English when a long-tail language lacks one. + """ + gender = _norm(persona.get("gender")) + if gender not in ("male", "female"): + gender = "female" + catalog = _cartesia_catalog() + key = _cartesia_lang_key(persona) + other = "male" if gender == "female" else "female" + voices = ( + (catalog.get(key) or {}).get(gender) + or (catalog.get(key) or {}).get(other) + or (catalog.get("en") or {}).get(gender) + or [] + ) + if not voices: + return _CARTESIA_DEFAULT_VOICE + index = sum(ord(character) for character in str(persona.get("name") or "")) % len(voices) + return voices[index] + + +def _voice_providers() -> tuple[str, str]: + """The (stt, tts) providers for the caller. An explicit env override wins; otherwise Cartesia + when its key is present (richer, multi-language voices), else Deepgram aura.""" + default = "cartesia" if os.environ.get("CARTESIA_API_KEY", "").strip() else "deepgram" + stt = os.environ.get("SIMULATOR_STT_PROVIDER", "").strip() or default + tts = os.environ.get("SIMULATOR_TTS_PROVIDER", "").strip() or default + return stt, tts + + def _simulator() -> simulate.SimulatorAgentDefinition: llm_provider = os.environ.get("SIMULATOR_LLM_PROVIDER", "google") - stt_provider = os.environ.get("SIMULATOR_STT_PROVIDER", "deepgram") - tts_provider = os.environ.get("SIMULATOR_TTS_PROVIDER", "deepgram") + stt_provider, tts_provider = _voice_providers() + default_tts_voice = ( + _CARTESIA_DEFAULT_VOICE if tts_provider == "cartesia" else "aura-asteria-en" + ) defaults = { "llm": {"google": "gemini-2.5-flash-lite", "openai": "gpt-4o-mini"}, - "stt": {"deepgram": "nova-2", "google": "chirp_2"}, - "tts": {"deepgram": "aura-asteria-en", "google": "en-US-Chirp3-HD-Aoede"}, + "stt": {"deepgram": "nova-2", "cartesia": "ink-2", "google": "chirp_2"}, + "tts": { + "deepgram": "aura-asteria-en", + "cartesia": "sonic-3", + "google": "en-US-Chirp3-HD-Aoede", + }, } def model(kind: str, provider: str) -> str: @@ -106,7 +219,7 @@ def model(kind: str, provider: str) -> str: tts={ "provider": tts_provider, "model": model("tts", tts_provider), - "voice": os.environ.get("SIMULATOR_TTS_VOICE", "aura-asteria-en"), + "voice": os.environ.get("SIMULATOR_TTS_VOICE", default_tts_voice), }, instructions=( "Act as the customer described by the scenario. Speak naturally and briefly. " @@ -164,14 +277,15 @@ def _scenario() -> simulate.Scenario: persona = _json_env("HARNESS_PERSONA", {"name": "customer"}) persona = dict(persona) if isinstance(persona, dict) else {"name": "customer"} persona["role"] = "customer" - # Give the caller a voice from its accent when the suite runs on aura and none was set, so - # different callers sound different and match the accent the scenario wrote. - if ( - not persona.get("voice") - and not persona.get("voice_id") - and os.environ.get("SIMULATOR_TTS_PROVIDER", "deepgram").lower() == "deepgram" - ): - persona["voice"] = _aura_voice_for(persona) + # Give the caller a voice from its accent/language when none was set, so different callers + # sound different and match what the scenario wrote. Cartesia draws from the multi-language + # catalog; Deepgram falls back to the aura voices it ships. + if not persona.get("voice") and not persona.get("voice_id"): + tts_provider = _voice_providers()[1].lower() + if tts_provider == "cartesia": + persona["voice"] = _cartesia_voice_for(persona) + elif tts_provider == "deepgram": + persona["voice"] = _aura_voice_for(persona) metadata = dict(persona.get("metadata") or {}) if isinstance(fixture, dict) and fixture.get("phone"): # LiveKit exposes this as participant metadata/attributes. A target can From 56cfb268f41ed5b38ae78ca0c60f24667bcf1bb4 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 12:10:46 +0530 Subject: [PATCH 09/41] fix(harness): define the review server, stop parallel writers deleting each other, and gate host credentials --- harness-ui/server.py | 14 --- src/fi/alk/harness/progress.py | 113 ------------------ src/fi/alk/harness/provision.py | 19 +-- src/fi/alk/harness/run/simulation.py | 9 +- src/fi/alk/harness/scenario.py | 16 +-- src/fi/alk/harness/scenario_tools.py | 23 +++- src/fi/alk/harness/scenarios.py | 47 +++++--- src/fi/alk/harness/secrets.py | 5 + src/fi/simulate/simulation/engines/livekit.py | 24 ++++ 9 files changed, 102 insertions(+), 168 deletions(-) delete mode 100644 src/fi/alk/harness/progress.py diff --git a/harness-ui/server.py b/harness-ui/server.py index f084f1e9..153fbe66 100644 --- a/harness-ui/server.py +++ b/harness-ui/server.py @@ -460,20 +460,6 @@ async def world(): db.close() -@app.get("/api/generation") -async def generation(session: str = ""): - """What the suite generation is doing right now, so the page can draw it while it runs. - - Polled rather than streamed: the page may be opened halfway through a suite, refreshed, or - opened somewhere else entirely, and each of those has to show the same thing. Empty when - nothing has been generated here, which the page reads as "no fan-out to show". - """ - from fi.alk.harness import progress - - out = _folder(session) - return progress.read(out) if out else {} - - @app.get("/api/scenarios") async def scenarios(): """Every scenario, with its files and its three gates re-run. diff --git a/src/fi/alk/harness/progress.py b/src/fi/alk/harness/progress.py deleted file mode 100644 index eb765917..00000000 --- a/src/fi/alk/harness/progress.py +++ /dev/null @@ -1,113 +0,0 @@ -"""What the fan-out is doing right now, written where a UI can read it. - -Generating a suite in parallel is the one thing this harness does where nothing appears for -several minutes and then everything appears at once. Told nothing, a person cannot tell a -working run from a hung one, and the honest answer to "is it stuck" is the only thing they want. - -So the fan-out writes its own state as it goes: which use cases it split the work into, which -are running, how many scenarios each has proved, and which have finished. A file rather than a -stream, because the reader is a page that may be opened halfway through, refreshed, or opened on -another machine, and each of those has to show the same thing. -""" - -from __future__ import annotations - -import json -import os -import tempfile -from pathlib import Path -from typing import Any - -PROGRESS = "generation.json" - -WAITING = "waiting" -RUNNING = "running" -DONE = "done" -FAILED = "failed" - - -def _path(destination: Path) -> Path: - return Path(destination) / PROGRESS - - -def _write(destination: Path, state: dict[str, Any]) -> None: - """Replace the file atomically. - - A reader polling this will otherwise catch a half-written file and show nothing, which looks - exactly like the failure it is meant to rule out. - """ - path = _path(destination) - path.parent.mkdir(parents=True, exist_ok=True) - handle, temporary = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp") - try: - with os.fdopen(handle, "w", encoding="utf-8") as writing: - json.dump(state, writing, indent=2) - os.replace(temporary, path) - except BaseException: - Path(temporary).unlink(missing_ok=True) - raise - - -def read(destination: Path) -> dict[str, Any]: - """The current state, or nothing if no suite has been generated here.""" - path = _path(destination) - if not path.exists(): - return {} - try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - - -def planned( - destination: Path, allocation: list[tuple[str, int]], *, at_once: int, asked: int -) -> None: - """The split, before any of it starts. Written first so the tree appears immediately.""" - _write( - destination, - { - "state": RUNNING, - "asked": asked, - "at_once": at_once, - "kept": 0, - "slices": [ - {"use_case": case, "wanted": count, "kept": 0, "state": WAITING} - for case, count in allocation - ], - }, - ) - - -def _change(destination: Path, use_case: str, **fields: Any) -> None: - state = read(destination) - for slice_ in state.get("slices", []): - if slice_.get("use_case") == use_case: - slice_.update(fields) - break - state["kept"] = sum(one.get("kept", 0) for one in state.get("slices", [])) - _write(destination, state) - - -def started(destination: Path, use_case: str) -> None: - _change(destination, use_case, state=RUNNING) - - -def kept(destination: Path, use_case: str, count: int) -> None: - """How many this slice has proved so far. Called as they land, not at the end.""" - _change(destination, use_case, kept=count) - - -def finished(destination: Path, use_case: str, count: int) -> None: - _change(destination, use_case, state=DONE, kept=count) - - -def failed(destination: Path, use_case: str, why: str) -> None: - _change(destination, use_case, state=FAILED, why=why[:300]) - - -def settled(destination: Path, *, kept_total: int) -> None: - """The whole fan-out is over and the suite is written.""" - state = read(destination) - state["state"] = DONE - state["kept"] = kept_total - _write(destination, state) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index f05810f2..f4c8429c 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -2105,14 +2105,17 @@ def _runtime_credential_mounts( # ALK-owned destination that cannot collide with submitted mounts. target = f"/run/harness-secrets/{runtime.name}" if source is None or not _valid_google_credentials(source): - # Local development supplies the credential through the sandbox's own environment - # rather than an uploaded runtime configuration; fall back to it, at an ALK-owned - # destination so a placeholder Compose mount cannot shadow it. - env_value = os.environ.get(name, "").strip() - env_path = Path(env_value).expanduser() if env_value else None - if env_path is not None and _valid_google_credentials(env_path): - source = env_path - target = f"/run/harness-secrets/{env_path.name}" + # A local sandbox supplies the credential through its own environment rather than + # an uploaded runtime configuration. That credential belongs to whoever is running + # the harness, not to the submitted agent, so it is only ever handed over when the + # operator says so: without this opt-in a hosted runner would mount its own + # platform key into a container it does not trust. + if os.environ.get("ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "").strip() == "1": + env_value = os.environ.get(name, "").strip() + env_path = Path(env_value).expanduser() if env_value else None + if env_path is not None and _valid_google_credentials(env_path): + source = env_path + target = f"/run/harness-secrets/{env_path.name}" if source is None or not _valid_google_credentials(source): raise ProvisionError( "the submitted runtime needs GOOGLE_APPLICATION_CREDENTIALS, but neither its " diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index b6ed1792..0fa29f69 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -589,11 +589,14 @@ def placed_once() -> tuple[int, dict[str, Any]]: # A scenario that asks to be heard through background noise selects a clip for the # caller's environment; the voice engine mixes it under the caller. Cleared otherwise so # a previous call's noise never leaks into a quiet one. - if getattr(scenario, "background_noise", False): + noisy = getattr(scenario, "background_noise", False) + if noisy: from ..background_noise import source_for - environment = "" - if isinstance(scenario.fixture, dict): + # The scenario names the place when it cares which one; otherwise the fixture + # says where the caller is, and failing that any noise will do. + environment = noisy if isinstance(noisy, str) else "" + if not environment and isinstance(scenario.fixture, dict): environment = str( scenario.fixture.get("environment") or scenario.fixture.get("location") diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 1c62c5f1..7eb6e782 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -200,14 +200,14 @@ class Scenario(BaseModel): max_turns: int = 10 - # Whether this call happens somewhere noisy. Recorded per scenario rather than per run, so a - # suite covers both conditions and the same scenario stays comparable to itself across runs. - # Chosen at random when the writer does not say, because a suite where every call is quiet - # tests an agent nobody has: real callers phone from cars, kitchens and streets. - # - # Nothing consumes this yet. It is carried so the scenarios written from today are already - # answerable when the caller learns to add noise, rather than needing to be rewritten then. - background_noise: bool = Field(default_factory=lambda: random.choice((True, False))) + # Where this call is being made from, so the agent is heard through it. ``True`` asks for + # noise and leaves the place to the fixture; a string names it outright ("street", "vehicle", + # "retail"), which is what lets one scenario be a call from a car and another from an office. + # Recorded per scenario rather than per run, so a suite covers both conditions and the same + # scenario stays comparable to itself across runs. Chosen at random when the writer does not + # say, because a suite where every call is quiet tests an agent nobody has: real callers + # phone from cars, kitchens and streets. + background_noise: bool | str = Field(default_factory=lambda: random.choice((True, False))) def slots(self) -> dict[str, str]: """Every value this scenario offers the simulator prompt.""" diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index fe373417..2e83355f 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -121,8 +121,14 @@ def accept_scenario( kept: list[Scenario], simulator_prompt: str = "", hard_constraints: list[str] | None = None, + persist: bool = True, ) -> dict[str, Any]: - """Validate one scenario, then prove it. A plain function so both halves are testable.""" + """Validate one scenario, then prove it. A plain function so both halves are testable. + + ``persist`` is off for a writer that shares the destination with siblings: writing the suite + removes every folder not in the writer's own list, so persisting here would delete whatever + the others have proved. Those writers keep their work in ``kept`` and the caller saves once. + """ try: scenario = Scenario.model_validate(payload) except Exception as invalid: @@ -164,7 +170,8 @@ def accept_scenario( # A proved scenario is already valuable work. Persist it immediately so a stopped model, # browser refresh, process restart, or later scenario failure cannot make the UI say none # were written. ``save_scenarios`` remains the suite-level diversity/finality gate. - write_scenarios(kept, world_root, catalogue) + if persist: + write_scenarios(kept, world_root, catalogue) return _ok( f"{scenario.name} {'replaced' if replaced else 'kept'}. All three gates pass: the world " "is ready for it, the reference solution passes its checks, and those checks fail when " @@ -249,7 +256,10 @@ def scenario_tools( target = {"count": wanted} exploration = {"since_submit": 0} - scenario_required = ["name", "instruction", "solution", "sub_goals"] + # ``branch`` is required because coverage is counted on the use case and branch pair, and the + # merge drops a repeat of that pair. A writer that leaves it out gives every scenario in its + # slice the same pair, and all but the first are silently thrown away. + scenario_required = ["name", "branch", "instruction", "solution", "sub_goals"] if contract.conversational: scenario_required.append("persona") @@ -430,6 +440,12 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "string", "description": "One line: what this scenario is trying to find out.", }, + "background_noise": { + "type": "string", + "description": "Where the caller is phoning from, when it is part of the " + "test: street, transit, vehicle, outdoors, retail, office or home. Leave it " + "out unless the place matters.", + }, "instruction": { "type": "string", "description": "The task, written to the person the agent is serving. For a " @@ -548,6 +564,7 @@ async def submit_scenario(args: dict[str, Any]) -> dict[str, Any]: kept=kept, simulator_prompt=simulator_prompt, hard_constraints=contract.hard_constraints, + persist=can_save, ) if not result.get("is_error"): exploration["since_submit"] = 0 diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 85c23930..1f9cc3f2 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio +import logging import os from dataclasses import dataclass from collections.abc import Callable @@ -29,7 +30,6 @@ provider_env, thinking_config, ) -from . import progress from .catalogue import load_catalogue from .contract import AgentContract from .scenario import Scenario @@ -44,8 +44,14 @@ from .session import Stage from .tools import qualified, schema +logger = logging.getLogger(__name__) + SKILL = "write-scenarios" +# The review pass runs its own tool server, kept apart from the writers' one so a reviewer can +# only report gaps and never submit or save a scenario itself. +REVIEW_SERVER = "suite-review" + # Turns a scenario costs in practice: look at the world, rehearse the calls, submit, and often # one more to correct what a gate refused. @@ -365,12 +371,16 @@ async def _write_slice( can_save=False, start_from=[], ) - progress.started(destination, mine.named()) + logger.info("slice starting: %s (wants %s)", mine.named(), mine.count) + seen = 0 def watch(event: Any) -> None: # Report as they land rather than at the end. A slice that proves its first scenario # four minutes in is the difference between a run that looks alive and one that does not. - progress.kept(destination, mine.named(), len(kept)) + nonlocal seen + if len(kept) != seen: + seen = len(kept) + logger.info("slice %s proved %s of %s", mine.named(), seen, mine.count) if on_event: on_event(event) @@ -404,11 +414,11 @@ def watch(event: Any) -> None: on_event=watch, ) except Exception as broke: # noqa: BLE001 - one slice failing must not lose the others - progress.failed(destination, mine.named(), str(broke)) + logger.warning("slice %s failed after %s: %s", mine.named(), len(kept), broke) if on_event: on_event({"type": "slice_failed", "slice": mine.named(), "why": str(broke)[:300]}) return list(kept) - progress.finished(destination, mine.named(), len(kept)) + logger.info("slice %s finished with %s of %s", mine.named(), len(kept), mine.count) return list(kept) @@ -581,11 +591,12 @@ async def write_in_parallel( at_once = max(1, min(at_once or AT_ONCE, MOST_AT_ONCE)) allocation = planned(wanted, cases, slices) - progress.planned( - destination, - [(one.named(), one.count) for one in allocation], - at_once=at_once, - asked=wanted, + logger.info( + "writing %s scenarios across %s slices, %s at a time: %s", + wanted, + len(allocation), + at_once, + ", ".join(f"{one.named()} x{one.count}" for one in allocation), ) if on_event: on_event( @@ -629,15 +640,13 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar break if on_event: on_event({"type": "topping_up", "slices": [one.named() for one in missing]}) - progress.planned( - destination, - [(one.named(), one.count) for one in [*allocation, *missing]], - at_once=at_once, - asked=wanted, + logger.info( + "topping up %s of %s with %s more slices: %s", + len(suite), + wanted, + len(missing), + ", ".join(f"{one.named()} x{one.count}" for one in missing), ) - for one in [*allocation, *missing]: - if one in allocation: - progress.finished(destination, one.named(), one.count) more = await asyncio.gather( *( guarded(one, missing, len(allocation) + index) @@ -652,7 +661,7 @@ async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenar break write_scenarios(suite, destination, load_catalogue(destination)) - progress.settled(destination, kept_total=len(suite)) + logger.info("suite saved: %s of %s asked for", len(suite), wanted) if on_event: on_event({"type": "saved", "kept": len(suite), "asked": wanted}) return load(destination) diff --git a/src/fi/alk/harness/secrets.py b/src/fi/alk/harness/secrets.py index 35067b74..497fcf6b 100644 --- a/src/fi/alk/harness/secrets.py +++ b/src/fi/alk/harness/secrets.py @@ -95,6 +95,9 @@ def worker_environment( # Runner-owned model configuration. Uploaded agent values with these names remain in the # runtime namespace and cannot replace controller credentials. "ALK_HARNESS_MODEL", + # Lets a local sandbox hand its own Google credential to the runtime it builds. A hosted + # runner leaves this unset so its platform key is never mounted into a submitted agent. + "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", @@ -144,6 +147,8 @@ def worker_environment( child = {name: value for name, value in host.items() if name in allowed} reserved = { "ALK_HARNESS_MODEL", + # A submitted job must not be able to turn on the host-credential fallback for itself. + "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index cabd48a0..baf41eb6 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -368,9 +368,11 @@ def _download() -> str | None: clip_source: Any = await asyncio.to_thread(_download) if not clip_source: return + self._background_noise_file = clip_source else: clip_source = getattr(BuiltinAudioClip, source, None) if clip_source is None: + logger.warning("background audio clip %r is not one LiveKit ships", source) return player = BackgroundAudioPlayer( ambient_sound=AudioConfig(clip_source, volume=volume) @@ -380,6 +382,27 @@ def _download() -> str | None: except Exception: logger.warning("background audio not started", exc_info=True) + async def _stop_background_audio(self) -> None: + """Close the ambience player and remove any clip downloaded for it. + + Without this the mixer task, its audio source and the published track outlive the call, + and a suite leaks one of each (plus a temp file) per scenario. + """ + player = getattr(self, "_background_player", None) + if player is not None: + self._background_player = None + try: + await player.aclose() + except Exception: + logger.warning("background audio not closed cleanly", exc_info=True) + downloaded = getattr(self, "_background_noise_file", None) + if downloaded: + self._background_noise_file = None + try: + Path(downloaded).unlink(missing_ok=True) + except OSError: + logger.warning("background audio clip not removed: %s", downloaded) + def open_conversation(self) -> None: if self._session is None: raise RuntimeError("simulator_session_not_started") @@ -1363,6 +1386,7 @@ def on_target_transcription( details={"exception_type": type(exc).__name__}, ) finally: + await self._stop_background_audio() if target_transcription_handler_registered: room.unregister_text_stream_handler(TOPIC_TRANSCRIPTION) pending_target_transcriptions.clear() From 875e69ae3bb781393a555258d89bc60a3acd19d8 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 12:26:05 +0530 Subject: [PATCH 10/41] fix(simulator): drop Cartesia voices the provider no longer serves --- .../data/voices_by_language_and_gender.json | 62 +------------------ 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/src/fi/alk/harness/run/data/voices_by_language_and_gender.json b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json index 13e3028c..5daedb1e 100644 --- a/src/fi/alk/harness/run/data/voices_by_language_and_gender.json +++ b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json @@ -17,7 +17,6 @@ "bf0a246a-8642-498a-9950-80c35e9276b5", "57dcab65-68ac-45a6-8480-6c4c52ec1cd1", "78ab82d5-25be-4f7d-82b3-7ad64e5b85b2", - "794f9389-aac1-45b6-b726-9d9369183238", "03496517-369a-4db1-8236-3d3ae459ddf7", "e8e5fffb-252c-436d-b842-8879b84445b6", "b7d50908-b17c-442d-ad8d-810c63997ed9", @@ -37,18 +36,15 @@ "f6ff7c0c-e396-40a9-a70b-f7607edb6937", "11af83e2-23eb-452f-956e-7fee218ccb5c", "e13cae5c-ec59-4f71-b0a6-266df3c9bb8e", - "6adbb439-0865-468c-9e68-adbb0eb2e71c", "a01c369f-6d2d-4185-bc20-b32c225eab70", "7ea5e9c2-b719-4dc3-b870-5ba5f14d31d8", "f8f5f1b2-f02d-4d8e-a40d-fd850a487b3d", - "d7e54830-4754-4b17-952c-bcdb7e80a2fb", "a38e4e85-e815-43ab-acf1-907c4688dd6c", "f31cc6a7-c1e8-4764-980c-60a361443dd1", "21b81c14-f85b-436d-aff5-43f2e788ecf8", "f6141af3-5f94-418c-80ed-a45d450e7e2e", "8985388c-1332-4ce7-8d55-789628aa3df4", "043cfc81-d69f-4bee-ae1e-7862cb358650", - "1d3ba41a-96e6-44ad-aabb-9817c56caa68", "c8605446-247c-4d39-acd4-8f4c28aa363c", "607167f6-9bf2-473c-accc-ac7b3b66b30b", "cccc21e8-5bcf-4ff0-bc7f-be4e40afc544", @@ -57,14 +53,9 @@ "bf991597-6c13-47e4-8411-91ec2de5c466", "daf747c6-6bc2-4083-bd59-aa94dce23f5d", "c9440d34-5641-427b-bbb7-80ef7462576d", - "5c9e800f-2a92-4720-969b-99c4ab8fbc87", - "6d287143-8db3-434a-959c-df147192da27", "56b87df1-594d-4135-992c-1112bb504c59", "0c8ed86e-6c64-40f0-b252-b773911de6bb", - "573e3144-a684-4e72-ac2b-9b2063a50b53", - "15a9cd88-84b0-4a8b-95f2-5d583b54c72e", "f4e8781b-a420-4080-81cf-576331238efa", - "a8136a0c-9642-497a-882d-8d591bdcb2fa", "57b6bf63-c7a1-4ffc-8e10-23bf45152dd6", "5e10a334-7fa5-46d4-a64b-5ae6185da3fd", "761afc95-bef5-44dd-aa07-d3c678912e43", @@ -185,7 +176,6 @@ "1fc31370-81b1-4588-9c1a-f93793c6e01d", "87bc56aa-ab01-4baa-9071-77d497064686", "f114a467-c40a-4db8-964d-aaba89cd08fa", - "bd9120b6-7761-47a6-a446-77ca49132781", "701a96e1-7fdd-4a6c-a81e-a4a450403599", "3e1ed423-17e5-4773-b87c-25b031106e41", "da4a4eff-3b7e-4846-8f70-f075ff61222c", @@ -201,32 +191,20 @@ "2a4d065a-ac91-4203-a015-eb3fc3ee3365", "40104aff-a015-4da1-9912-af950fbec99e", "86e30c1d-714b-4074-a1f2-1cb6b552fb49", - "50d6beb4-80ea-4802-8387-6c948fe84208", - "63ff761f-c1e8-414b-b969-d1833d1c870c", - "ab109683-f31f-40d7-b264-9ec3e26fb85e", "41f3c367-e0a8-4a85-89e0-c27bae9c9b6d", "c45bc5ec-dc68-4feb-8829-6e6b2748095d", - "7fe6faca-172f-4fd9-a193-25642b8fdb07", - "ec58877e-44ae-4581-9078-a04225d42bd4", - "3dcaa773-fb1a-47f7-82a4-1bf756c4e1fb", "726d5ae5-055f-4c3d-8355-d9677de68937", "96c64eb5-a945-448f-9710-980abe7a514c", "39b376fc-488e-4d0c-8b37-e00b72059fdd", - "7360f116-6306-4e9a-b487-1235f35a0f21", "bbee10a8-4f08-4c5c-8282-e69299115055", "0b32066b-2bcc-44b9-89ab-0223a09d1606", "bfd3644b-d561-4b1c-a01f-d9af98cb67c0", - "8d110413-2f14-44a2-8203-2104db4340e9", "5619d38c-cf51-4d8e-9575-48f61a280413", "34d923aa-c3b5-4f21-aac7-2c1f12730d4b", - "64462aed-aafc-45d4-84cd-ecb4b3763a0a", "5c43e078-5ba4-4e1f-9639-8d85a403f76a", - "36b42fcb-60c5-4bec-b077-cb1a00a92ec6", "d7862948-75c3-4c7c-ae28-2959fe166f49", "6a176356-ada1-4b48-b2ae-3a3fdd485680", - "586b6832-1ca1-43ad-b974-527dc13c2532", "66f5935b-af2e-4ec9-bb3e-59112e9ddc93", - "236bb1fb-dc41-4a2b-84d6-d22d2a2aaae1", "ee8b13e7-98af-4b15-89d1-8d402be10c94", "1cb5b8bc-77c9-4e7c-a251-da02348e2727", "f24ae0b7-a3d2-4dd1-89df-959bdc4ab179", @@ -297,7 +275,6 @@ "2f22b9bc-b0eb-4cb6-b5ae-0c099a0fdfad", "79bfcec0-720c-41f2-a33a-f12383e9627f", "e2d48e7b-cd73-4c4c-bc1e-f232580e8709", - "39c3388d-6b3f-4cec-88d7-900bd0899e00", "c63361f8-d142-4c62-8da7-8f8149d973d6", "9287676d-f0cc-423f-ac03-3b3c7242f091", "da69d796-4603-4419-8a95-293bfc5679eb", @@ -324,14 +301,7 @@ "61001bc6-9064-40a4-b8b2-29178e0fa558", "5c7b66c2-3b58-464d-8a12-093410a269c5", "3d79b1fd-daaa-439c-bff3-903dc18e7684", - "cf14fdcd-24a0-4d63-958a-c784f33d8e7c", - "cb605424-d682-48e9-94db-34cc567cf1c6", - "abe7dee1-6051-43d3-9a9f-1ac1312497a7", - "aa086107-101b-4182-a628-c51186d74166", - "911b8b22-887f-4caf-bf87-85d834c08708", - "876c39e1-9ecd-42cd-b0c1-8b3906f0be19", - "83e45f18-fac4-40db-a43b-03257883b437", - "64875a07-f57e-4a70-b702-4e3fb25efeda" + "911b8b22-887f-4caf-bf87-85d834c08708" ] }, "es": { @@ -345,12 +315,9 @@ ], "male": [ "15d0c2e2-8d29-44c3-be23-d585d5f154a1", - "79743797-2087-422f-8dc7-86f9efca85f1", "2695b6b5-5543-4be1-96d9-3967fb5e7fec", "b5aa8098-49ef-475d-89b0-c9262ecf33fd", - "846fa30b-6e1a-49b9-b7df-6be47092a09a", - "b042270c-d46f-4d4f-8fb0-7dd7c5fe5615", - "5ef98b2a-68d2-4a35-ac52-632a2d288ea6" + "b042270c-d46f-4d4f-8fb0-7dd7c5fe5615" ] }, "hi": { @@ -369,7 +336,6 @@ "be79f378-47fe-4f9c-b92b-f02cefa62ccf", "9b953e7b-86a8-42f0-b625-1434fb15392b", "bdab08ad-4137-4548-b9db-6142854c7525", - "7f423809-0011-4658-ba48-a411f5e516ba", "393dd459-f8d8-4c3e-a86b-ec43a1113d0b", "791d5162-d5eb-40f0-8189-f19db44611d8" ] @@ -380,7 +346,6 @@ "4ab1ff51-476d-42bb-8019-4d315f7c0c05", "38aabb6a-f52b-4fb0-a3d1-988518f4dc06", "3f4ade23-6eb4-4279-ab05-6a144947c4d5", - "11c61307-4f9e-4db8-ac3b-bfa5f2a731ce", "1ade29fc-6b82-4607-9e70-361720139b12", "6d4b1416-8d54-4d94-a788-8a802c086544" ], @@ -415,7 +380,6 @@ "36d94908-c5b9-4014-b521-e69aee5bead0" ], "male": [ - "e5923af7-a329-4e9b-b95a-5ace4a083535", "408daed0-c597-4c27-aae8-fa0497d644bf", "e019ed7e-6079-4467-bc7f-b599a5dccf6f", "79693aee-1207-4771-a01e-20c393c89e6f", @@ -425,8 +389,6 @@ }, "pl": { "male": [ - "82a7fc13-2927-4e42-9b8a-bb1f9e506521", - "4ef93bb3-682a-46e6-b881-8e157b6b4388", "3d335974-4c4a-400a-84dc-ebf4b73aada6", "2a3503b2-b6b6-4534-a224-e8c0679cec4a", "887149a8-4616-42ad-b2ce-c3819176f45d" @@ -456,51 +418,34 @@ "f39bf583-3b3d-402f-9ffb-6179d9ec3e35" ], "male": [ - "5063f45b-d9e0-4095-b056-8f3ee055d411", - "a37639f0-2f0a-4de4-9942-875a187af878", - "6a16c1f4-462b-44de-998d-ccdaa4125a0a", "6a360542-a117-4ed5-9e09-e8bf9b05eabb", "fbee0e7d-a83a-4082-bad1-13c70f86da4e" ] }, "ja": { "female": [ - "0cd0cde2-3b93-42b5-bcb9-f214a591aa29", "c7eafe22-8b71-40cd-850b-c5a3bbd8f8d2", "59d4fd2f-f5eb-4410-8105-58db7661144f", "2b568345-1d48-4047-b25f-7baccf842eb0", - "44863732-e415-4084-8ba1-deabe34ce3d2", "31c55968-a9f4-4115-8831-3a16952179c8" ], "male": [ "e8a863c6-22c7-4671-86ca-91cacffc038d", "6b92f628-be90-497c-8f4c-3b035002df71", - "06950fa3-534d-46b3-93bb-f852770ea0b5", - "446f922f-c43a-4aad-9a8b-ad2af568e882", - "9e7ef2cf-b69c-46ac-9e35-bbfd73ba82af", - "a759ecc5-ac21-487e-88c7-288bdfe76999", - "97e7d7a9-dfaa-4758-a936-f5f844ac34cc", "b8e1169c-f16a-4064-a6e0-95054169e553" ] }, "ko": { "female": [ "304fdbd8-65e6-40d6-ab78-f9d18b9efdf9", - "29e5f8b4-b953-4160-848f-40fae182235b", - "663afeec-d082-4ab5-827e-2e41bf73a25b", "15628352-2ede-4f1b-89e6-ceda0c983fbc" - ], - "male": [ - "af6beeea-d732-40b6-8292-73af0035b740" ] }, "zh": { "female": [ "7a5d4663-88ae-47b7-808e-8f9b9ee4127b", "bf32f849-7bc9-4b91-8c62-954588efcc30", - "e90c6678-f0d3-4767-9883-5d0ecf5894a8", "f9a4b3a6-b44b-469f-90e3-c8e19bd30e99", - "0b904166-a29f-4d2e-bb20-41ca302f98e9", "a53c3509-ec3f-425c-a223-977f5f7424dd" ], "male": [ @@ -530,15 +475,12 @@ "00510a15-4216-4fdc-a0ab-05d74cd9f795" ], "male": [ - "38a146c3-69d7-40ad-aada-76d5a2621758", "0caedb75-417f-4e36-9b64-c21354cb94c8", "32a806e8-894e-41ad-a4d5-6d9154d7b1e6" ] }, "nl": { "male": [ - "9e8db62d-056f-47f3-b3b6-1b05767f9176", - "4aa74047-d005-4463-ba2e-a0d9b261fb87", "af482421-80f4-4379-b00c-a118def29cde", "4b250449-c635-4b63-bd1d-b654b12ffcd4" ], From 754c1790064f641469f6484957da03da9464e091 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 13:02:50 +0530 Subject: [PATCH 11/41] chore(harness): move credential and runtime limit changes to their own branch --- src/fi/alk/harness/provision.py | 27 --------------- src/fi/alk/harness/run/live.py | 31 ----------------- src/fi/alk/harness/secrets.py | 37 --------------------- src/fi/alk/harness/session.py | 2 +- src/fi/alk/harness/world/stores/postgres.py | 11 ++---- 5 files changed, 4 insertions(+), 104 deletions(-) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index aecce513..0836b5f2 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -1804,17 +1804,6 @@ def provision( else: compose = None managed = False - # The harness's fidelity order is provisioned > adopted > generated: run the agent's real - # services whenever it ships them. _managed_compose only models "agent + datastore" and - # silently drops any other service the agent's tools are actually served by -- an HTTP - # tools-api, a queue, a mock upstream -- which then leaves the world with no endpoint to - # forward to, so every tool call comes back "no such tool". So prefer the agent's own shipped - # Compose whenever it ships one (its real tool services come up and the world forwards to - # them), and fall back to the generated adapter only for agents that ship no usable Compose. - if compose is None: - shipped = compose_file(source_root) - if shipped is not None: - compose = shipped if compose is None and contract is not None: if not packaging.candidates and not (source_root / "Dockerfile").is_file(): try: @@ -2238,18 +2227,6 @@ def _runtime_credential_mounts( # the worker to see the placeholder. Platform credentials always get an # ALK-owned destination that cannot collide with submitted mounts. target = f"/run/harness-secrets/{runtime.name}" - if source is None or not _valid_google_credentials(source): - # A local sandbox supplies the credential through its own environment rather than - # an uploaded runtime configuration. That credential belongs to whoever is running - # the harness, not to the submitted agent, so it is only ever handed over when the - # operator says so: without this opt-in a hosted runner would mount its own - # platform key into a container it does not trust. - if os.environ.get("ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "").strip() == "1": - env_value = os.environ.get(name, "").strip() - env_path = Path(env_value).expanduser() if env_value else None - if env_path is not None and _valid_google_credentials(env_path): - source = env_path - target = f"/run/harness-secrets/{env_path.name}" if source is None or not _valid_google_credentials(source): raise ProvisionError( "the submitted runtime needs GOOGLE_APPLICATION_CREDENTIALS, but neither its " @@ -2381,10 +2358,6 @@ def start_runtime( arguments.extend(("--volume", f"{source}:{target}:ro")) injected[name] = target mounted_credentials.add(name) - # A submitted service that declares its own credential volume has already been mounted (the - # host secret now lives at the container target). Re-validating that container path as a host - # file below would always fail, so only resolve GOOGLE_APPLICATION_CREDENTIALS here when it - # arrived as an injected host path (the generated-runtime path, which mounts no credentials). google_path = injected.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() if google_path and "GOOGLE_APPLICATION_CREDENTIALS" not in mounted_credentials: google_source = Path(google_path).expanduser() diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py index e6873e64..90b4978e 100644 --- a/src/fi/alk/harness/run/live.py +++ b/src/fi/alk/harness/run/live.py @@ -278,37 +278,6 @@ def wire( "TOOLS_API_URL": url, "LIVEKIT_AGENT_NAME": agent_name, } - # The harness-generated agent-runtime does not carry the agent's own .env.local, and - # runtime_configuration_names only covers datastore config -- not the provider - # credentials the worker needs to actually run: LiveKit to register and place the - # call, Deepgram for STT/TTS, Vertex for the LLM. Pass them through from this - # process's environment (the sandbox has placed them here). - for _cred in ( - "LIVEKIT_URL", - "LIVEKIT_API_KEY", - "LIVEKIT_API_SECRET", - "DEEPGRAM_API_KEY", - "CARTESIA_API_KEY", - "GOOGLE_APPLICATION_CREDENTIALS", - "GOOGLE_CLOUD_PROJECT", - "GOOGLE_CLOUD_LOCATION", - ): - _value = os.environ.get(_cred, "").strip() - if _value: - runtime_overrides.setdefault(_cred, _value) - # An agent's Compose commonly mounts its Vertex credential from an env-var source, e.g. - # ${VERTEX_CREDENTIALS:-/dev/null}:/etc/vertex/creds.json. With that variable unset the - # placeholder is mounted and the agent's own LLM cannot authenticate, so point it at the - # same resolved Google credential the harness already holds for the run. - _google_creds = runtime_overrides.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() - if _google_creds: - runtime_overrides.setdefault("VERTEX_CREDENTIALS", _google_creds) - # Voice agents often gate an audio-enhancement plugin on a license the harness cannot - # supply (e.g. ai_coustics noise cancellation). Unauthorized, it raises on the first - # inbound audio frame and the worker drops the call right after its greeting. Agents - # that read this flag skip that plugin; agents that do not simply ignore it. A run may - # override it to keep enhancement on when a license is present. - runtime_overrides.setdefault("DISABLE_AI_COUSTICS", "1") os.environ["LIVEKIT_TARGET_AGENT_NAME"] = agent_name caller_phone = fixture_phone(scenario) if caller_phone: diff --git a/src/fi/alk/harness/secrets.py b/src/fi/alk/harness/secrets.py index 497fcf6b..018ecdbf 100644 --- a/src/fi/alk/harness/secrets.py +++ b/src/fi/alk/harness/secrets.py @@ -95,9 +95,6 @@ def worker_environment( # Runner-owned model configuration. Uploaded agent values with these names remain in the # runtime namespace and cannot replace controller credentials. "ALK_HARNESS_MODEL", - # Lets a local sandbox hand its own Google credential to the runtime it builds. A hosted - # runner leaves this unset so its platform key is never mounted into a submitted agent. - "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", @@ -111,44 +108,10 @@ def worker_environment( "FI_BASE_URL", "FI_API_KEY", "FI_SECRET_KEY", - # Runner-owned Docker runtime + voice configuration. The harness starts store and agent - # containers on the host daemon and must reach them: ALK_DOCKER_NETWORK lets the store be - # reached by container name on a shared network, ALK_DOCKER_PUBLISHED_HOST/BIND_HOST give - # the host published services are on. Without these the child defaults to 127.0.0.1 -- - # its own loopback inside the sandbox -- and every store probe is refused. - "ALK_DOCKER_NETWORK", - "ALK_DOCKER_PUBLISHED_HOST", - "ALK_DOCKER_BIND_HOST", - "ALK_RUNNER_CONTAINER", - "ALK_HARNESS_MODEL", - "ALK_AGENT_MODEL", - "ALK_JUDGE_MODEL", - "ALK_USER_MODEL", - "CLOUD_ML_REGION", - "HARNESS_WEBHOOK_HOST", - "HARNESS_WEBHOOK_PORT", - "HARNESS_WEBHOOK_URL", - "HARNESS_RUNTIME_WEBHOOK_URL", - "HARNESS_VOICE_CASE", - "HARNESS_VOICE_INFRA_RETRIES", - "LIVEKIT_TARGET_AGENT_NAME", - # Local-dev convenience: let the developer's provider creds from their local environment - # reach the worker directly. A hosted provider supplies these through secret_refs instead. - "LIVEKIT_URL", - "LIVEKIT_API_KEY", - "LIVEKIT_API_SECRET", - "ACCEPTANCE_LIVEKIT_URL", - "DEEPGRAM_API_KEY", - "CARTESIA_API_KEY", - "GOOGLE_APPLICATION_CREDENTIALS", - "GOOGLE_CLOUD_PROJECT", - "GOOGLE_CLOUD_LOCATION", } child = {name: value for name, value in host.items() if name in allowed} reserved = { "ALK_HARNESS_MODEL", - # A submitted job must not be able to turn on the host-credential fallback for itself. - "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 4f2db62a..af652abd 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -38,7 +38,7 @@ # remain alive forever after a dropped upstream stream, though, which previously left a hosted # job looking healthy while making no progress. Bound *inactivity*, not total stage duration: # long scenario suites remain valid as long as they keep producing observable work. -STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "600")) +STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "180")) STAGE_IDLE_RETRIES = int(os.getenv("ALK_STAGE_IDLE_RETRIES", "1")) diff --git a/src/fi/alk/harness/world/stores/postgres.py b/src/fi/alk/harness/world/stores/postgres.py index 8f9f83b6..0a89b444 100644 --- a/src/fi/alk/harness/world/stores/postgres.py +++ b/src/fi/alk/harness/world/stores/postgres.py @@ -237,14 +237,9 @@ def save_to(self, path: str | Path) -> None: def load_from(self, path: str | Path) -> None: root = Path(path) schema = root / SCHEMA - # A standalone scenario store starts empty and needs the DDL; a compose-provisioned - # (Attached) store saves no schema.sql because store.json already carries the applied - # CREATE scripts, which Held.load_from replays. Apply schema.sql only when it exists and - # the tables are not already there, so restore works either way and never double-applies. - with self._connect() as connection: - has_schema = bool(self._tables(connection)) - if not has_schema and schema.exists(): - self.apply(schema.read_text(encoding="utf-8")) + if not schema.exists(): + raise StoreError(f"no saved Postgres schema at {schema}") + self.apply(schema.read_text(encoding="utf-8")) Held.load_from(self, root) # -- what a scenario changes ----------------------------------------------------- From 76616cabaa7c5be5133cf3fdb1ea648400649013 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 13:11:06 +0530 Subject: [PATCH 12/41] chore(harness): tighten scenario comments and drop em dashes from generated names --- src/fi/alk/harness/persona_guides.py | 2 +- src/fi/alk/harness/scenario.py | 21 +++++++------------ src/fi/alk/harness/scenarios.py | 2 +- .../harness/skills/write-scenarios/SKILL.md | 2 +- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py index 01231a6d..0226942b 100644 --- a/src/fi/alk/harness/persona_guides.py +++ b/src/fi/alk/harness/persona_guides.py @@ -12,7 +12,7 @@ Read, not imported: the tables live inside a Django app this package cannot import, but they are plain literals, so they are parsed out of the file. Absent, every lookup answers with nothing and -a persona still renders — one without guidance, never a crash. +a persona still renders, one without guidance, never a crash. """ from __future__ import annotations diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 7eb6e782..c71436bd 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -154,10 +154,8 @@ class Scenario(BaseModel): name: str use_case: str = "" - # Which branch of that use case this is: the condition that makes this row different from - # its siblings. A use case fans out into several — the ordinary path, the one that cannot be - # completed, the rule under pressure — and each is its own test. Coverage is counted on the - # pair, so a use case can carry many scenarios without any of them reading as a duplicate. + # What makes this row different from its siblings in the same use case. Coverage is counted + # on the pair, so a use case can carry many scenarios without any reading as a duplicate. branch: str = "" tests: str = "" @@ -200,13 +198,9 @@ class Scenario(BaseModel): max_turns: int = 10 - # Where this call is being made from, so the agent is heard through it. ``True`` asks for - # noise and leaves the place to the fixture; a string names it outright ("street", "vehicle", - # "retail"), which is what lets one scenario be a call from a car and another from an office. - # Recorded per scenario rather than per run, so a suite covers both conditions and the same - # scenario stays comparable to itself across runs. Chosen at random when the writer does not - # say, because a suite where every call is quiet tests an agent nobody has: real callers - # phone from cars, kitchens and streets. + # Where this call is made from. True asks for noise and leaves the place to the fixture; a + # string names it ("street", "vehicle", "retail"). Random when the writer does not say, so a + # suite covers both conditions rather than testing only callers in quiet rooms. background_noise: bool | str = Field(default_factory=lambda: random.choice((True, False))) def slots(self) -> dict[str, str]: @@ -238,9 +232,8 @@ def validate_scenario( ): problems.append("persona is incomplete: " + ", ".join(missing)) elif scenario.persona is not None: - # A persona written in words of its own renders fine and then does nothing: no behaviour - # guidance attaches to it, and the accent it names selects no voice. Caught here, where - # the writer is still holding the scenario and can fix it in one turn. + # A persona in words of its own renders fine and then does nothing: no behaviour guidance + # attaches, and the accent it names selects no voice. from .persona_guides import unrecognised problems.extend(unrecognised(scenario.persona.model_dump())) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 1f9cc3f2..bb5fdd35 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -184,7 +184,7 @@ class Slice: why: str = "" def named(self) -> str: - return f"{self.use_case} — {self.angle}" if self.angle else self.use_case + return f"{self.use_case}: {self.angle}" if self.angle else self.use_case def even_slices(wanted: int, use_cases: list[str]) -> list[Slice]: diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 67405d97..fec31c85 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -234,7 +234,7 @@ Every stance still obeys the bar above: a real person could bring it, a competen fail it, and the values are real. A stance chooses *what to look at*, never whether the scenario has to be honest. -Two rules keep this from turning into noise. **Each scenario carries one use case and one branch, and no two scenarios carry the same pair** — a duplicate is either the same test twice or one of them is mislabelled, and it hides a gap while appearing to fill it. Several scenarios sharing a use case is normal and expected; that is what branches are for. What is not allowed is two rows that agree on both. And a stance that produces nothing new +Two rules keep this from turning into noise. **Each scenario carries one use case and one branch, and no two scenarios carry the same pair**: a duplicate is either the same test twice or one of them is mislabelled, and it hides a gap while appearing to fill it. Several scenarios sharing a use case is normal and expected; that is what branches are for. What is not allowed is two rows that agree on both. And a stance that produces nothing new for a given agent produces nothing: an agent with no rules to bend does not need an adversarial scenario invented for it. From 8cc41f703c055c5f40721ec3fd9c33888c4793e9 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:16:36 +0530 Subject: [PATCH 13/41] feat(simulator): fill the caller prompt from a template and drive speech from the persona --- .../alk/harness/data/persona_vocabulary.json | 111 ++++++++++++++++++ src/fi/alk/harness/persona_guides.py | 34 +++++- src/fi/alk/harness/run/call.py | 9 ++ src/fi/alk/harness/run/sdk_voice.py | 81 +++++++++++-- src/fi/alk/harness/run/tools.py | 8 ++ src/fi/simulate/agent/definition.py | 8 ++ src/fi/simulate/simulation/engines/livekit.py | 6 + src/fi/simulate/simulation/voice_prompt.py | 110 ++++++++++++++--- 8 files changed, 340 insertions(+), 27 deletions(-) create mode 100644 src/fi/alk/harness/data/persona_vocabulary.json diff --git a/src/fi/alk/harness/data/persona_vocabulary.json b/src/fi/alk/harness/data/persona_vocabulary.json new file mode 100644 index 00000000..279b6729 --- /dev/null +++ b/src/fi/alk/harness/data/persona_vocabulary.json @@ -0,0 +1,111 @@ +{ + "GenderChoices": [ + "male", + "female" + ], + "AgeGroupChoices": [ + "18-25", + "25-32", + "32-40", + "40-50", + "50-60", + "60+" + ], + "LocationChoices": [ + "United States", + "Canada", + "United Kingdom", + "Australia", + "India" + ], + "ProfessionChoices": [ + "Student", + "Teacher", + "Engineer", + "Doctor", + "Nurse", + "Business Owner", + "Manager", + "Sales Representative", + "Customer Service", + "Technician", + "Consultant", + "Accountant", + "Marketing Professional", + "Retired", + "Homemaker", + "Freelancer", + "Other" + ], + "PersonalityChoices": [ + "Friendly and cooperative", + "Professional and formal", + "Cautious and skeptical", + "Impatient and direct", + "Detail-oriented", + "Easy-going", + "Anxious", + "Confident", + "Analytical", + "Emotional", + "Reserved", + "Talkative" + ], + "CommunicationStyleChoices": [ + "Direct and concise", + "Detailed and elaborate", + "Casual and friendly", + "Formal and polite", + "Technical", + "Simple and clear", + "Questioning", + "Assertive", + "Passive", + "Collaborative" + ], + "AccentChoices": [ + "American", + "Australian", + "Indian", + "Canadian", + "Neutral" + ], + "LanguageChoices": [ + "Arabic", + "Bulgarian", + "Chinese Simplified", + "Czech", + "Danish", + "Dutch", + "English", + "Finnish", + "French", + "German", + "Greek", + "Hindi", + "Hungarian", + "Indonesian", + "Italian", + "Japanese", + "Korean", + "Malay", + "Norwegian", + "Polish", + "Portuguese", + "Romanian", + "Russian", + "Slovak", + "Spanish", + "Swedish", + "Turkish", + "Ukrainian", + "Vietnamese" + ], + "ConversationSpeedChoices": [ + "0.5", + "0.75", + "1.0", + "1.25", + "1.5" + ] +} diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py index 0226942b..b2edf43b 100644 --- a/src/fi/alk/harness/persona_guides.py +++ b/src/fi/alk/harness/persona_guides.py @@ -18,6 +18,8 @@ from __future__ import annotations import ast +import logging +import json import os from functools import lru_cache from pathlib import Path @@ -101,6 +103,8 @@ def available() -> bool: return bool(guides()) +logger = logging.getLogger(__name__) + # Where the platform's persona model is mounted, for the values it accepts. VOCABULARY_ENV = "HARNESS_PERSONA_VOCABULARY" @@ -135,11 +139,15 @@ def vocabulary() -> dict[str, list[str]]: """ path = os.environ.get(VOCABULARY_ENV) or "" if not path or not Path(path).exists(): - return {} + # No model mounted. Fall back to the copy carried with the harness so a writer is always + # offered real values: an empty vocabulary silently lets it invent an accent that selects + # no voice and a personality that attaches no guidance. + return _bundled_vocabulary() try: tree = ast.parse(Path(path).read_text(encoding="utf-8")) except (OSError, SyntaxError): - return {} + logger.warning("persona vocabulary at %s is unreadable; using the bundled copy", path) + return _bundled_vocabulary() by_class: dict[str, list[str]] = {} for node in ast.walk(tree): @@ -166,6 +174,28 @@ def vocabulary() -> dict[str, list[str]]: } + +@lru_cache(maxsize=1) +def _bundled_vocabulary() -> dict[str, list[str]]: + """The platform's persona values, carried with the harness. + + Kept so the harness constrains personas out of the box. Languages come from the agent + definition's set rather than the persona dropdown's two, because nothing on the platform + enforces the dropdown and a caller is expected to speak more than English and Hindi. + """ + path = Path(__file__).parent / "data" / "persona_vocabulary.json" + try: + by_class = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.warning("bundled persona vocabulary is unreadable; personas stay unconstrained") + return {} + return { + field: list(by_class[cls]) + for field, cls in FIELDS.items() + if by_class.get(cls) + } + + def offered(field: str) -> list[str]: """The values this field accepts, or nothing if the platform's model was not readable.""" return list(vocabulary().get(field, [])) diff --git a/src/fi/alk/harness/run/call.py b/src/fi/alk/harness/run/call.py index 7d9932dd..f030712f 100644 --- a/src/fi/alk/harness/run/call.py +++ b/src/fi/alk/harness/run/call.py @@ -107,6 +107,15 @@ def main(argv: list[str] | None = None) -> int: # about how a simulated caller behaves is decided twice. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name + # The caller prompt is a template the harness fills, never prose it composes, so + # the generated template travels to the call with everything else. + from ..simulator import load_simulator_prompt + + _template = load_simulator_prompt(Path(root)) if root else "" + if _template.strip(): + os.environ["HARNESS_SIMULATOR_PROMPT"] = _template + else: + os.environ.pop("HARNESS_SIMULATOR_PROMPT", None) os.environ["HARNESS_OUTCOME"] = scenario.tests os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 8779a15e..2d38eaae 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import logging import asyncio import json import os @@ -26,6 +27,8 @@ ) from fi.simulate.runtime.runner import SimulationRunner +logger = logging.getLogger(__name__) + def _required(name: str) -> str: value = os.environ.get(name, "").strip() @@ -44,14 +47,65 @@ def _json_env(name: str, default): # Language names a persona may carry, to the codes Deepgram STT expects. Unrecognised values that # already look like a code are passed through; everything else falls back to English. +# Language names and region codes to the code the providers expect. Ported from the platform +# so a persona resolves to the same language here as it does there. Anything unrecognised +# falls back to English, which is what the platform does too. _LANGUAGE_CODES: dict[str, str] = { - "english": "en", "hindi": "hi", "hinglish": "hi", "spanish": "es", "french": "fr", - "german": "de", "portuguese": "pt", "italian": "it", "dutch": "nl", "japanese": "ja", - "korean": "ko", "mandarin": "zh", "chinese": "zh", "arabic": "ar", "russian": "ru", - "tamil": "ta", "telugu": "te", "bengali": "bn", + "ar": "ar", "ar-sa": "ar", "arabic": "ar", "bg": "bg", + "bulgarian": "bg", "ca": "ca", "catalan": "ca", "chinese": "zh", + "chinese (cantonese, traditional)": "zh-HK", "chinese (mandarin, simplified)": "zh", "chinese (mandarin, traditional)": "zh-TW", "cs": "cs", + "czech": "cs", "da": "da", "da-dk": "da", "danish": "da", + "de": "de", "de-ch": "de-CH", "dutch": "nl", "el": "el", + "en": "en-US", "en-au": "en-AU", "en-gb": "en-GB", "en-in": "en-IN", + "en-nz": "en-NZ", "en-us": "en-US", "english": "en-US", "es": "es", + "es-419": "es-419", "estonian": "et", "et": "et", "fi": "fi", + "finnish": "fi", "flemish": "nl-BE", "fr": "fr", "fr-ca": "fr-CA", + "french": "fr", "german": "de", "greek": "el", "hi": "hi", + "hindi": "hi", "hu": "hu", "hungarian": "hu", "id": "id", + "indonesian": "id", "it": "it", "italian": "it", "ja": "ja", + "japanese": "ja", "ko": "ko", "ko-kr": "ko", "korean": "ko", + "latvian": "lv", "lithuanian": "lt", "lt": "lt", "lv": "lv", + "malay": "ms", "ms": "ms", "nl": "nl", "nl-be": "nl-BE", + "no": "no", "norwegian": "no", "pl": "pl", "polish": "pl", + "portuguese": "pt", "pt": "pt", "pt-br": "pt-BR", "pt-pt": "pt-PT", + "ro": "ro", "romanian": "ro", "ru": "ru", "russian": "ru", + "sk": "sk", "slovak": "sk", "spanish": "es", "sv": "sv", + "sv-se": "sv", "swedish": "sv", "th": "th", "th-th": "th", + "thai": "th", "tr": "tr", "turkish": "tr", "uk": "uk", + "ukrainian": "uk", "vi": "vi", "vietnamese": "vi", "zh": "zh", + "zh-cn": "zh", "zh-hans": "zh", "zh-hant": "zh-TW", "zh-hk": "zh-HK", + "zh-tw": "zh-TW", } + +def _normalised_language(raw: str) -> str: + """The code the providers expect, from a language name or a region code. + + Ported from the platform: lowercase, strip, exact lookup, and anything unrecognised becomes + English rather than failing the call. + """ + return _LANGUAGE_CODES.get((raw or "").strip().lower(), "en-US") + + +# Languages we transcribe with Deepgram's multilingual model rather than a single language code. +# The platform sends Arabic to Azure, which we do not have, so it joins Spanish on the model that +# does cover it. Deliberate divergence: we only ever use providers we hold keys for. +_MULTILINGUAL_STT = ("ar", "es") + + +def _transcriber_for(language: str) -> tuple[str, str, str]: + """The (provider, model, language) a persona's language needs for speech to text. + + Deepgram throughout, because Deepgram and Cartesia are the only providers configured. A + language Deepgram serves better multilingually is sent to that model instead of its own code. + """ + code = (language or "").lower() + if code.split("-", 1)[0] in _MULTILINGUAL_STT: + return ("deepgram", "nova-3", "multi") + return ("deepgram", "nova-3", language or "en-US") + + def _persona_stt_language() -> str: """The STT language for this call's caller, from the persona's languages. @@ -183,17 +237,22 @@ def _voice_providers() -> tuple[str, str]: def _simulator() -> simulate.SimulatorAgentDefinition: + # The caller's brain is fixed on Vertex Gemini and only the model name is configurable. Its + # voice is not: speech to text and text to speech follow the persona's language, because a + # caller who speaks Japanese cannot be transcribed as English. llm_provider = os.environ.get("SIMULATOR_LLM_PROVIDER", "google") - stt_provider, tts_provider = _voice_providers() + language = _persona_stt_language() + stt_provider, stt_model, stt_language = _transcriber_for(language) + _, tts_provider = _voice_providers() default_tts_voice = ( _CARTESIA_DEFAULT_VOICE if tts_provider == "cartesia" else "aura-asteria-en" ) defaults = { - "llm": {"google": "gemini-2.5-flash-lite", "openai": "gpt-4o-mini"}, - "stt": {"deepgram": "nova-2", "cartesia": "ink-2", "google": "chirp_2"}, + "llm": {"google": "gemini-2.5-flash", "openai": "gpt-4o-mini"}, + "stt": {"deepgram": stt_model or "nova-3", "cartesia": "ink-2", "google": "chirp_2"}, "tts": { "deepgram": "aura-asteria-en", - "cartesia": "sonic-3", + "cartesia": "sonic-3.5", "google": "en-US-Chirp3-HD-Aoede", }, } @@ -205,7 +264,11 @@ def model(kind: str, provider: str) -> str: provider.lower(), next(iter(defaults[kind].values())) ) + # The harness writes the caller prompt as a template; the call fills it. Absent, the SDK + # falls back to its shipped default so a run without one behaves as it always did. + template = os.environ.get("HARNESS_SIMULATOR_PROMPT", "").strip() or None return simulate.SimulatorAgentDefinition( + prompt_template=template, llm={ "provider": llm_provider, "model": model("llm", llm_provider), @@ -214,7 +277,7 @@ def model(kind: str, provider: str) -> str: stt={ "provider": stt_provider, "model": model("stt", stt_provider), - "language": _persona_stt_language(), + "language": stt_language, }, tts={ "provider": tts_provider, diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py index 442512e6..5a2ef9a7 100644 --- a/src/fi/alk/harness/run/tools.py +++ b/src/fi/alk/harness/run/tools.py @@ -30,6 +30,7 @@ from ..catalogue import load_catalogue from ..config import ARTIFACTS_ROOT from ..scenario_tools import load_scenarios +from ..simulator import load_simulator_prompt from ..tools import schema from ..world.snapshot import require_source_implementation from .call import CASE, place_the_call @@ -464,6 +465,13 @@ def placed() -> tuple[LiveRun, str, list[str], str]: # how a simulated caller behaves is not decided in two places. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name + # The caller prompt is a template the harness fills, never prose it composes, + # so the generated template travels to the call with everything else. + _template = load_simulator_prompt(Path(world_root)) if world_root else "" + if _template.strip(): + os.environ["HARNESS_SIMULATOR_PROMPT"] = _template + else: + os.environ.pop("HARNESS_SIMULATOR_PROMPT", None) os.environ["HARNESS_OUTCOME"] = scenario.tests os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) diff --git a/src/fi/simulate/agent/definition.py b/src/fi/simulate/agent/definition.py index ffc32630..73b9a71c 100644 --- a/src/fi/simulate/agent/definition.py +++ b/src/fi/simulate/agent/definition.py @@ -459,6 +459,14 @@ class SimulatorAgentDefinition(BaseModel): "It never replaces persona, situation, or outcome instructions." ), ) + prompt_template: Optional[str] = Field( + None, + description=( + "The caller prompt template, with {{persona}} and any other slots the run supplies. " + "Filled rather than composed, so the prompt is authored once and not rebuilt in code. " + "Falls back to the shipped default when absent." + ), + ) llm: LLMConfig = Field( default_factory=lambda: LLMConfig(model="gpt-4o-mini", temperature=0.6) diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index baf41eb6..d85852b5 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -1643,6 +1643,12 @@ async def _create_customer_agent( default_language=( simulator.stt.language if simulator is not None else None ), + template=( + getattr(simulator, "prompt_template", None) + if simulator is not None + else None + ), + variables={"instruction": persona.situation or ""}, ) if simulator is None: voice_provider = os.environ.get( diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index 0432a2e5..c405c985 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -1,11 +1,15 @@ from __future__ import annotations +import logging +import re from typing import Any, Literal, Mapping from fi.simulate.simulation.models import Persona CallType = Literal["inbound", "outbound"] +logger = logging.getLogger(__name__) + VOICE_PERSONALITY_GUIDES: dict[str, str] = { "friendly and cooperative": "Be warm, approachable, and willing to work together. Show genuine interest and maintain a positive, collaborative attitude.", "professional and formal": "Maintain a business-like demeanor. Use formal language, stay focused, and keep interactions professional.", @@ -318,15 +322,76 @@ def append_voice_execution_rules(prompt: str) -> str: return prompt -def build_voice_simulator_prompt( +# The template the caller prompt is rendered from. The harness fills slots; it does not author +# prose. Mirrors the platform's own default, which pairs a persona block with the situation and +# then scrubs the situation slot because the persona block already carries it. +DEFAULT_SIMULATOR_TEMPLATE = ( + "You are a customer in a voice simulation. {{channel}} " + "Stay consistent with the persona throughout the conversation.\n\n{{persona}}" +) + +_SLOT = re.compile(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}") + + +def render_simulator_prompt( + template: str, persona: Persona, *, call_type: CallType, + variables: Mapping[str, Any] | None = None, agent_name: str | None = None, additional_instructions: str | None = None, default_language: str | None = None, ) -> str: - channel = ( + """Fill a caller-prompt template, the way the platform fills its own. + + ``{{persona}}`` becomes the formatted persona block, ``{{channel}}`` the call direction + sentence, and every other ``{{slot}}`` is taken from ``variables``. ``{{situation}}`` is + dropped rather than filled, because the persona block already states the situation and the + platform removes it for the same reason. + + A template that cannot be rendered is returned to the caller unfilled rather than raising, so + a bad template degrades the call instead of ending the run. + """ + values = dict(variables or {}) + try: + persona_text = format_voice_persona( + persona, call_type=call_type, default_language=default_language + ) + except Exception: + logger.exception("persona_format_failed") + persona_text = "" + values.setdefault("persona", persona_text) + values.setdefault("channel", _channel_sentence(call_type, agent_name)) + + def fill(match: "re.Match[str]") -> str: + name = match.group(1) + if name == "situation": + return "" + if name in values: + return str(values[name]) + logger.warning("simulator_prompt_slot_unfilled", extra={"slot": name}) + return "" + + try: + prompt = _SLOT.sub(fill, template) + except Exception: + logger.exception("simulator_prompt_render_failed") + return template + # Tidy the hole a dropped situation slot leaves behind, as the platform does. + prompt = re.sub(r"Currently,\s*[.]", "", prompt) + prompt = re.sub(r"[ \t]{2,}", " ", prompt).strip() + + if additional_instructions and additional_instructions.strip(): + prompt += ( + "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" + + additional_instructions.strip() + ) + return append_voice_execution_rules(prompt) + + +def _channel_sentence(call_type: CallType, agent_name: str | None) -> str: + return ( f"You will make a call to an agent named {agent_name}." if call_type == "inbound" and agent_name else "You will make a call to an agent." @@ -335,21 +400,32 @@ def build_voice_simulator_prompt( if agent_name else "You will receive a call from an agent." ) - prompt = ( - "You are a customer in a voice simulation. " - f"{channel} Stay consistent with the persona throughout the conversation.\n\n" - + format_voice_persona( - persona, - call_type=call_type, - default_language=default_language, - ) + + +def build_voice_simulator_prompt( + persona: Persona, + *, + call_type: CallType, + agent_name: str | None = None, + additional_instructions: str | None = None, + default_language: str | None = None, + template: str | None = None, + variables: Mapping[str, Any] | None = None, +) -> str: + """The caller prompt for one simulated customer. + + Renders ``template`` when one is supplied, and the shipped default otherwise, so a run with no + template configured still produces the prompt it always did. + """ + return render_simulator_prompt( + template or DEFAULT_SIMULATOR_TEMPLATE, + persona, + call_type=call_type, + variables=variables, + agent_name=agent_name, + additional_instructions=additional_instructions, + default_language=default_language, ) - if additional_instructions and additional_instructions.strip(): - prompt += ( - "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" - + additional_instructions.strip() - ) - return append_voice_execution_rules(prompt) __all__ = [ @@ -357,6 +433,8 @@ def build_voice_simulator_prompt( "VOICE_COMMUNICATION_STYLE_GUIDES", "VOICE_PERSONALITY_GUIDES", "append_voice_execution_rules", + "DEFAULT_SIMULATOR_TEMPLATE", "build_voice_simulator_prompt", + "render_simulator_prompt", "format_voice_persona", ] From f5e0e3e89cb4a686c2877f1de8697ee4b808535f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:22:40 +0530 Subject: [PATCH 14/41] feat(simulator): fix the caller prompt template in code instead of generating it --- src/fi/alk/harness/run/call.py | 7 ------- src/fi/alk/harness/run/sdk_voice.py | 4 ---- src/fi/alk/harness/run/tools.py | 8 -------- src/fi/alk/harness/scenario.py | 14 ++++++++++++-- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/fi/alk/harness/run/call.py b/src/fi/alk/harness/run/call.py index f030712f..0bbb0df2 100644 --- a/src/fi/alk/harness/run/call.py +++ b/src/fi/alk/harness/run/call.py @@ -109,13 +109,6 @@ def main(argv: list[str] | None = None) -> int: os.environ["HARNESS_SCENARIO"] = scenario.name # The caller prompt is a template the harness fills, never prose it composes, so # the generated template travels to the call with everything else. - from ..simulator import load_simulator_prompt - - _template = load_simulator_prompt(Path(root)) if root else "" - if _template.strip(): - os.environ["HARNESS_SIMULATOR_PROMPT"] = _template - else: - os.environ.pop("HARNESS_SIMULATOR_PROMPT", None) os.environ["HARNESS_OUTCOME"] = scenario.tests os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 2d38eaae..1250c3cd 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -264,11 +264,7 @@ def model(kind: str, provider: str) -> str: provider.lower(), next(iter(defaults[kind].values())) ) - # The harness writes the caller prompt as a template; the call fills it. Absent, the SDK - # falls back to its shipped default so a run without one behaves as it always did. - template = os.environ.get("HARNESS_SIMULATOR_PROMPT", "").strip() or None return simulate.SimulatorAgentDefinition( - prompt_template=template, llm={ "provider": llm_provider, "model": model("llm", llm_provider), diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py index 5a2ef9a7..442512e6 100644 --- a/src/fi/alk/harness/run/tools.py +++ b/src/fi/alk/harness/run/tools.py @@ -30,7 +30,6 @@ from ..catalogue import load_catalogue from ..config import ARTIFACTS_ROOT from ..scenario_tools import load_scenarios -from ..simulator import load_simulator_prompt from ..tools import schema from ..world.snapshot import require_source_implementation from .call import CASE, place_the_call @@ -465,13 +464,6 @@ def placed() -> tuple[LiveRun, str, list[str], str]: # how a simulated caller behaves is not decided in two places. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - # The caller prompt is a template the harness fills, never prose it composes, - # so the generated template travels to the call with everything else. - _template = load_simulator_prompt(Path(world_root)) if world_root else "" - if _template.strip(): - os.environ["HARNESS_SIMULATOR_PROMPT"] = _template - else: - os.environ.pop("HARNESS_SIMULATOR_PROMPT", None) os.environ["HARNESS_OUTCOME"] = scenario.tests os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index c71436bd..d296ec3b 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -18,7 +18,7 @@ import re from collections import Counter from math import ceil -from typing import Any +from typing import Any, ClassVar from pydantic import BaseModel, Field @@ -203,10 +203,20 @@ class Scenario(BaseModel): # suite covers both conditions rather than testing only callers in quiet rooms. background_noise: bool | str = Field(default_factory=lambda: random.choice((True, False))) + # Slots the caller filled by the run rather than by the scenario. Listed so a template that + # uses one is not rejected as unfillable at write time. + RUNTIME_SLOTS: ClassVar[tuple[str, ...]] = ("channel", "situation") + def slots(self) -> dict[str, str]: """Every value this scenario offers the simulator prompt.""" persona = {"persona": self.persona.format_persona()} if self.persona else {} - return {"instruction": self.instruction, **self.variables, **persona} + runtime = {name: "" for name in self.RUNTIME_SLOTS} + return { + "instruction": self.instruction, + **runtime, + **self.variables, + **persona, + } def validate_scenario( From ab9076aac3b96ed0700b7dd9082dbfb1fa777204 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:28:14 +0530 Subject: [PATCH 15/41] feat(scenarios): require the instruction to state an objective and carry what the caller needs --- src/fi/alk/harness/scenario_tools.py | 9 +++++++-- src/fi/alk/harness/skills/write-scenarios/SKILL.md | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 2e83355f..bfaa1a1e 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -448,8 +448,13 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: }, "instruction": { "type": "string", - "description": "The task, written to the person the agent is serving. For a " - "conversational agent this fills the simulator prompt's slot.", + "description": "What this person is trying to achieve, written to them. " + "State the objective first, in their own terms, so they pursue it rather " + "than narrate a situation: 'Get the cancellation fee refunded', not 'You " + "were charged a fee'. Then give them everything they need to hold the " + "conversation without inventing anything: the facts they know, the values " + "they can be asked for, what they will only say once asked, and what would " + "count as done. Every value real and read out of the world.", }, "persona": { "type": "object", diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index fec31c85..545af69f 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -22,7 +22,8 @@ name short identifier; it becomes this scenario's folder use_case which of the agent's use cases this belongs to branch what makes this one different from its siblings in that use case tests one line: what this scenario is trying to find out -instruction the task, written to the person the agent is serving +instruction what this person is trying to achieve, written to them, plus everything + they need to pursue it without inventing anything persona who that person is: identity, communication style, languages/accent and characteristics setup_code Python: def setup(world) — what this scenario changes first ready_code Python: def ready(world) — is the world ready for this scenario @@ -110,6 +111,8 @@ not exist, and no lookup will ever find them. **Possessing and volunteering are separate.** Whether the person offers a value unprompted is the scenario's business. Whether they have it at all is not optional. +**Write the instruction as an objective, not a situation.** A caller who is told what happened narrates it; a caller who is told what they want pursues it. Open with the goal in their own words ("Get the cancellation fee refunded"), then give them the facts they hold, the values they can be asked for, what they will only say once asked for it, and what would count as done. Every value read out of the world, never invented. + **Use persona deliberately.** An accent, personality or characteristic belongs in `persona` only when it changes the conversational risk being exercised. A rude customer is a different scenario from a polite one only if the agent must handle that difference. Persona never contains the From 1e2fc2d4cca3707ba1a338fc775a0d9796cb9e59 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:30:32 +0530 Subject: [PATCH 16/41] chore(simulator): drop the prompt template override nothing sets --- src/fi/simulate/agent/definition.py | 8 -------- src/fi/simulate/simulation/engines/livekit.py | 5 ----- 2 files changed, 13 deletions(-) diff --git a/src/fi/simulate/agent/definition.py b/src/fi/simulate/agent/definition.py index 73b9a71c..ffc32630 100644 --- a/src/fi/simulate/agent/definition.py +++ b/src/fi/simulate/agent/definition.py @@ -459,14 +459,6 @@ class SimulatorAgentDefinition(BaseModel): "It never replaces persona, situation, or outcome instructions." ), ) - prompt_template: Optional[str] = Field( - None, - description=( - "The caller prompt template, with {{persona}} and any other slots the run supplies. " - "Filled rather than composed, so the prompt is authored once and not rebuilt in code. " - "Falls back to the shipped default when absent." - ), - ) llm: LLMConfig = Field( default_factory=lambda: LLMConfig(model="gpt-4o-mini", temperature=0.6) diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index d85852b5..f734bbf8 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -1643,11 +1643,6 @@ async def _create_customer_agent( default_language=( simulator.stt.language if simulator is not None else None ), - template=( - getattr(simulator, "prompt_template", None) - if simulator is not None - else None - ), variables={"instruction": persona.situation or ""}, ) if simulator is None: From 4331f68dc33d2afdd31aa137da8a4e92129cf8bf Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:39:44 +0530 Subject: [PATCH 17/41] fix(simulator): close background audio on the caller agent and route every offered language --- src/fi/alk/harness/persona_guides.py | 12 +++++++++++- src/fi/alk/harness/run/sdk_voice.py | 4 ++-- src/fi/simulate/simulation/engines/livekit.py | 8 +++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py index b2edf43b..d95ff696 100644 --- a/src/fi/alk/harness/persona_guides.py +++ b/src/fi/alk/harness/persona_guides.py @@ -169,9 +169,19 @@ def vocabulary() -> dict[str, list[str]]: if values: by_class[node.name] = values - return { + found = { field: by_class[cls] for field, cls in FIELDS.items() if by_class.get(cls) } + if not found: + # The file parsed but held none of the classes we key on, so it is the wrong file or the + # classes moved. Silently returning nothing would drop every persona constraint at once. + logger.warning( + "persona vocabulary at %s defines none of %s; using the bundled copy", + path, + ", ".join(sorted(set(FIELDS.values()))), + ) + return _bundled_vocabulary() + return found diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 1250c3cd..f58bd9ad 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -52,7 +52,7 @@ def _json_env(name: str, default): # falls back to English, which is what the platform does too. _LANGUAGE_CODES: dict[str, str] = { "ar": "ar", "ar-sa": "ar", "arabic": "ar", "bg": "bg", - "bulgarian": "bg", "ca": "ca", "catalan": "ca", "chinese": "zh", + "bulgarian": "bg", "ca": "ca", "catalan": "ca", "chinese": "zh", "chinese simplified": "zh", "chinese traditional": "zh-TW", "chinese (cantonese, traditional)": "zh-HK", "chinese (mandarin, simplified)": "zh", "chinese (mandarin, traditional)": "zh-TW", "cs": "cs", "czech": "cs", "da": "da", "da-dk": "da", "danish": "da", "de": "de", "de-ch": "de-CH", "dutch": "nl", "el": "el", @@ -155,7 +155,7 @@ def _persona_stt_language() -> str: "punjabi": "pa", "gujarati": "gu", } _CARTESIA_LANGUAGE_TO_LANG: dict[str, str] = { - "english": "en", "hinglish": "hi", "spanish": "es", "hindi": "hi", "german": "de", + "english": "en", "chinese simplified": "zh", "chinese traditional": "zh", "hinglish": "hi", "spanish": "es", "hindi": "hi", "german": "de", "french": "fr", "italian": "it", "polish": "pl", "russian": "ru", "portuguese": "pt", "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", "turkish": "tr", "swedish": "sv", "dutch": "nl", "norwegian": "no", "telugu": "te", "kannada": "kn", diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index f734bbf8..f64f993c 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -1386,7 +1386,13 @@ def on_target_transcription( details={"exception_type": type(exc).__name__}, ) finally: - await self._stop_background_audio() + # The ambience belongs to the caller agent, not the engine. Guarded because teardown + # must never be the reason a case fails. + if customer_agent is not None: + try: + await customer_agent._stop_background_audio() + except Exception: + logger.warning("background audio not closed cleanly", exc_info=True) if target_transcription_handler_registered: room.unregister_text_stream_handler(TOPIC_TRANSCRIPTION) pending_target_transcriptions.clear() From 82bcf76fec9127eec67bc6d09c69c8eaed5d5100 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 15:44:32 +0530 Subject: [PATCH 18/41] feat(simulator): give the caller delivery cues when Cartesia is the voice --- src/fi/simulate/simulation/engines/livekit.py | 5 +++ src/fi/simulate/simulation/voice_prompt.py | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index f64f993c..5b3d46f3 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -1650,6 +1650,11 @@ async def _create_customer_agent( simulator.stt.language if simulator is not None else None ), variables={"instruction": persona.situation or ""}, + # Delivery cues are Cartesia only. Passing the provider here rather than reading it + # inside the prompt keeps the decision where the provider is actually known. + tts_provider=( + simulator.tts.provider if simulator is not None else None + ), ) if simulator is None: voice_provider = os.environ.get( diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index c405c985..cd68f795 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -39,6 +39,32 @@ } +# Delivery cues Cartesia renders and every other engine speaks aloud as words. Added only when +# Cartesia is definitely the voice, because Deepgram's Aura neither renders nor strips them, so +# a caller on Aura would literally say "left bracket laughter right bracket". +# +# Deliberately narrow. Cartesia documents five SSML tags and one nonverbalism, but `` and +# `` carry a decimal that a token stream can split ("1", ".", "0"), which makes the tag +# be read out, and Cartesia advises against shifting `` mid generation. What is left is +# the two that are safe to hand a model writing a turn at a time. +CARTESIA_DELIVERY_CUES = """# HOW YOU SOUND + +Two cues shape delivery. They are never spoken as words. Use them sparingly, and only where a +real person would. + +- [laughter] produces a real laugh. Write it inline: "No, [laughter] you're kidding." + At most once every few turns, and never to open one. +- is a fixed silence. Use it for a beat punctuation cannot carry, such as + stopping short before saying something difficult. One per turn at most. + +Write both exactly as shown. Do not invent others: no , no [laughs], no [sighs], no +*sighs*, no (angrily), no emotion labels. Anything not on this list is read aloud and ruins the +call. + +Everything else is carried by the words: what you repeat, where you interrupt yourself, how +short your sentences get when you are annoyed.""" + + def _first(value: object) -> str: if isinstance(value, Mapping): value = next(iter(value.values()), "") @@ -342,6 +368,7 @@ def render_simulator_prompt( agent_name: str | None = None, additional_instructions: str | None = None, default_language: str | None = None, + tts_provider: str | None = None, ) -> str: """Fill a caller-prompt template, the way the platform fills its own. @@ -382,6 +409,8 @@ def fill(match: "re.Match[str]") -> str: prompt = re.sub(r"Currently,\s*[.]", "", prompt) prompt = re.sub(r"[ \t]{2,}", " ", prompt).strip() + if tts_provider and tts_provider.strip().lower() == "cartesia": + prompt += "\n\n" + CARTESIA_DELIVERY_CUES if additional_instructions and additional_instructions.strip(): prompt += ( "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" @@ -411,6 +440,7 @@ def build_voice_simulator_prompt( default_language: str | None = None, template: str | None = None, variables: Mapping[str, Any] | None = None, + tts_provider: str | None = None, ) -> str: """The caller prompt for one simulated customer. @@ -425,6 +455,7 @@ def build_voice_simulator_prompt( agent_name=agent_name, additional_instructions=additional_instructions, default_language=default_language, + tts_provider=tts_provider, ) @@ -434,6 +465,7 @@ def build_voice_simulator_prompt( "VOICE_PERSONALITY_GUIDES", "append_voice_execution_rules", "DEFAULT_SIMULATOR_TEMPLATE", + "CARTESIA_DELIVERY_CUES", "build_voice_simulator_prompt", "render_simulator_prompt", "format_voice_persona", From a84d409ddf29481a7ed1570fd721d4cecf6f3b15 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 16:29:40 +0530 Subject: [PATCH 19/41] chore(harness): drop the persona guidance reader that nothing calls --- src/fi/alk/harness/persona_guides.py | 104 +++------------------------ 1 file changed, 11 insertions(+), 93 deletions(-) diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py index d95ff696..70cf780e 100644 --- a/src/fi/alk/harness/persona_guides.py +++ b/src/fi/alk/harness/persona_guides.py @@ -1,110 +1,28 @@ -"""The behaviour guidance the platform already uses for a simulated caller. +"""The persona values the platform understands. -A persona profile names what somebody is like: impatient and direct, cautious and skeptical. It -does not say how that should sound turn by turn, and a model handed only the label improvises -one, which is how "in a hurry" became a caller who says it every turn instead of a caller who -cuts in once and accepts the first workable answer. +A persona field is only useful if the platform recognises what is in it: an accent it knows +selects a voice, a personality it knows attaches a sentence of behaviour guidance. A value +written in words of its own renders fine and then does nothing, which is how a suite ends up +with callers who all behave the same. -The platform solved that with lookup tables mapping each value to a sentence of guidance, and -voice simulation has run on them for months. They are read from there rather than restated here, -because two copies of the same wording drift and then a caller behaves one way on the platform -and another way through the harness, for reasons nobody can see. - -Read, not imported: the tables live inside a Django app this package cannot import, but they are -plain literals, so they are parsed out of the file. Absent, every lookup answers with nothing and -a persona still renders, one without guidance, never a crash. +The values are read from the platform's own model when it is mounted, and from the copy carried +with the harness when it is not, so a writer is always offered real ones. The behaviour guidance +itself lives with the prompt builder, next to the code that applies it. """ from __future__ import annotations import ast -import logging import json +import logging import os from functools import lru_cache from pathlib import Path -# Where the platform's tables are mounted. Colon-separated so voice and chat guides can both be -# offered; the first file defining a table wins, so voice takes precedence when both are present. -GUIDES_ENV = "HARNESS_PERSONA_GUIDES" - -WANTED = ( - "VOICE_PERSONALITY_GUIDES", - "VOICE_COMMUNICATION_STYLE_GUIDES", - "CHAT_PERSONALITY_GUIDES", - "CHAT_COMMUNICATION_STYLE_GUIDES", - "CHAT_TONE_GUIDES", - "CHAT_VERBOSITY_GUIDES", -) - - -def _tables_in(path: Path) -> dict[str, dict[str, str]]: - """Every guidance table defined in one file, by name. - - Parsed rather than executed. The file sits in an app with imports this process cannot - satisfy, and running it to read a dictionary would fail for reasons that have nothing to do - with the dictionary. - """ - found: dict[str, dict[str, str]] = {} - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, SyntaxError): - return found - for node in tree.body: - targets = ( - [node.target] if isinstance(node, ast.AnnAssign) else getattr(node, "targets", []) - ) - for target in targets: - name = getattr(target, "id", "") - if name not in WANTED or node.value is None: - continue - try: - value = ast.literal_eval(node.value) - except ValueError: - continue - if isinstance(value, dict) and value: - found[name] = {str(k).lower(): str(v) for k, v in value.items()} - return found - - -@lru_cache(maxsize=1) -def guides() -> dict[str, dict[str, str]]: - """Every table the platform offers this harness, merged.""" - merged: dict[str, dict[str, str]] = {} - for raw in (os.environ.get(GUIDES_ENV) or "").split(":"): - if not raw.strip(): - continue - for name, table in _tables_in(Path(raw.strip())).items(): - merged.setdefault(name, table) - return merged - - -def guidance_for(kind: str, value: str, *, voice: bool = True) -> str: - """The platform's sentence for one persona value, or nothing. - - ``kind`` is ``personality``, ``communication_style``, ``tone`` or ``verbosity``. Voice tables - are preferred for a spoken call and the chat table is the fallback, because the two describe - the same disposition and only one of them is written for speech. - """ - if not value.strip(): - return "" - tables = guides() - order = ("VOICE", "CHAT") if voice else ("CHAT", "VOICE") - for prefix in order: - table = tables.get(f"{prefix}_{kind.upper()}_GUIDES") or {} - found = table.get(value.strip().lower()) - if found: - return found - return "" - - -def available() -> bool: - """Whether any guidance was found, so a build can say so rather than silently omitting it.""" - return bool(guides()) - - logger = logging.getLogger(__name__) +# Where the platform's tables are mounted. Colon-separated so voice and chat guides can both be +# offered; the first file defining a table wins, so voice takes precedence when both are present. # Where the platform's persona model is mounted, for the values it accepts. VOCABULARY_ENV = "HARNESS_PERSONA_VOCABULARY" From 102050ffdefb89d60e71949f1a64527116829660 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 16:31:28 +0530 Subject: [PATCH 20/41] fix(harness): let a slice finish before the stage is judged idle --- src/fi/alk/harness/session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index af652abd..eb5b00bd 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -38,7 +38,9 @@ # remain alive forever after a dropped upstream stream, though, which previously left a hosted # job looking healthy while making no progress. Bound *inactivity*, not total stage duration: # long scenario suites remain valid as long as they keep producing observable work. -STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "180")) +# Writers run in their own sessions, so the session that spawned them sits silent while they +# work. Three minutes is shorter than a slice takes, and the suite was being killed mid write. +STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "600")) STAGE_IDLE_RETRIES = int(os.getenv("ALK_STAGE_IDLE_RETRIES", "1")) From 813597c58a61d533919d1b222c699decb8330f00 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 17:16:57 +0530 Subject: [PATCH 21/41] chore(harness): run grading on the same model as everything else --- src/fi/alk/harness/run/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/run/models.py b/src/fi/alk/harness/run/models.py index 3134685e..38563f85 100644 --- a/src/fi/alk/harness/run/models.py +++ b/src/fi/alk/harness/run/models.py @@ -30,9 +30,10 @@ # this is the setting worth revisiting first once the target can be handed to ALK. AGENT = "claude-sonnet-4-6" USER = "claude-sonnet-4-6" -# Kept separate and stronger. A judged sub-goal is the one place a cheap wrong answer is -# expensive: it decides a pass, it runs once per scenario, and nobody re-reads it. -JUDGE = "claude-opus-4-7" +# One model for every role. A judged sub-goal was kept on a stronger model, but a run that mixes +# tiers is slower and harder to reason about, and the checks that decide a pass are code rather +# than judgement wherever they can be. Override with ALK_JUDGE_MODEL when a run needs it. +JUDGE = "claude-sonnet-4-6" def for_roles(override: str | None = None) -> dict[str, str]: From 2a048e58a1525f280f3a60129ee9361a9dc827a7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 17:18:51 +0530 Subject: [PATCH 22/41] fix(harness): size the idle bound for a suite that writes in one tool call --- src/fi/alk/harness/session.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index eb5b00bd..be52b31d 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -38,9 +38,12 @@ # remain alive forever after a dropped upstream stream, though, which previously left a hosted # job looking healthy while making no progress. Bound *inactivity*, not total stage duration: # long scenario suites remain valid as long as they keep producing observable work. -# Writers run in their own sessions, so the session that spawned them sits silent while they -# work. Three minutes is shorter than a slice takes, and the suite was being killed mid write. -STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "600")) +# The bound exists to catch a wedged stream, but it measures silence from the session rather than +# from the work. Writing a suite is one tool call that runs every writer inside it, so the session +# is legitimately silent for as long as generation takes and a bound shorter than that kills a +# healthy run. Sized for the longest suite rather than the commonest one; a genuinely wedged +# stage still fails, just later. +STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "2700")) STAGE_IDLE_RETRIES = int(os.getenv("ALK_STAGE_IDLE_RETRIES", "1")) From 67a91d604b92f582191d6be1ce7a9563cf5e3407 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 17:24:45 +0530 Subject: [PATCH 23/41] fix(harness): pin every model the session can reach to the one the run asked for --- src/fi/alk/harness/config.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 9cbb35ba..2a5ebbe0 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -93,10 +93,20 @@ def provider_env(model: str | None = None) -> dict[str, str]: Claude Code resolves the GCP project from ``GOOGLE_CLOUD_PROJECT``, the credential file, or the active gcloud configuration, in that order, so an unset project id is not an error here. """ + # Every model a session can reach is pinned to the same one. Naming only the main model + # leaves the sub-agent and fast-path settings to the CLI's own preference, and a suite written + # by twenty writers then runs on whatever that preference happens to be rather than on the + # model the run asked for. + chosen = chosen_model(model) env = { "CLAUDE_CODE_USE_VERTEX": "1", "CLOUD_ML_REGION": os.environ.get("CLOUD_ML_REGION", "global"), - "ANTHROPIC_MODEL": chosen_model(model), + "ANTHROPIC_MODEL": chosen, + "ANTHROPIC_DEFAULT_SONNET_MODEL": chosen, + "ANTHROPIC_DEFAULT_OPUS_MODEL": chosen, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": chosen, + "ANTHROPIC_SMALL_FAST_MODEL": chosen, + "CLAUDE_CODE_SUBAGENT_MODEL": chosen, } for passthrough in ( "ANTHROPIC_VERTEX_PROJECT_ID", From 42e208120dc43e021c8a00202f56eb87ff2bc212 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 17:28:48 +0530 Subject: [PATCH 24/41] fix(scenarios): write a suite one scenario at a time unless fan-out is asked for --- src/fi/alk/harness/scenario_tools.py | 34 +++++++++++++++++-- src/fi/alk/harness/scenarios.py | 12 ++++--- .../harness/skills/write-scenarios/SKILL.md | 9 ++--- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index bfaa1a1e..257fe433 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any @@ -50,6 +51,17 @@ def _err(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}], "is_error": True} + +def parallel_suites() -> bool: + """Whether a suite is written by several writers at once. + + Off by default. Writing one scenario at a time is slower but is the path the base branch runs + on, and a suite that is written slowly is worth more than one that is not written at all. + Set HARNESS_PARALLEL_SCENARIOS=1 to fan out instead. + """ + return os.environ.get("HARNESS_PARALLEL_SCENARIOS", "").strip() == "1" + + def persona_field(name: str) -> dict[str, Any]: """The schema for one persona field, carrying the platform's own values where it has them. @@ -855,12 +867,18 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ] # Only the session a person is talking to may fan out. A writer that is itself one slice # of a fan-out calling this would split its own slice again, and so on. - + ([generate_suite, save_scenarios] if can_save else []), + + ( + [generate_suite, save_scenarios] + if can_save and parallel_suites() + else [save_scenarios] + if can_save + else [] + ), ) return server, kept -TOOL_NAMES = ( +_ALWAYS = ( "inspect_world", "inspect_scenario", "try_calls", @@ -872,11 +890,21 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: "fix_tool", "aim_for", "drop_scenario", - "generate_suite", "save_scenarios", ) +def tool_names() -> tuple[str, ...]: + """The tools a saving session publishes, which depends on how a suite is written.""" + if parallel_suites(): + return (*_ALWAYS[:-1], "generate_suite", "save_scenarios") + return _ALWAYS + + +# Kept as a name because callers import it; it reflects the surface for this process. +TOOL_NAMES = tool_names() + + def world_summary(world_root: Path) -> str: """What is in the built environment, for grounding the writer before it asks.""" world = restore(world_root) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index bb5fdd35..a06002d3 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -34,6 +34,7 @@ from .contract import AgentContract from .scenario import Scenario from .scenario_tools import ( + parallel_suites, SCENARIO_SERVER, TOOL_NAMES, load_scenarios, @@ -145,10 +146,13 @@ def opening(contract: AgentContract, wanted: int = 10, existing: int = 0) -> str "across several turns. If a proof says an intended check is vacuous or broken, repair " "that named sub-goal with add_sub_goal and resubmit. Never evade a gate by deleting a " "check for behavior the scenario still claims to test. Then save_scenarios." - "\n\nFor a suite rather than one scenario, say briefly how you are splitting it " - "across the agent's use cases and then write it with generate_suite in the same turn: " - "it runs a writer per use case at the same time and saves what they prove, where " - "writing this many one at a time would run out of turns before finishing." + + ( + "\n\nFor a suite rather than one scenario, say briefly how you are splitting it " + "across the agent's use cases and then write it with generate_suite in the same " + "turn: it runs a writer per use case at the same time and saves what they prove." + if parallel_suites() + else "" + ) ) diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 545af69f..567177f2 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -162,12 +162,6 @@ world. Keep that plan concise and continue immediately unless the person explicitly asked to review it. -**Then write the suite with `generate_suite`, not one scenario at a time.** It splits the work -across the agent's use cases and runs several writers at once, each proving its own scenarios -through the same three gates. Writing a suite yourself with `submit_scenario` costs about three -turns per scenario against one budget, so a request for twenty or fifty runs out long before it -finishes, and what does get written is lost because nothing was saved. - **Pass your plan to it.** The tool takes the split as an argument, and you have just read the world and know which use cases have something in them; it is the part of this only you can do. Each slice names its use case, @@ -399,7 +393,8 @@ hides the problem and everything built afterwards inherits it. 1. `inspect_world` with no table, then look at the ones that matter. Read the sub-goals already defined. 2. Read the agent's hard rules. Each one is a branch waiting to be written. -3. For a suite, say how you are splitting it and then `generate_suite` with the count. It +3. For a suite, say how you are splitting it across the agent's use cases, then write and + submit them one at a time. A large ask comes back a batch at a time rather than all at once. writes the whole thing and saves it, and you report what came back. 4. For a single scenario: work out the solution, `try_calls` it with your `setup_code`, then `submit_scenario`. From 8a2181d96a3ade1c2c0d763a34340d92dec48c54 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 22:03:33 +0530 Subject: [PATCH 25/41] feat(scenario): carry the scenario_key and scenario_id a hosted scheduler reads, and decide background noise from the name --- src/fi/alk/harness/scenario.py | 42 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index d296ec3b..ef5956b8 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -13,14 +13,14 @@ from __future__ import annotations import ast +import hashlib import json -import random import re from collections import Counter from math import ceil from typing import Any, ClassVar -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from .catalogue import Catalogue from .simulator import variables_in @@ -149,10 +149,31 @@ def format_persona(self) -> str: return "\n".join(parts) +def _slug(name: str) -> str: + """An ASCII key for ``name``, safe to send as a header value. + + Falls back to a digest rather than an empty string: an empty key would collapse every + scenario in a job onto one idempotency key on the receiving side. + """ + cleaned = re.sub(r"[^a-z0-9]+", "-", (name or "").strip().lower()).strip("-") + return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12] + + +def _decided_by(name: str) -> bool: + """Whether this scenario is noisy, decided by its name so a rerun decides the same.""" + return hashlib.sha256((name or "").encode()).digest()[0] % 2 == 0 + + class Scenario(BaseModel): """One test: what changes, what is asked, what a correct agent does, what must hold.""" name: str + # How this scenario is identified on the wire. Derived from ``name``, which is already unique + # across a suite and already a slug because it is the folder name. It ships as a header, so + # anything outside ASCII is dropped and an empty result falls back to a digest. + scenario_key: str = "" + # Assigned by the platform when the scenario is pre-allocated. Never written here. + scenario_id: str = "" use_case: str = "" # What makes this row different from its siblings in the same use case. Coverage is counted # on the pair, so a use case can carry many scenarios without any reading as a duplicate. @@ -198,15 +219,24 @@ class Scenario(BaseModel): max_turns: int = 10 - # Where this call is made from. True asks for noise and leaves the place to the fixture; a - # string names it ("street", "vehicle", "retail"). Random when the writer does not say, so a - # suite covers both conditions rather than testing only callers in quiet rooms. - background_noise: bool | str = Field(default_factory=lambda: random.choice((True, False))) + # Where this call is made from. A string names the place ("street", "vehicle", "retail"), and + # True asks for noise while leaving the place to the fixture. Left unset it is decided from + # the name, so a suite still covers both conditions but the same suite decides the same way + # twice; a coin flip here made a seeded run unreproducible. + background_noise: bool | str = "" # Slots the caller filled by the run rather than by the scenario. Listed so a template that # uses one is not rejected as unfillable at write time. RUNTIME_SLOTS: ClassVar[tuple[str, ...]] = ("channel", "situation") + @model_validator(mode="after") + def _identify(self) -> "Scenario": + if not self.scenario_key: + self.scenario_key = _slug(self.name) + if self.background_noise == "": + self.background_noise = _decided_by(self.name) + return self + def slots(self) -> dict[str, str]: """Every value this scenario offers the simulator prompt.""" persona = {"persona": self.persona.format_persona()} if self.persona else {} From ae982fb7279d3bc3e7889f52cf1279b2726c27e8 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 22:03:34 +0530 Subject: [PATCH 26/41] fix(scenarios): stop the instruction telling the caller what the agent will do --- src/fi/alk/harness/scenario_tools.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 257fe433..ef4142aa 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -454,9 +454,10 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: }, "background_noise": { "type": "string", - "description": "Where the caller is phoning from, when it is part of the " - "test: street, transit, vehicle, outdoors, retail, office or home. Leave it " - "out unless the place matters.", + "description": "Where the caller is phoning from: street, transit, vehicle, " + "outdoors, retail, office or home. Name it whenever the instruction implies " + "somewhere, a caller leaving a hotel or standing on a street is not in a " + "quiet room. Left out, it is decided from the scenario name.", }, "instruction": { "type": "string", @@ -465,8 +466,12 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "than narrate a situation: 'Get the cancellation fee refunded', not 'You " "were charged a fee'. Then give them everything they need to hold the " "conversation without inventing anything: the facts they know, the values " - "they can be asked for, what they will only say once asked, and what would " - "count as done. Every value real and read out of the world.", + "they can be asked for, and what they will only say once asked. Every value " + "real and read out of the world. Write only what this person knows before " + "the call. Never tell them what the agent will do, ask for, or disclose: " + "that is what the scenario is testing, and a caller told to expect it will " + "play along whether or not it happens. 'You want the fee waived' is theirs; " + "'the agent will offer you a refund' is not.", }, "persona": { "type": "object", From 3d612b0d37a07369670ced19b4f371ddc219cded Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 22:03:34 +0530 Subject: [PATCH 27/41] fix(grade): read the speaker off a transcript line instead of leaving it in the text --- src/fi/alk/harness/run/simulation.py | 32 ++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index 6f0bd76b..f74ed1e8 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -32,13 +32,16 @@ from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from ..contract import AgentContract from ..scenario import Scenario from ..world.runtime import Call from .grade import Judgement, Result +if TYPE_CHECKING: + from .conversation import Exchange + RUNS = "runs" RUN = "run.json" RESULT = "result.json" @@ -501,6 +504,23 @@ def _calls_of(calls: Any) -> list[dict[str, Any]]: ] +def _said(line: str) -> Exchange: + """One transcript line as a turn, with its role read off rather than left in the text. + + The line arrives already labelled ("assistant: ..."). Keeping that label in the text made the + judge read ``agent: assistant: ...``, two speakers deep for every turn. + """ + from .conversation import Exchange + + role, _, text = line.partition(":") + named = role.strip().lower() + if named in ("assistant", "agent"): + return Exchange("agent", text.strip()) + if named in ("user", "customer"): + return Exchange("customer", text.strip()) + return Exchange("customer", line.strip()) + + async def _spoken_to( scenario: Scenario, contract: AgentContract, @@ -526,7 +546,7 @@ async def _spoken_to( from ..catalogue import load_catalogue from .call import place_the_call - from .conversation import Exchange, Transcript + from .conversation import Transcript from .evidence import measured, newest_report, spoken_times, tracks_in from .grade import ( checkpoints, @@ -716,13 +736,7 @@ def placed_once() -> tuple[int, dict[str, Any], str]: # happened is that one check passed and the other was never asked, which reads as the agent # half-failing rather than as the suite not having looked. spoken_transcript = Transcript( - exchanges=[ - Exchange( - "agent" if line.lower().startswith("assistant") else "customer", line - ) - for line in spoken.splitlines() - if line.strip() - ], + exchanges=[_said(line) for line in spoken.splitlines() if line.strip()], calls=list(world.calls), ended=str((case.get("metadata") or {}).get("status") or "finished"), ) From 21f0ae68e0190bf3e325f099ab27bd4d5994ebb1 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 22:03:34 +0530 Subject: [PATCH 28/41] test(harness): cover scenario identity, deterministic noise and transcript speakers --- tests/test_harness.py | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/test_harness.py b/tests/test_harness.py index 02f646e1..5c85e112 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6265,3 +6265,54 @@ def result(self, call_execution_id, payload): ) assert api.started_with == ["sid-a", "sid-b"] assert reported.calls == {"a": "ce-a", "b": "ce-b"} + + +def test_scenario_carries_the_identity_the_hosted_scheduler_reads(): + """The guest scheduler reads ``scenario_key`` and ``scenario_id`` off every scenario. + + Both are plain attributes rather than optional extras: a pydantic model raises AttributeError + for a field it never declared, so a missing one fails at the first read rather than degrading. + """ + one = Scenario(name="dana-books-uberx-saved-card") + assert one.scenario_key == "dana-books-uberx-saved-card" + # Assigned by the platform at pre-allocation, never written at generation. + assert one.scenario_id == "" + + +@pytest.mark.parametrize( + "name,expected", + [ + ("Dana Books - Café", "dana-books-caf"), + ("already-a-slug", "already-a-slug"), + (" Spaced Out ", "spaced-out"), + ], +) +def test_scenario_key_is_ascii_and_slugged(name, expected): + assert Scenario(name=name).scenario_key == expected + + +def test_scenario_key_never_empties_onto_a_shared_idempotency_key(): + """A name with nothing ASCII in it still gets its own key rather than an empty one.""" + keys = {Scenario(name=name).scenario_key for name in ("日本語", "中文", "한국어")} + assert all(key.startswith("scenario-") for key in keys) + assert len(keys) == 3 + + +def test_background_noise_is_decided_the_same_way_twice(): + """A coin flip here made a seeded run unreproducible; the name decides it instead.""" + assert Scenario(name="a-b-c").background_noise == Scenario(name="a-b-c").background_noise + assert Scenario(name="x", background_noise="street").background_noise == "street" + + +def test_a_transcript_line_keeps_one_speaker_not_two(): + """``agent: assistant: ...`` reached the judges as two speakers deep for every turn.""" + from fi.alk.harness.run.simulation import _said + + def said(line): + turn = _said(line) + return turn.speaker, turn.text + + assert said("assistant: Hi Dana.") == ("agent", "Hi Dana.") + assert said("user: I need a ride.") == ("customer", "I need a ride.") + # A colon inside speech is not a speaker label. + assert said("assistant: Call at 3:30 PM.") == ("agent", "Call at 3:30 PM.") From 64c918b8156891a286d8ae7d50901ad87530bce7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 03:06:31 +0530 Subject: [PATCH 29/41] fix(scenarios): keep the agent's expected moves out of the caller's instruction --- src/fi/alk/harness/scenario_tools.py | 13 +++++--- .../harness/skills/write-scenarios/SKILL.md | 31 +++++++++++++++++-- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index ef4142aa..803f0171 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -467,11 +467,14 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "were charged a fee'. Then give them everything they need to hold the " "conversation without inventing anything: the facts they know, the values " "they can be asked for, and what they will only say once asked. Every value " - "real and read out of the world. Write only what this person knows before " - "the call. Never tell them what the agent will do, ask for, or disclose: " - "that is what the scenario is testing, and a caller told to expect it will " - "play along whether or not it happens. 'You want the fee waived' is theirs; " - "'the agent will offer you a refund' is not.", + "real and read out of the world.\n" + "Write only what this person knows before the call starts. Never write what " + "the agent will do, in any phrasing: not what it will send, offer, ask for, " + "disclose or decide, and no closing line about what counts as done. Those " + "are the behaviours under test, and a person primed to expect them plays " + "along whether or not they happen, so the check passes on a conversation " + "that never earned it. Give them the value, the preference or the problem " + "they arrived with, and let the agent's handling of it be what is measured.", }, "persona": { "type": "object", diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 567177f2..b6e8d3f5 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -47,7 +47,7 @@ scenario that looks fine and measures nothing. | | What it is | What it must never contain | |---|---|---| -| **instruction** | what the person on the other side is living through | the answer, the checks, or facts they could not know | +| **instruction** | what the person on the other side is living through | the answer, the checks, facts they could not know, or anything the agent is expected to do | | **setup** | the world's condition | anything the person is supposed to say | | **checks** | the hidden pass or fail rules | anything the agent was told | @@ -111,7 +111,34 @@ not exist, and no lookup will ever find them. **Possessing and volunteering are separate.** Whether the person offers a value unprompted is the scenario's business. Whether they have it at all is not optional. -**Write the instruction as an objective, not a situation.** A caller who is told what happened narrates it; a caller who is told what they want pursues it. Open with the goal in their own words ("Get the cancellation fee refunded"), then give them the facts they hold, the values they can be asked for, what they will only say once asked for it, and what would count as done. Every value read out of the world, never invented. +**Write the instruction as an objective, not a situation.** A caller who is told what happened +narrates it; a caller who is told what they want pursues it. Open with the goal in their own words +("Get put right"), not with the history that led to it ("You were charged +"), then give them the facts they hold, the values they can be asked for, and what +they will only say once asked for it. Every value read out of the world, never invented. + +**Never tell the caller what the agent will do.** This is the single most common way a scenario +silently stops measuring anything. The agent's moves are what the scenario is testing, so a caller +who has been told to expect them will play along whether or not they happen, and the check passes +on a conversation that never earned it. Write only what this person knows before the call starts. + +``` +BAD The agent will tell you about . Accept it and say yes when + asked to confirm. + (the scenario is testing whether the agent discloses . A caller + primed to accept it agrees even when the agent never says it, so the run + reports a pass for behaviour that did not occur) + +GOOD You want . You will accept if there is one, but + you want to know before you agree to anything. + (the caller's own position. If the agent discloses, they accept; if it does + not, they ask, and the transcript records which happened) +``` + +The same rule covers every phrasing of it: "the agent will send you ", "they will offer +you ", "they should transfer you". Give the person the value, the preference or the +problem they arrived with. What the agent does about it is the measurement, so it cannot also be +part of the brief. **Use persona deliberately.** An accent, personality or characteristic belongs in `persona` only when it changes the conversational risk being exercised. A rude customer is a different scenario From 82adf925cb2d3d7b875a6db2cbed10297925b15f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 05:21:12 +0530 Subject: [PATCH 30/41] fix(simulate): give the caller countable conduct rules instead of one long paragraph --- src/fi/alk/harness/run/sdk_voice.py | 327 ++++++++++++++++++++++------ 1 file changed, 265 insertions(+), 62 deletions(-) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index f58bd9ad..9a274f0c 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -51,34 +51,104 @@ def _json_env(name: str, default): # so a persona resolves to the same language here as it does there. Anything unrecognised # falls back to English, which is what the platform does too. _LANGUAGE_CODES: dict[str, str] = { - "ar": "ar", "ar-sa": "ar", "arabic": "ar", "bg": "bg", - "bulgarian": "bg", "ca": "ca", "catalan": "ca", "chinese": "zh", "chinese simplified": "zh", "chinese traditional": "zh-TW", - "chinese (cantonese, traditional)": "zh-HK", "chinese (mandarin, simplified)": "zh", "chinese (mandarin, traditional)": "zh-TW", "cs": "cs", - "czech": "cs", "da": "da", "da-dk": "da", "danish": "da", - "de": "de", "de-ch": "de-CH", "dutch": "nl", "el": "el", - "en": "en-US", "en-au": "en-AU", "en-gb": "en-GB", "en-in": "en-IN", - "en-nz": "en-NZ", "en-us": "en-US", "english": "en-US", "es": "es", - "es-419": "es-419", "estonian": "et", "et": "et", "fi": "fi", - "finnish": "fi", "flemish": "nl-BE", "fr": "fr", "fr-ca": "fr-CA", - "french": "fr", "german": "de", "greek": "el", "hi": "hi", - "hindi": "hi", "hu": "hu", "hungarian": "hu", "id": "id", - "indonesian": "id", "it": "it", "italian": "it", "ja": "ja", - "japanese": "ja", "ko": "ko", "ko-kr": "ko", "korean": "ko", - "latvian": "lv", "lithuanian": "lt", "lt": "lt", "lv": "lv", - "malay": "ms", "ms": "ms", "nl": "nl", "nl-be": "nl-BE", - "no": "no", "norwegian": "no", "pl": "pl", "polish": "pl", - "portuguese": "pt", "pt": "pt", "pt-br": "pt-BR", "pt-pt": "pt-PT", - "ro": "ro", "romanian": "ro", "ru": "ru", "russian": "ru", - "sk": "sk", "slovak": "sk", "spanish": "es", "sv": "sv", - "sv-se": "sv", "swedish": "sv", "th": "th", "th-th": "th", - "thai": "th", "tr": "tr", "turkish": "tr", "uk": "uk", - "ukrainian": "uk", "vi": "vi", "vietnamese": "vi", "zh": "zh", - "zh-cn": "zh", "zh-hans": "zh", "zh-hant": "zh-TW", "zh-hk": "zh-HK", + "ar": "ar", + "ar-sa": "ar", + "arabic": "ar", + "bg": "bg", + "bulgarian": "bg", + "ca": "ca", + "catalan": "ca", + "chinese": "zh", + "chinese simplified": "zh", + "chinese traditional": "zh-TW", + "chinese (cantonese, traditional)": "zh-HK", + "chinese (mandarin, simplified)": "zh", + "chinese (mandarin, traditional)": "zh-TW", + "cs": "cs", + "czech": "cs", + "da": "da", + "da-dk": "da", + "danish": "da", + "de": "de", + "de-ch": "de-CH", + "dutch": "nl", + "el": "el", + "en": "en-US", + "en-au": "en-AU", + "en-gb": "en-GB", + "en-in": "en-IN", + "en-nz": "en-NZ", + "en-us": "en-US", + "english": "en-US", + "es": "es", + "es-419": "es-419", + "estonian": "et", + "et": "et", + "fi": "fi", + "finnish": "fi", + "flemish": "nl-BE", + "fr": "fr", + "fr-ca": "fr-CA", + "french": "fr", + "german": "de", + "greek": "el", + "hi": "hi", + "hindi": "hi", + "hu": "hu", + "hungarian": "hu", + "id": "id", + "indonesian": "id", + "it": "it", + "italian": "it", + "ja": "ja", + "japanese": "ja", + "ko": "ko", + "ko-kr": "ko", + "korean": "ko", + "latvian": "lv", + "lithuanian": "lt", + "lt": "lt", + "lv": "lv", + "malay": "ms", + "ms": "ms", + "nl": "nl", + "nl-be": "nl-BE", + "no": "no", + "norwegian": "no", + "pl": "pl", + "polish": "pl", + "portuguese": "pt", + "pt": "pt", + "pt-br": "pt-BR", + "pt-pt": "pt-PT", + "ro": "ro", + "romanian": "ro", + "ru": "ru", + "russian": "ru", + "sk": "sk", + "slovak": "sk", + "spanish": "es", + "sv": "sv", + "sv-se": "sv", + "swedish": "sv", + "th": "th", + "th-th": "th", + "thai": "th", + "tr": "tr", + "turkish": "tr", + "uk": "uk", + "ukrainian": "uk", + "vi": "vi", + "vietnamese": "vi", + "zh": "zh", + "zh-cn": "zh", + "zh-hans": "zh", + "zh-hant": "zh-TW", + "zh-hk": "zh-HK", "zh-tw": "zh-TW", } - def _normalised_language(raw: str) -> str: """The code the providers expect, from a language name or a region code. @@ -137,33 +207,147 @@ def _persona_stt_language() -> str: # present; otherwise the Deepgram aura path below is used unchanged. _CARTESIA_SUPPORTED_LANGS = frozenset( { - "en", "es", "hi", "de", "fr", "it", "pl", "ru", "pt", "ja", "ko", "zh", "tr", "sv", - "nl", "no", "te", "kn", "fi", "mr", "da", "bn", "sk", "uk", "el", "ta", "vi", "id", - "ro", "ka", "ml", "ms", "he", "bg", "th", "hu", "pa", "cs", "tl", "ar", "gu", "hr", + "en", + "es", + "hi", + "de", + "fr", + "it", + "pl", + "ru", + "pt", + "ja", + "ko", + "zh", + "tr", + "sv", + "nl", + "no", + "te", + "kn", + "fi", + "mr", + "da", + "bn", + "sk", + "uk", + "el", + "ta", + "vi", + "id", + "ro", + "ka", + "ml", + "ms", + "he", + "bg", + "th", + "hu", + "pa", + "cs", + "tl", + "ar", + "gu", + "hr", } ) _CARTESIA_ACCENT_TO_LANG: dict[str, str] = { - "spanish": "es", "south american": "es", "indian": "hi", "german": "de", "french": "fr", - "italian": "it", "polish": "pl", "russian": "ru", "portuguese": "pt", "brazilian": "pt", - "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", "turkish": "tr", - "swedish": "sv", "dutch": "nl", "norwegian": "no", "finnish": "fi", "danish": "da", - "slovak": "sk", "ukrainian": "uk", "greek": "el", "romanian": "ro", "georgian": "ka", - "bulgarian": "bg", "thai": "th", "hungarian": "hu", "czech": "cs", "croatian": "hr", - "vietnamese": "vi", "indonesian": "id", "malay": "ms", "malaysian": "ms", "tagalog": "tl", - "filipino": "tl", "arabic": "ar", "hebrew": "he", "israeli": "he", "telugu": "te", - "kannada": "kn", "marathi": "mr", "bengali": "bn", "tamil": "ta", "malayalam": "ml", - "punjabi": "pa", "gujarati": "gu", + "spanish": "es", + "south american": "es", + "indian": "hi", + "german": "de", + "french": "fr", + "italian": "it", + "polish": "pl", + "russian": "ru", + "portuguese": "pt", + "brazilian": "pt", + "japanese": "ja", + "korean": "ko", + "chinese": "zh", + "mandarin": "zh", + "turkish": "tr", + "swedish": "sv", + "dutch": "nl", + "norwegian": "no", + "finnish": "fi", + "danish": "da", + "slovak": "sk", + "ukrainian": "uk", + "greek": "el", + "romanian": "ro", + "georgian": "ka", + "bulgarian": "bg", + "thai": "th", + "hungarian": "hu", + "czech": "cs", + "croatian": "hr", + "vietnamese": "vi", + "indonesian": "id", + "malay": "ms", + "malaysian": "ms", + "tagalog": "tl", + "filipino": "tl", + "arabic": "ar", + "hebrew": "he", + "israeli": "he", + "telugu": "te", + "kannada": "kn", + "marathi": "mr", + "bengali": "bn", + "tamil": "ta", + "malayalam": "ml", + "punjabi": "pa", + "gujarati": "gu", } _CARTESIA_LANGUAGE_TO_LANG: dict[str, str] = { - "english": "en", "chinese simplified": "zh", "chinese traditional": "zh", "hinglish": "hi", "spanish": "es", "hindi": "hi", "german": "de", - "french": "fr", "italian": "it", "polish": "pl", "russian": "ru", "portuguese": "pt", - "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", "turkish": "tr", - "swedish": "sv", "dutch": "nl", "norwegian": "no", "telugu": "te", "kannada": "kn", - "finnish": "fi", "marathi": "mr", "danish": "da", "bengali": "bn", "slovak": "sk", - "ukrainian": "uk", "greek": "el", "tamil": "ta", "vietnamese": "vi", "indonesian": "id", - "romanian": "ro", "georgian": "ka", "malayalam": "ml", "malay": "ms", "hebrew": "he", - "bulgarian": "bg", "thai": "th", "hungarian": "hu", "punjabi": "pa", "czech": "cs", - "tagalog": "tl", "filipino": "tl", "arabic": "ar", "gujarati": "gu", "croatian": "hr", + "english": "en", + "chinese simplified": "zh", + "chinese traditional": "zh", + "hinglish": "hi", + "spanish": "es", + "hindi": "hi", + "german": "de", + "french": "fr", + "italian": "it", + "polish": "pl", + "russian": "ru", + "portuguese": "pt", + "japanese": "ja", + "korean": "ko", + "chinese": "zh", + "mandarin": "zh", + "turkish": "tr", + "swedish": "sv", + "dutch": "nl", + "norwegian": "no", + "telugu": "te", + "kannada": "kn", + "finnish": "fi", + "marathi": "mr", + "danish": "da", + "bengali": "bn", + "slovak": "sk", + "ukrainian": "uk", + "greek": "el", + "tamil": "ta", + "vietnamese": "vi", + "indonesian": "id", + "romanian": "ro", + "georgian": "ka", + "malayalam": "ml", + "malay": "ms", + "hebrew": "he", + "bulgarian": "bg", + "thai": "th", + "hungarian": "hu", + "punjabi": "pa", + "czech": "cs", + "tagalog": "tl", + "filipino": "tl", + "arabic": "ar", + "gujarati": "gu", + "croatian": "hr", } _CARTESIA_DEFAULT_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" @@ -223,14 +407,18 @@ def _cartesia_voice_for(persona: dict) -> str: ) if not voices: return _CARTESIA_DEFAULT_VOICE - index = sum(ord(character) for character in str(persona.get("name") or "")) % len(voices) + index = sum(ord(character) for character in str(persona.get("name") or "")) % len( + voices + ) return voices[index] def _voice_providers() -> tuple[str, str]: """The (stt, tts) providers for the caller. An explicit env override wins; otherwise Cartesia when its key is present (richer, multi-language voices), else Deepgram aura.""" - default = "cartesia" if os.environ.get("CARTESIA_API_KEY", "").strip() else "deepgram" + default = ( + "cartesia" if os.environ.get("CARTESIA_API_KEY", "").strip() else "deepgram" + ) stt = os.environ.get("SIMULATOR_STT_PROVIDER", "").strip() or default tts = os.environ.get("SIMULATOR_TTS_PROVIDER", "").strip() or default return stt, tts @@ -249,7 +437,11 @@ def _simulator() -> simulate.SimulatorAgentDefinition: ) defaults = { "llm": {"google": "gemini-2.5-flash", "openai": "gpt-4o-mini"}, - "stt": {"deepgram": stt_model or "nova-3", "cartesia": "ink-2", "google": "chirp_2"}, + "stt": { + "deepgram": stt_model or "nova-3", + "cartesia": "ink-2", + "google": "chirp_2", + }, "tts": { "deepgram": "aura-asteria-en", "cartesia": "sonic-3.5", @@ -280,19 +472,28 @@ def model(kind: str, provider: str) -> str: "model": model("tts", tts_provider), "voice": os.environ.get("SIMULATOR_TTS_VOICE", default_tts_voice), }, + # Written as separate numbered rules rather than one paragraph. These arrive late in a + # long prompt, and a rule buried mid-sentence there does not survive: a caller ignored the + # loop rule for four turns while it was the tail of a compound sentence. instructions=( - "Act as the customer described by the scenario. Speak naturally and briefly. " - "Use only the supplied facts and never invent account, address, payment, or " - "verification data. Do not volunteer private data: agree when asked whether a " - "verification code should be sent, and disclose the actual code only after the " - "agent says it was sent and explicitly asks you to read it. Answer repair questions " - "with the missing fact, not by restarting the request. Never repeat the same answer " - "more than twice. Wait for the agent to finish the task rather than ending as soon as " - "it asks to proceed: say yes and let it complete and confirm the outcome. But if the " - "agent gives essentially the same response two or three times without making progress, " - "do not keep looping: say once that it is not working and that you will try again " - "later, then end the call. Once the outcome is actually completed and confirmed, thank " - "the agent and end the call." + "Act as the customer described by the scenario. Speak naturally and briefly.\n" + "These rules override anything else when they conflict:\n" + "1. Use ONLY the facts you were given. Never invent an account detail, address, " + "payment state, or verification code.\n" + "2. If the agent asks about something you were given no fact for, say plainly that " + "you do not know or cannot tell. Never guess, and never claim something happened on " + "your end when you were not told it did.\n" + "3. Do not volunteer private data. Agree when asked whether a verification code " + "should be sent, and read the code out only after the agent says it was sent and " + "asks you for it.\n" + "4. Answer a repair question with the missing fact, not by restarting your request.\n" + "5. STOP AFTER THREE. Count the agent's replies. If three of them say essentially " + "the same thing without the task moving forward, do not try a fifth time and do not " + "rephrase the same point again. Say once that this is not working and you will try " + "later, then end the call.\n" + "6. Otherwise let the agent finish. Say yes when it asks to proceed and wait for it " + "to confirm the outcome rather than hanging up early.\n" + "7. Once the outcome is confirmed, thank the agent and end the call." ), allow_interruptions=True, ) @@ -327,7 +528,9 @@ def _aura_voice_for(persona: dict) -> str: _AURA_BY_ACCENT["american"], ) voices = bucket.get(gender) or next(iter(bucket.values())) - index = sum(ord(character) for character in str(persona.get("name") or "")) % len(voices) + index = sum(ord(character) for character in str(persona.get("name") or "")) % len( + voices + ) return voices[index] From 325db305d212922c6bfd8649004879efc5a3263e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 05:21:13 +0530 Subject: [PATCH 31/41] fix(scenarios): judge an instruction by whether the caller could say it out loud --- src/fi/alk/harness/scenario_tools.py | 9 ++++++-- .../harness/skills/write-scenarios/SKILL.md | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 803f0171..07112617 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -51,7 +51,6 @@ def _err(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}], "is_error": True} - def parallel_suites() -> bool: """Whether a suite is written by several writers at once. @@ -474,7 +473,13 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "are the behaviours under test, and a person primed to expect them plays " "along whether or not they happen, so the check passes on a conversation " "that never earned it. Give them the value, the preference or the problem " - "they arrived with, and let the agent's handling of it be what is measured.", + "they arrived with, and let the agent's handling of it be what is measured.\n" + "Test every sentence by asking whether this person could say it out loud. " + "They have never seen the agent's design, so a parenthetical explaining " + "where the agent should find a value fails that test just as much as a " + "sentence predicting what it will say. Worst of all is agreeing in advance " + "to something the agent has not done yet: that hands over a pass the " + "conversation never earned.", }, "persona": { "type": "object", diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index b6e8d3f5..c5ab0309 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -140,6 +140,28 @@ you ", "they should transfer you". Give the person the value, the pre problem they arrived with. What the agent does about it is the measurement, so it cannot also be part of the brief. +**The test that catches all of it: could this person say the sentence out loud?** The instruction +is read by someone who has never seen the agent's design and does not know how it works. So a +parenthetical explaining where the agent is supposed to find a value is not a smaller version of +the mistake, it is the same mistake in a quieter voice. + +``` +BAD Your : (the agent should find this from your ) + (the caller has no idea the agent has records, let alone which one. The note is + written for whoever reads the scenario, not for the person on the call, and it + tells them the mechanism that is being tested) + +GOOD Your is the same one you used last time. You do not remember the + exact address and would rather not look it up. + (now the caller has a reason to expect the agent to know, which is what makes + the agent's lookup worth testing, without being told the lookup exists) +``` + +Pre-agreeing to something the agent has not done yet is the most damaging form. "You have already + that the agent will " hands the agent a pass: the person confirms it +whether or not it happened. Write what they have done, never what they have done in response to an +action the agent has not taken. + **Use persona deliberately.** An accent, personality or characteristic belongs in `persona` only when it changes the conversational risk being exercised. A rude customer is a different scenario from a polite one only if the agent must handle that difference. Persona never contains the From 134a6c46d89802990b00b34a28da3d21c31ed794 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 07:11:30 +0530 Subject: [PATCH 32/41] fix(scenarios): give out-of-band steps a state the caller holds, not an answer written in advance --- .../harness/skills/write-scenarios/SKILL.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index c5ab0309..37ab0169 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -162,6 +162,30 @@ Pre-agreeing to something the agent has not done yet is the most damaging form. whether or not it happened. Write what they have done, never what they have done in response to an action the agent has not taken. +**Steps that happen outside the conversation need a state, not a response.** Some flows depend on +the person doing something the simulation cannot actually perform: following a link, checking +another device, reading a message. The temptation is to write the person's answer in advance, and +that is exactly the pass-handing form above, because the answer arrives whether or not the agent +ever asked. + +Give them a standing disposition instead, and let the agent's action trigger it: + +``` +BAD The agent will send you . Tell them you have + completed it when asked. + (the scenario is testing whether the agent sends it. This person confirms + completing it even in a run where nothing was ever sent) + +GOOD You have your with you and you are willing to follow anything you + are sent. You have not been sent anything yet. + (a state. If the agent sends it, this person can act on it and say so + truthfully. If the agent never does, they have nothing to confirm, and the + transcript shows the difference) +``` + +The closing sentence matters: stating what has **not** happened yet is what stops the person +assuming it has. + **Use persona deliberately.** An accent, personality or characteristic belongs in `persona` only when it changes the conversational risk being exercised. A rude customer is a different scenario from a polite one only if the agent must handle that difference. Persona never contains the From 4fe3d325020de544e1cc87e8cba3c87254292915 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 08:33:29 +0530 Subject: [PATCH 33/41] feat(simulate): add ALK_BACKGROUND_NOISE to silence every call on a run --- src/fi/alk/harness/background_noise.py | 16 ++++++++++++++++ src/fi/alk/harness/run/simulation.py | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/background_noise.py b/src/fi/alk/harness/background_noise.py index b6130245..03a4d201 100644 --- a/src/fi/alk/harness/background_noise.py +++ b/src/fi/alk/harness/background_noise.py @@ -29,6 +29,22 @@ _DEFAULT_BUILTIN = "OFFICE_AMBIENCE" +def enabled() -> bool: + """Whether any scenario may be heard through background noise on this run. + + On by default. Set ``ALK_BACKGROUND_NOISE=0`` to silence every call regardless of what the + scenarios ask for, which is the switch to reach for while diagnosing turn-taking: continuous + ambient audio under the caller competes with endpoint detection, and removing it separates a + scenario that fails from a call that could not be heard. + """ + return os.environ.get("ALK_BACKGROUND_NOISE", "1").strip().lower() not in ( + "0", + "off", + "false", + "no", + ) + + def source_for(environment: str = "", seed: str = "") -> str: """A background-noise source for a scenario. diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index f74ed1e8..14452cd7 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -611,7 +611,9 @@ def placed_once() -> tuple[int, dict[str, Any], str]: # A scenario that asks to be heard through background noise selects a clip for the # caller's environment; the voice engine mixes it under the caller. Cleared otherwise so # a previous call's noise never leaks into a quiet one. - noisy = getattr(scenario, "background_noise", False) + from ..background_noise import enabled as noise_enabled + + noisy = getattr(scenario, "background_noise", False) and noise_enabled() if noisy: from ..background_noise import source_for From 027d46b237adf5a6d5d9e4f420ac14670b68bc20 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 08:33:29 +0530 Subject: [PATCH 34/41] fix(scenarios): rename a duplicate folder name instead of dropping the scenario --- src/fi/alk/harness/scenarios.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index a06002d3..f45d557e 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -427,23 +427,27 @@ def watch(event: Any) -> None: def merged(written: list[list[Scenario]]) -> list[Scenario]: - """One suite out of several writers, with the collisions they could not see removed. + """One suite out of several writers, with folder-name collisions renamed rather than dropped. - The writers run blind to each other, so two can land on the same folder name or on the same - use case and branch. Both are dropped here rather than at save time, where the loser would - silently overwrite the winner's folder. + Asking for twenty scenarios has to return twenty. Two scenarios may legitimately share a use + case and a branch and still test different things, so sharing them is not a reason to discard + one; an earlier version dropped those and quietly returned eighteen. + + The one collision that cannot be tolerated is the folder name, because the folder is where a + scenario lives on disk and the loser would overwrite the winner. Those are given a numbered + suffix instead of being thrown away, so nothing generated is ever lost. """ suite: list[Scenario] = [] - names: set[str] = set() - pairs: set[tuple[str, str]] = set() + taken: set[str] = set() for batch in written: for one in batch: - pair = ((one.use_case or "").strip().lower(), (one.branch or "").strip().lower()) - if one.name in names or (pair[0] and pair in pairs): - continue - names.add(one.name) - if pair[0]: - pairs.add(pair) + if one.name in taken: + stem, suffix = one.name, 2 + while f"{stem}-{suffix}" in taken: + suffix += 1 + one = one.model_copy(update={"name": f"{stem}-{suffix}", "scenario_key": ""}) + logger.info("renamed a duplicate folder name to %s", one.name) + taken.add(one.name) suite.append(one) return suite From 1441f3aac37e3c731b101eae9ad43b831b37e794 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 10:33:19 +0530 Subject: [PATCH 35/41] fix(platform): send the scenario's ascii key rather than its folder name --- src/fi/alk/harness/platform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/platform.py b/src/fi/alk/harness/platform.py index 3d8a4169..5fe4a4a9 100644 --- a/src/fi/alk/harness/platform.py +++ b/src/fi/alk/harness/platform.py @@ -275,7 +275,11 @@ def persona_of(scenario: Any) -> dict[str, Any]: "name": str(persona.get("name") or getattr(scenario, "name", "") or "caller")[ :255 ], - "scenario_key": str(getattr(scenario, "name", "") or "")[:255], + # The scenario's own key, not its folder name: the key is ASCII-sanitised and falls back + # to a digest, which the name does not, and this value travels as an HTTP header. + "scenario_key": str( + getattr(scenario, "scenario_key", "") or getattr(scenario, "name", "") or "" + )[:255], "scenario_name": display_scenario_name(scenario), "role": str(persona.get("role") or persona.get("occupation") or "")[:255], "situation": str(getattr(scenario, "instruction", "") or ""), From 7cf0c18731372355a26f6b06ddb417f793723501 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 11:33:52 +0530 Subject: [PATCH 36/41] fix(simulate): close the caller prompt on its objective rather than on style advice --- src/fi/simulate/simulation/voice_prompt.py | 25 +++++++++++++++++++--- tests/test_voice_prompt.py | 5 ++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index cd68f795..4dd94b5e 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -314,12 +314,28 @@ def format_voice_persona( return "\n\n".join(sections) -def append_voice_execution_rules(prompt: str) -> str: +def _closing_anchor(objective: str) -> str: + """The last thing the caller reads. A rule given once at the top of a long prompt loses to the + last few turns as the call grows, so the objective and the precedence rule are restated here.""" + anchor = "\n\n---\n\n" + if objective.strip(): + anchor += f"**What you came for:** {objective.strip()}\n\n" + anchor += ( + "**Your instructions do not expire.** A rule you were given before the call started " + "applies at turn twenty exactly as it applied at turn one.\n" + ) + return anchor + + +def append_voice_execution_rules( + prompt: str, objective: str = "", *, anchor: bool = True +) -> str: prompt += "\n\n---\n\n" prompt += "# CONVERSATION EXECUTION RULES\n\n" prompt += "*These are internal instructions. Never reference or quote them in your responses.*\n\n" prompt += "## CRITICAL REMINDERS FOR THIS CONVERSATION\n\n" prompt += "Before each response, mentally confirm:\n" + prompt += "✓ What am I here to get, and what have I not done yet?\n" prompt += "✓ Am I speaking AS this person (not ABOUT them)?\n" prompt += "✓ Does this match my personality and communication style?\n" prompt += "✓ Am I using my accent and natural speech patterns?\n" @@ -345,7 +361,7 @@ def append_voice_execution_rules(prompt: str) -> str: prompt += "- Let the situation guide your behavior, not your narration\n" prompt += "- Only mention situational details if they naturally come up\n\n" prompt += "Be natural and conversational.\n" - return prompt + return (prompt + _closing_anchor(objective)) if anchor else prompt # The template the caller prompt is rendered from. The harness fills slots; it does not author @@ -411,12 +427,15 @@ def fill(match: "re.Match[str]") -> str: if tts_provider and tts_provider.strip().lower() == "cartesia": prompt += "\n\n" + CARTESIA_DELIVERY_CUES + # The generic style rules go first and the scenario's own instructions after them: whatever + # lands last survives a long call best, and the scenario's rules are the ones worth keeping. + prompt = append_voice_execution_rules(prompt, anchor=False) if additional_instructions and additional_instructions.strip(): prompt += ( "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" + additional_instructions.strip() ) - return append_voice_execution_rules(prompt) + return prompt + _closing_anchor(persona.outcome or "") def _channel_sentence(call_type: CallType, agent_name: str | None) -> str: diff --git a/tests/test_voice_prompt.py b/tests/test_voice_prompt.py index a06009a1..e006d6cb 100644 --- a/tests/test_voice_prompt.py +++ b/tests/test_voice_prompt.py @@ -59,6 +59,9 @@ def test_simulator_instructions_supplement_scenario_prompt() -> None: assert "Ask for an escalation" in prompt assert "Your specialist appointment was cancelled without notice." in prompt assert "Get a new appointment time and confirm the clinic location." in prompt - assert prompt.index("# ADDITIONAL SIMULATOR INSTRUCTIONS") < prompt.index( + # The call's own instructions come after the general rules, and the objective closes the + # prompt: what lands last is what survives a long conversation. + assert prompt.index("# ADDITIONAL SIMULATOR INSTRUCTIONS") > prompt.index( "# CONVERSATION EXECUTION RULES" ) + assert prompt.rstrip().endswith("applies at turn twenty exactly as it applied at turn one.") From 8f1c5c5a6e6ad1d5dd423fcc1c82a8b169e16cb7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 11:33:53 +0530 Subject: [PATCH 37/41] fix(simulate): make background noise opt in so a run with no environment is silent --- src/fi/alk/harness/background_noise.py | 19 ++++++++++--------- tests/test_harness.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/harness/background_noise.py b/src/fi/alk/harness/background_noise.py index 03a4d201..9a908587 100644 --- a/src/fi/alk/harness/background_noise.py +++ b/src/fi/alk/harness/background_noise.py @@ -32,16 +32,17 @@ def enabled() -> bool: """Whether any scenario may be heard through background noise on this run. - On by default. Set ``ALK_BACKGROUND_NOISE=0`` to silence every call regardless of what the - scenarios ask for, which is the switch to reach for while diagnosing turn-taking: continuous - ambient audio under the caller competes with endpoint detection, and removing it separates a - scenario that fails from a call that could not be heard. + Off unless ``ALK_BACKGROUND_NOISE`` opts in, so a run needs no environment at all to be + silent. Continuous ambient audio under the caller competes with endpoint detection, and calls + carrying it end earlier and on fewer turns, so silence is the setting a run should fall into + rather than the one it has to ask for. Opting in still only permits noise: a scenario that + asked for none stays silent either way. """ - return os.environ.get("ALK_BACKGROUND_NOISE", "1").strip().lower() not in ( - "0", - "off", - "false", - "no", + return os.environ.get("ALK_BACKGROUND_NOISE", "0").strip().lower() in ( + "1", + "on", + "true", + "yes", ) diff --git a/tests/test_harness.py b/tests/test_harness.py index 6049f282..9c624cc4 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6331,6 +6331,21 @@ def test_background_noise_is_decided_the_same_way_twice(): assert Scenario(name="x", background_noise="street").background_noise == "street" +def test_background_noise_needs_opting_in(monkeypatch): + """Silence is what a run with no environment set falls into: noise shortens calls, so it is + asked for rather than escaped. Anything unrecognised stays silent instead of turning it on.""" + from fi.alk.harness.background_noise import enabled + + monkeypatch.delenv("ALK_BACKGROUND_NOISE", raising=False) + assert enabled() is False + for off in ("", "0", "off", "false", "no", "ture"): + monkeypatch.setenv("ALK_BACKGROUND_NOISE", off) + assert enabled() is False, off + for on in ("1", "on", "TRUE", "yes"): + monkeypatch.setenv("ALK_BACKGROUND_NOISE", on) + assert enabled() is True, on + + def test_a_transcript_line_keeps_one_speaker_not_two(): """``agent: assistant: ...`` reached the judges as two speakers deep for every turn.""" from fi.alk.harness.run.simulation import _said From ce4498c6b229063721f0456b9ac8771cc8f1cae5 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 11:33:53 +0530 Subject: [PATCH 38/41] fix(simulate): warn when a missing cartesia key collapses every persona onto one voice --- src/fi/alk/harness/run/sdk_voice.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 9a274f0c..5b2a61ef 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -416,11 +416,18 @@ def _cartesia_voice_for(persona: dict) -> str: def _voice_providers() -> tuple[str, str]: """The (stt, tts) providers for the caller. An explicit env override wins; otherwise Cartesia when its key is present (richer, multi-language voices), else Deepgram aura.""" - default = ( - "cartesia" if os.environ.get("CARTESIA_API_KEY", "").strip() else "deepgram" - ) + keyed = bool(os.environ.get("CARTESIA_API_KEY", "").strip()) + default = "cartesia" if keyed else "deepgram" stt = os.environ.get("SIMULATOR_STT_PROVIDER", "").strip() or default tts = os.environ.get("SIMULATOR_TTS_PROVIDER", "").strip() or default + if tts == "deepgram" and not keyed and not os.environ.get("SIMULATOR_TTS_PROVIDER"): + # Deepgram aura is one voice, so every persona sounds the same and the accent, language + # and gender the scenario chose are silently dropped. The call still runs, which is why + # this has to be said out loud rather than left to whoever listens to the recording. + logger.warning( + "cartesia_key_missing_personas_share_one_voice", + extra={"tts": "deepgram/aura-asteria-en"}, + ) return stt, tts From a536744d97f9508680377e7debe235e60f359ed3 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 12:08:13 +0530 Subject: [PATCH 39/41] fix(simulate): close the caller prompt by naming who the caller is --- src/fi/simulate/simulation/voice_prompt.py | 15 +++++++++++++-- tests/test_voice_prompt.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index 4dd94b5e..320cbf98 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -314,10 +314,19 @@ def format_voice_persona( return "\n\n".join(sections) -def _closing_anchor(objective: str) -> str: +def _closing_anchor(objective: str, name: str = "") -> str: """The last thing the caller reads. A rule given once at the top of a long prompt loses to the last few turns as the call grows, so the objective and the precedence rule are restated here.""" anchor = "\n\n---\n\n" + who = name.strip() + if who: + # A caller that drifts answers as the agent and says the agent's own lines back, its own + # name included, which reads as the agent talking to itself and scores as a real turn. + anchor += ( + f"**You are {who}, the person on the customer's side of this call.** You never answer " + f"as the other side, never say their lines back to them, and never address {who}, " + "because that is you.\n\n" + ) if objective.strip(): anchor += f"**What you came for:** {objective.strip()}\n\n" anchor += ( @@ -435,7 +444,9 @@ def fill(match: "re.Match[str]") -> str: "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" + additional_instructions.strip() ) - return prompt + _closing_anchor(persona.outcome or "") + return prompt + _closing_anchor( + persona.outcome or "", str(_persona_data(persona).get("name") or "") + ) def _channel_sentence(call_type: CallType, agent_name: str | None) -> str: diff --git a/tests/test_voice_prompt.py b/tests/test_voice_prompt.py index e006d6cb..1a21f0b4 100644 --- a/tests/test_voice_prompt.py +++ b/tests/test_voice_prompt.py @@ -65,3 +65,14 @@ def test_simulator_instructions_supplement_scenario_prompt() -> None: "# CONVERSATION EXECUTION RULES" ) assert prompt.rstrip().endswith("applies at turn twenty exactly as it applied at turn one.") + + +def test_prompt_closes_by_naming_who_the_caller_is() -> None: + """A drifting caller answered as the agent and addressed itself by its own name, which reads + as the agent talking to itself. The identity is restated last, where it survives a long call.""" + prompt = build_voice_simulator_prompt(_persona(), call_type="inbound") + + tail = prompt.rsplit("---", 1)[-1] + assert "You are Priya" in tail + assert "never address Priya" in tail + assert tail.index("You are Priya") < tail.index("What you came for:") From 152e02caa3a595297902ebfbf8c637c8f26f205b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 16:13:20 +0530 Subject: [PATCH 40/41] fix(simulate): stop handing the caller the grader's pass question as its objective --- src/fi/alk/harness/run/call.py | 5 ++++- src/fi/alk/harness/run/sdk_voice.py | 7 +++---- src/fi/alk/harness/run/simulation.py | 5 ++++- src/fi/alk/harness/run/tools.py | 7 +++++-- src/fi/alk/harness/scenarios.py | 2 +- src/fi/alk/harness/skills/write-scenarios/SKILL.md | 7 ++++++- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/fi/alk/harness/run/call.py b/src/fi/alk/harness/run/call.py index 0bbb0df2..afe1e52b 100644 --- a/src/fi/alk/harness/run/call.py +++ b/src/fi/alk/harness/run/call.py @@ -109,7 +109,10 @@ def main(argv: list[str] | None = None) -> int: os.environ["HARNESS_SCENARIO"] = scenario.name # The caller prompt is a template the harness fills, never prose it composes, so # the generated template travels to the call with everything else. - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller is never handed the grader's pass question. `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 5b2a61ef..05d4604f 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -579,10 +579,9 @@ def _scenario() -> simulate.Scenario: simulate.Persona( persona=persona, situation=_required("HARNESS_INSTRUCTION"), - outcome=os.environ.get( - "HARNESS_OUTCOME", - "Complete the requested task and close naturally.", - ), + # Empty by default: the instruction already says what this person wants, in + # their own words. A generic objective here only competes with it. + outcome=os.environ.get("HARNESS_OUTCOME", ""), knowledge=knowledge, behavior_policy={ "disclosure_policy": 0.72, diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index 14452cd7..d205ce6d 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -590,7 +590,10 @@ def placed_once() -> tuple[int, dict[str, Any], str]: os.environ["HARNESS_VOICE_OUTPUT_ROOT"] = str(sdk_output.resolve()) os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller is never handed the grader's pass question: `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py index 442512e6..93e2f238 100644 --- a/src/fi/alk/harness/run/tools.py +++ b/src/fi/alk/harness/run/tools.py @@ -270,7 +270,7 @@ async def list_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) ) lines.append( - f"{one.name}{mark}\n tests: {one.tests or one.use_case or '—'}\n" + f"{one.name}{mark}\n passes when: {one.tests or one.use_case or '—'}\n" f" settled by code: {', '.join(settled) or 'none'}\n" f" judged: {', '.join(judged) or 'none'}" ) @@ -464,7 +464,10 @@ def placed() -> tuple[LiveRun, str, list[str], str]: # how a simulated caller behaves is not decided in two places. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller is never handed the grader's pass question. `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index f45d557e..5832b7b2 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -455,7 +455,7 @@ def merged(written: list[list[Scenario]]) -> list[Scenario]: def _suite_summary(suite: list[Scenario]) -> str: """The whole suite as a reviewer needs to see it: what each row claims to test.""" return "\n".join( - f" {one.name} | use case: {one.use_case} | branch: {one.branch} | tests: {one.tests}" + f" {one.name} | use case: {one.use_case} | branch: {one.branch} | passes when: {one.tests}" for one in suite ) diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 37ab0169..67b1fdf0 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -21,7 +21,12 @@ afterwards. name short identifier; it becomes this scenario's folder use_case which of the agent's use cases this belongs to branch what makes this one different from its siblings in that use case -tests one line: what this scenario is trying to find out +tests one line: the condition this scenario passes on. It is shown to people as + "passes when", so write it to complete that phrase. Both this and branch are + read by whoever looks at results, so write them about the agent's behaviour + and never about how the scenario was built. "synthetic", "seeded", + "setup_code", "fixture" and the like name your own machinery, not anything + the agent did, and they are noise in a report instruction what this person is trying to achieve, written to them, plus everything they need to pursue it without inventing anything persona who that person is: identity, communication style, languages/accent and characteristics From a18caae3be19669dd530346411a0ed3a62840f4a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Wed, 26 Aug 2026 21:34:56 +0530 Subject: [PATCH 41/41] fix(skills): drop the identity claim from the preamble and pin use_case to the contract wording --- src/fi/alk/harness/skills/harness.md | 35 ++++++++----------- .../harness/skills/write-scenarios/SKILL.md | 5 ++- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/fi/alk/harness/skills/harness.md b/src/fi/alk/harness/skills/harness.md index dd3597d2..4f410ea7 100644 --- a/src/fi/alk/harness/skills/harness.md +++ b/src/fi/alk/harness/skills/harness.md @@ -1,40 +1,35 @@ # The harness -You are a harness that builds test suites for AI agents. +You build test suites for AI agents, working with a person in a conversation they can see all of. -Somebody has an agent — a support assistant, a voice ordering system, something that books or -cancels or looks things up — and no reliable way to know whether it works. Reading its +Somebody has an agent, a support assistant, a voice ordering system, something that books or +cancels or looks things up, and no reliable way to know whether it works. Reading its transcripts tells you what it said, not whether what it said was true. Your job is to produce something better: a real environment the agent's tools act on, a set of tests that are provably worth running, and results that can be trusted because they were settled by code rather than by opinion. -You work with a person, in a conversation. They can see everything you do. - -**You are this thing, so speak as it.** Your tools refuse you sometimes; that is the design, and -it is still you being refused. "Two scenarios ended up sharing a use case, fixing them" is what -happened. "The harness needs unique use cases" is the same event narrated from outside, and it -reads as blaming a system you are not part of. Never refer to the harness in the third person, -and never explain your own tooling's rules as though they were somebody else's requirements: say -what you are doing about it. - -Where a limit genuinely is not yours, say whose it is and what to do: a stage you cannot reach -from here, a credential nobody has set, an agent that cannot be run without editing it. Those are -facts about the situation, not deflections. +**Write as the one doing the work.** "Two scenarios ended up sharing a use case, fixing them" is +what happened. "The harness needs unique use cases" is the same event narrated from outside, as +though a system you were not part of had imposed it on you. Report what you did and what you are +doing about it, including when a tool refuses you. Where a limit is genuinely someone else's, say +whose and what to do: a stage you cannot reach from here, a credential nobody has set, an agent +that cannot be run without editing it. Those are facts about the situation, not deflections. ## What you produce, in order -Four stages. Each one produces something the next needs, and each is a conversation you can be -interrupted in, corrected in, and resumed in. +Each stage produces something the next needs, and each is a conversation you can be interrupted +in, corrected in, and resumed in. **1. Understand.** Read the agent's source and write down what is verifiably true about it: the tools it really has with their exact argument names and permitted values, the rules it obeys, what it depends on, its data, and what it is for. This is the contract, and everything afterwards is confined to it. -**2. Build the environment.** From that contract, build the world the agent acts in — a database, -a service, whatever its tools need — so that every call it makes resolves against something real -and gets a truthful answer, including a truthful refusal. Also written here: the prompt for the +**2. Build or provision the environment.** The world the agent acts in, so that every call it +makes resolves against something real and gets a truthful answer, including a truthful refusal. +Either build it from the contract, a database, a service, whatever its tools need, or provision +the runtime the agent already ships, when it ships one. Also written here: the prompt for the person the agent talks to, and the catalogue of named sub-goals the agent can be checked on. **3. Write the scenarios.** Each one changes the world a little, gives the person a task, and diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 67b1fdf0..6e49a593 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -19,7 +19,10 @@ afterwards. ``` name short identifier; it becomes this scenario's folder -use_case which of the agent's use cases this belongs to +use_case which of the agent's use cases this belongs to, copied from the contract + word for word. Not paraphrased, not shortened, not reworded to fit this + scenario: results are grouped by matching this string exactly, so a + rewording silently becomes a group of its own branch what makes this one different from its siblings in that use case tests one line: the condition this scenario passes on. It is shown to people as "passes when", so write it to complete that phrase. Both this and branch are